Skip to content

Commit 764297b

Browse files
gh-74876: Add copyreg.Proxy
A copyreg.Proxy is pickled as the wrapped object, which must be strongly referenced elsewhere in the same pickle. It replaces the native support of weak references in pickle: weakref.ref, WeakMethod, WeakSet, WeakValueDictionary and WeakKeyDictionary are now reduced using Proxy. The copy module supports Proxy too. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent b94fdb7 commit 764297b

14 files changed

Lines changed: 279 additions & 182 deletions

File tree

‎Doc/library/copyreg.rst‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,30 @@ Such constructors may be factory functions or class instances.
3939
object or subclass of :class:`pickle.Pickler` can also be used for
4040
declaring reduction functions.
4141

42+
43+
.. class:: Proxy(obj)
44+
45+
A wrapper for an object which is pickled weakly.
46+
Reduction functions can use it to pickle objects
47+
to which the reconstructed object only holds weak references.
48+
For example, a weak reference is reduced to
49+
``(weakref.ref, (Proxy(referent),))``.
50+
51+
A proxy is pickled as the wrapped object *obj*,
52+
which is unpickled in its place.
53+
*obj* must also be strongly referenced elsewhere in the same pickle;
54+
otherwise pickling raises :exc:`pickle.PicklingError`.
55+
56+
:func:`copy.copy` of a proxy returns *obj*,
57+
and :func:`copy.deepcopy` returns the deep copy of *obj*.
58+
59+
.. attribute:: obj
60+
61+
The wrapped object.
62+
63+
.. versionadded:: next
64+
65+
4266
Example
4367
-------
4468

‎Doc/library/weakref.rst‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -276,8 +276,10 @@ These have the same issues as the :meth:`WeakKeyDictionary.keyrefs` method.
276276

277277
A :class:`!WeakSet` can be pickled if the elements are strongly
278278
referenced elsewhere in the same pickle (see :class:`ref`).
279-
A copy or a deep copy shares the elements with the original;
280-
user-defined attributes are copied or deep-copied.
279+
A shallow copy contains the same elements as the original.
280+
A deep copy refers to the deep copies of the elements,
281+
so they must be deep-copied together with it;
282+
its user-defined attributes are deep-copied.
281283

282284
.. versionchanged:: next
283285
:class:`!WeakSet` can now be pickled and copied.
@@ -314,7 +316,9 @@ These have the same issues as the :meth:`WeakKeyDictionary.keyrefs` method.
314316
A :class:`!WeakMethod` can be copied,
315317
and pickled if the object the method is bound to is strongly referenced
316318
elsewhere in the same pickle (see :class:`ref`).
317-
A copy or a deep copy is an equal weak method to the same object.
319+
A copy is an equal weak method bound to the same object.
320+
A deep copy is bound to the deep copy of the object,
321+
so it must be deep-copied together with it.
318322

319323
.. versionadded:: 3.4
320324

‎Doc/whatsnew/3.16.rst‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,17 @@ concurrent.futures
193193
(Contributed by xzmeng and Serhiy Storchaka in :gh:`108518`.)
194194

195195

196+
copyreg
197+
-------
198+
199+
* Add :class:`copyreg.Proxy`, a wrapper that is pickled as the wrapped object.
200+
Reduction functions can use it to pickle objects
201+
to which the reconstructed object only holds weak references.
202+
The wrapped object must be strongly referenced elsewhere in the same pickle;
203+
otherwise pickling raises :exc:`pickle.PicklingError`.
204+
(Contributed by Serhiy Storchaka in :gh:`74876`.)
205+
206+
196207
csv
197208
---
198209

‎Lib/_weakrefset.py‎

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
# by abc.py to load everything else at startup.
44

55
from _weakref import ref
6+
from copyreg import Proxy
67
from types import GenericAlias
78

89
__all__ = ['WeakSet']
@@ -44,14 +45,13 @@ def __contains__(self, item):
4445
return wr in self.data
4546

4647
def __reduce__(self):
47-
# Pickle the elements as weak references; data and _remove are
48-
# rebuilt by the reconstructor. Copying shares the elements.
48+
# The elements must be strongly referenced elsewhere in the pickle.
49+
# data and _remove are rebuilt by the constructor.
4950
dict, slots = self.__getstate__()
5051
for name in ('data', '_remove'):
5152
slots.pop(name, None)
5253
state = (dict, slots) if dict or slots else None
53-
return (_restore_weakset, (self.__class__, [ref(item) for item in self]),
54-
state)
54+
return (self.__class__, ([Proxy(item) for item in self],), state)
5555

5656
def add(self, item):
5757
self.data.add(ref(item, self._remove))
@@ -62,6 +62,8 @@ def clear(self):
6262
def copy(self):
6363
return self.__class__(self)
6464

65+
__copy__ = copy
66+
6567
def pop(self):
6668
while True:
6769
try:
@@ -156,10 +158,3 @@ def __repr__(self):
156158
return repr(self.data)
157159

158160
__class_getitem__ = classmethod(GenericAlias)
159-
160-
161-
def _restore_weakset(cls, refs):
162-
self = cls.__new__(cls)
163-
WeakSet.__init__(self)
164-
self.update(item for r in refs for item in [r()] if item is not None)
165-
return self

‎Lib/copy.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ class instances).
5151

5252
import types
5353
import weakref
54-
from copyreg import dispatch_table
54+
from copyreg import dispatch_table, Proxy
5555

5656
class Error(Exception):
5757
pass
@@ -71,6 +71,8 @@ def copy(x):
7171
return x
7272
if cls in _copy_builtin_containers:
7373
return cls.copy(x)
74+
if cls is Proxy:
75+
return x.obj
7476

7577

7678
if issubclass(cls, type):
@@ -221,6 +223,10 @@ def _deepcopy_method(x, memo): # Copy instance methods
221223
return type(x)(x.__func__, deepcopy(x.__self__, memo))
222224
d[types.MethodType] = _deepcopy_method
223225

226+
def _deepcopy_proxy(x, memo, deepcopy=deepcopy):
227+
return deepcopy(x.obj, memo)
228+
d[Proxy] = _deepcopy_proxy
229+
224230
del d
225231

226232
def _keep_alive(x, memo):

‎Lib/copyreg.py‎

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
C, not for instances of user-defined classes.
55
"""
66

7-
__all__ = ["pickle", "constructor",
7+
__all__ = ["pickle", "constructor", "Proxy",
88
"add_extension", "remove_extension", "clear_extension_cache"]
99

1010
dispatch_table = {}
@@ -23,6 +23,36 @@ def constructor(object):
2323
if not callable(object):
2424
raise TypeError("constructors must be callable")
2525

26+
class Proxy:
27+
"""Wrapper for an object that is pickled as a weak reference.
28+
29+
Pickling a Proxy pickles the wrapped object, which is unpickled in
30+
its place. Like the referent of a weak reference, the wrapped object
31+
must also be strongly referenced elsewhere in the same pickle,
32+
otherwise pickling fails. This lets a reduction function pickle
33+
objects that the reconstructed object will only refer to weakly.
34+
"""
35+
36+
__slots__ = ('obj',)
37+
38+
def __init__(self, obj):
39+
self.obj = obj
40+
41+
def __repr__(self):
42+
return f'{self.__class__.__qualname__}({self.obj!r})'
43+
44+
class _Reducer:
45+
# An object that is pickled and copied as the result of a call:
46+
# reduce is a (callable, args) pair returned from __reduce__().
47+
48+
__slots__ = ('_reduce',)
49+
50+
def __init__(self, reduce):
51+
self._reduce = reduce
52+
53+
def __reduce__(self):
54+
return self._reduce
55+
2656
# Example: provide pickling support for complex numbers.
2757

2858
def pickle_complex(c):
@@ -41,6 +71,21 @@ def pickle_super(obj):
4171

4272
pickle(super, pickle_super)
4373

74+
from _weakref import ref as _ref
75+
76+
# All dead weak references are interchangeable, so they are pickled as
77+
# references to this set. It is unpickled as a new set which dies with
78+
# the unpickler's memo, and all dead references as the same reference.
79+
_dead_referent = set()
80+
81+
def pickle_weakref(r):
82+
obj = r()
83+
if obj is None:
84+
return _ref, (_dead_referent,)
85+
return _ref, (Proxy(obj),)
86+
87+
pickle(_ref, pickle_weakref)
88+
4489
# Support for pickling new-style objects
4590

4691
def _reconstructor(cls, base, state):

‎Lib/pickle.py‎

Lines changed: 15 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
"""
2424

2525
from types import FunctionType
26-
from copyreg import dispatch_table
26+
from copyreg import dispatch_table, Proxy
2727
from copyreg import _extension_registry, _inverted_registry, _extension_cache
2828
from itertools import batched
2929
from functools import partial
@@ -32,7 +32,6 @@
3232
from struct import pack, unpack
3333
import io
3434
import codecs
35-
import weakref
3635
import _compat_pickle
3736

3837
__all__ = ["PickleError", "PicklingError", "UnpicklingError", "Pickler",
@@ -487,7 +486,7 @@ def __init__(self, file, protocol=None, *, fix_imports=True,
487486
self._write_large_bytes = self.framer.write_large_bytes
488487
self.memo = {}
489488
# Ids of objects reached so far only through weak references; any
490-
# left at the end of dump() is an error. See save_weakref.
489+
# left at the end of dump() is an error. See save_proxy.
491490
self._weak_only = set()
492491
self.proto = int(protocol)
493492
self.bin = protocol >= 1
@@ -1286,39 +1285,20 @@ def save_type(self, obj):
12861285
dispatch[FunctionType] = save_global
12871286
dispatch[type] = save_type
12881287

1289-
def save_weakref(self, obj):
1290-
# Reconstruct as weakref.ref(referent). The referent is saved
1291-
# *weakly* (no anchoring, see save()) and tagged weak-only only after
1288+
def save_proxy(self, obj):
1289+
# A Proxy is pickled as its wrapped object, saved *weakly* (no
1290+
# anchoring, see save()). The object is tagged weak-only only after
12921291
# its whole subtree, so a strong cycle inside it cannot anchor it.
1293-
referent = obj()
1294-
if referent is None:
1295-
# All dead references are interchangeable: pickle by name as the
1296-
# weakref._dead_ref singleton.
1297-
self.write(GLOBAL + b'weakref\n_dead_ref\n')
1298-
return
1299-
1300-
save = self.save
1301-
write = self.write
1302-
save(weakref.ref)
1303-
# A referent already in the memo is anchored or already tracked;
1304-
# otherwise it is tagged below, until a strong save of it.
1305-
anchored = id(referent) in self.memo
1306-
# Build the 1-tuple (referent,); TUPLE1 needs protocol 2.
1307-
if self.proto < 2:
1308-
write(MARK)
1309-
save(referent, weak=True)
1310-
if not anchored and id(referent) in self.memo:
1311-
self._weak_only.add(id(referent))
1312-
write(TUPLE1 if self.proto >= 2 else TUPLE)
1313-
write(REDUCE)
1314-
# The referent's subtree may have pickled this very (cached) weakref
1315-
# already; reuse the memoized one.
1316-
if id(obj) in self.memo:
1317-
write(POP + self.get(self.memo[id(obj)][0]))
1318-
else:
1319-
self.memoize(obj)
1320-
1321-
dispatch[weakref.ReferenceType] = save_weakref
1292+
# An object already in the memo is anchored or already tracked;
1293+
# otherwise it stays tagged until a strong save of it.
1294+
obj = obj.obj
1295+
i = id(obj)
1296+
anchored = i in self.memo
1297+
self.save(obj, weak=True)
1298+
if not anchored and i in self.memo:
1299+
self._weak_only.add(i)
1300+
1301+
dispatch[Proxy] = save_proxy
13221302

13231303

13241304
# Unpickling machinery

‎Lib/test/pickletester.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1866,6 +1866,17 @@ def test_weakref_reused_pickler(self):
18661866
obj2, wr2 = pickle.loads(f.getvalue())
18671867
self.assertIs(wr2(), obj2)
18681868

1869+
def test_proxy_without_strong_ref(self):
1870+
obj = Object()
1871+
for proto in protocols:
1872+
with self.subTest(proto=proto):
1873+
self.assertRaises(pickle.PicklingError, self.dumps,
1874+
copyreg.Proxy(obj), proto)
1875+
# Another Proxy is not a strong reference.
1876+
self.assertRaises(pickle.PicklingError, self.dumps,
1877+
[copyreg.Proxy(obj), copyreg.Proxy(obj)],
1878+
proto)
1879+
18691880
def test_bad_reduce_result(self):
18701881
obj = REX([print, ()])
18711882
for proto in protocols:
@@ -3684,6 +3695,23 @@ def test_dead_weakref(self):
36843695
support.gc_collect()
36853696
self.assertIsNone(a())
36863697

3698+
def test_proxy(self):
3699+
# A Proxy is unpickled as the wrapped object.
3700+
obj = Object()
3701+
for proto in protocols:
3702+
with self.subTest(proto=proto):
3703+
x, y = self.loads(self.dumps([obj, copyreg.Proxy(obj)], proto))
3704+
self.assertIs(type(x), Object)
3705+
self.assertIs(y, x)
3706+
# Identity holds regardless of pickling order.
3707+
x, y = self.loads(self.dumps([copyreg.Proxy(obj), obj], proto))
3708+
self.assertIs(type(x), Object)
3709+
self.assertIs(y, x)
3710+
# Objects which are not memoized need no strong reference.
3711+
x = self.loads(self.dumps([copyreg.Proxy(None),
3712+
copyreg.Proxy(42)], proto))
3713+
self.assertEqual(x, [None, 42])
3714+
36873715
def test_newobj_overridden_new(self):
36883716
# Test that Python class with C implemented __new__ is pickleable
36893717
for proto in protocols:

‎Lib/test/test_copy.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -869,6 +869,24 @@ def test_copy_weakref(self):
869869
def test_deepcopy_weakref(self):
870870
self._check_weakref(copy.deepcopy)
871871

872+
def test_copy_proxy(self):
873+
class C:
874+
pass
875+
x = C()
876+
self.assertIs(copy.copy(copyreg.Proxy(x)), x)
877+
878+
def test_deepcopy_proxy(self):
879+
# A Proxy is replaced by the deep copy of the wrapped object.
880+
class C:
881+
pass
882+
x = C()
883+
y, z = copy.deepcopy([x, copyreg.Proxy(x)])
884+
self.assertIsNot(y, x)
885+
self.assertIs(z, y)
886+
z, y = copy.deepcopy([copyreg.Proxy(x), x])
887+
self.assertIsNot(y, x)
888+
self.assertIs(z, y)
889+
872890
def _check_copy_weakdict(self, _dicttype):
873891
class C(object):
874892
pass

0 commit comments

Comments
 (0)