Merge remote-tracking branch 'upstream/main' into pr-148445

Co-authored-by: Muse Code (Meta Muse Spark)

Author: ambv

Doc/library/ast.rst

@@ -48,10 +48,10 @@ Node classes
    module and re-exported in :mod:`!ast`.
 
    There is one class defined for each left-hand side symbol in the abstract
-   grammar (for example, :class:`ast.stmt` or :class:`ast.expr`).  In addition,
+   grammar (for example, :class:`!ast.stmt` or :class:`!ast.expr`).  In addition,
    there is one class defined for each constructor on the right-hand side; these
    classes inherit from the classes for the left-hand side trees.  For example,
-   :class:`ast.BinOp` inherits from :class:`ast.expr`.  For production rules
+   :class:`ast.BinOp` inherits from :class:`!ast.expr`.  For production rules
    with alternatives (aka "sums"), the left-hand side class is abstract: only
    instances of specific constructor nodes are ever created.
 
@@ -65,7 +65,7 @@ Node classes
 
       Each instance of a concrete class has one attribute for each child node,
       of the type as defined in the grammar.  For example, :class:`ast.BinOp`
-      instances have an attribute :attr:`left` of type :class:`ast.expr`.
+      instances have an attribute :attr:`!left` of type :class:`!ast.expr`.
 
       If these attributes are marked as optional in the grammar (using a
       question mark), the value might be ``None``.  If the attributes can have
@@ -104,10 +104,11 @@ Node classes
       one can get the source segment of a one-line expression node using
       ``source_line[node.col_offset : node.end_col_offset]``.
 
-   The constructor of a class :class:`ast.T` parses its arguments as follows:
+   The constructors of AST node classes parse their arguments as follows:
 
    * If there are positional arguments, there must be as many as there are items
-     in :attr:`T._fields`; they will be assigned as attributes of these names.
+     in :attr:`!_fields`; they will be assigned to attributes with
+     the corresponding names.
    * If there are keyword arguments, they will set the attributes of the same
      names to the given values.
 
@@ -2223,8 +2224,8 @@ Async and await
 
 .. note::
    When a string is parsed by :func:`ast.parse`, operator nodes (subclasses
-   of :class:`ast.operator`, :class:`ast.unaryop`, :class:`ast.cmpop`,
-   :class:`ast.boolop` and :class:`ast.expr_context`) on the returned tree
+   of :class:`!ast.operator`, :class:`!ast.unaryop`, :class:`!ast.cmpop`,
+   :class:`!ast.boolop` and :class:`!ast.expr_context`) on the returned tree
    will be singletons. Changes to one will be reflected in all other
    occurrences of the same value (for example, :class:`ast.Add`).
 

Doc/library/datetime.rst

@@ -2699,6 +2699,10 @@ convenience.
 |           | number, zero-padded to 6       | 999999                 |       |
 |           | digits.                        |                        |       |
 +-----------+--------------------------------+------------------------+-------+
+|  ``%Nf``  | Fraction of a second truncated | 000, 001, ..., 999     | \(5)  |
+|           | to *N* digits, where *N* is    | (``%3f``)              |       |
+|           | from 1 to 6.                   |                        |       |
++-----------+--------------------------------+------------------------+-------+
 | ``%:z``   | UTC offset in the form         | (empty), +00:00,       | \(6)  |
 |           | ``±HH:MM[:SS[.ffffff]]``       | -04:00, +10:30,        |       |
 |           | (empty string if the object is | +06:34:15,             |       |
@@ -2721,6 +2725,9 @@ differences between platforms in handling of unsupported format specifiers.
    ``%D``, ``%F``, ``%n``, ``%t``, and ``%:z`` were added for
    :meth:`~.datetime.strptime`.
 
+.. versionadded:: next
+   ``%Nf`` was added.
+
 
 Technical detail
 ^^^^^^^^^^^^^^^^
@@ -2833,7 +2840,8 @@ Notes:
 
 (5)
    When used with the :meth:`~.datetime.strptime` method, the ``%f`` directive
-   accepts from one to six digits and zero pads on the right. ``%f`` is
+   accepts from one to six digits and zero pads on the right, while ``%Nf``
+   accepts exactly *N* digits. ``%f`` is
    an extension to the set of format characters in the C standard (but
    implemented separately in datetime objects, and therefore always
    available).

Doc/library/pickle.rst

@@ -517,7 +517,11 @@ The following types can be pickled:
 * classes accessible from the top level of a module;
 
 * instances of such classes for which the result of calling :meth:`~object.__getstate__`
-  is picklable  (see section :ref:`pickle-inst` for details).
+  is picklable  (see section :ref:`pickle-inst` for details);
+
+* weak references and the weak containers from the :mod:`weakref` module,
+  provided that the weakly referenced objects are strongly referenced
+  elsewhere in the same pickle (see :class:`weakref.ref`).
 
 Attempts to pickle unpicklable objects will raise the :exc:`PicklingError`
 exception; when this happens, an unspecified number of bytes may have already

Doc/library/weakref.rst

@@ -126,6 +126,17 @@ See :ref:`__slots__ documentation <slots>` for details.
    Weak references are :ref:`generic <generics>` over the type of the object they
    reference.
 
+   A weak reference can be pickled
+   if the referent is strongly referenced elsewhere in the same pickle;
+   otherwise pickling raises :exc:`pickle.PicklingError`.
+   The unpickled weak reference refers to the unpickled referent.
+   A reference to a dead object unpickles as a reference to a dead object.
+   The *callback* is not preserved.
+
+   Weak references are atomic for the :mod:`copy` module:
+   :func:`copy.copy` and :func:`copy.deepcopy` return
+   the original weak reference without copying the referent.
+
    .. attribute:: __callback__
 
       This read-only attribute returns the callback currently associated to the
@@ -138,6 +149,9 @@ See :ref:`__slots__ documentation <slots>` for details.
    .. versionchanged:: 3.16
       Raise :exc:`!TypeError` if *callback* is not callable or ``None``.
 
+   .. versionchanged:: next
+      Weak references can now be pickled.
+
 
 .. function:: proxy(object[, callback])
 
@@ -203,9 +217,16 @@ See :ref:`__slots__ documentation <slots>` for details.
       >>> d[k2] = 2   # d = {k2: 2}
       >>> del k1      # d = {k2: 2}
 
+   A :class:`!WeakKeyDictionary` can be copied,
+   and pickled if the keys are strongly referenced
+   elsewhere in the same pickle (see :class:`ref`).
+
    .. versionchanged:: 3.9
       Added support for ``|`` and ``|=`` operators, as specified in :pep:`584`.
 
+   .. versionchanged:: next
+      :class:`!WeakKeyDictionary` can now be pickled.
+
 :class:`WeakKeyDictionary` objects have an additional method that
 exposes the internal references directly.  The references are not guaranteed to
 be "live" at the time they are used, so the result of calling the references
@@ -224,9 +245,16 @@ than needed.
    Mapping class that references values weakly.  Entries in the dictionary will be
    discarded when no strong reference to the value exists any more.
 
+   A :class:`!WeakValueDictionary` can be copied,
+   and pickled if the values are strongly referenced
+   elsewhere in the same pickle (see :class:`ref`).
+
    .. versionchanged:: 3.9
       Added support for ``|`` and ``|=`` operators, as specified in :pep:`584`.
 
+   .. versionchanged:: next
+      :class:`!WeakValueDictionary` can now be pickled.
+
 :class:`WeakValueDictionary` objects have the following additional methods.
 These have the same issues as the :meth:`WeakKeyDictionary.keyrefs` method.
 
@@ -246,6 +274,14 @@ These have the same issues as the :meth:`WeakKeyDictionary.keyrefs` method.
    Set class that keeps weak references to its elements.  An element will be
    discarded when no strong reference to it exists any more.
 
+   A :class:`!WeakSet` can be pickled if the elements are strongly
+   referenced elsewhere in the same pickle (see :class:`ref`).
+   A copy or a deep copy shares the elements with the original;
+   user-defined attributes are copied or deep-copied.
+
+   .. versionchanged:: next
+      :class:`!WeakSet` can now be pickled and copied.
+
 
 .. class:: WeakMethod(method[, callback])
 
@@ -275,8 +311,16 @@ These have the same issues as the :meth:`WeakKeyDictionary.keyrefs` method.
 
    *callback* is the same as the parameter of the same name to the :func:`ref` function.
 
+   A :class:`!WeakMethod` can be copied,
+   and pickled if the object the method is bound to is strongly referenced
+   elsewhere in the same pickle (see :class:`ref`).
+   A copy or a deep copy is an equal weak method to the same object.
+
    .. versionadded:: 3.4
 
+   .. versionchanged:: next
+      :class:`!WeakMethod` can now be pickled and copied.
+
 .. class:: finalize(obj, func, /, *args, **kwargs)
 
    Return a callable finalizer object which will be called when *obj*

Doc/whatsnew/3.16.rst

@@ -362,6 +362,16 @@ curses
   (Contributed by Serhiy Storchaka in :gh:`133031`.)
 
 
+datetime
+--------
+
+* Add the ``%Nf`` format code (where *N* is from 1 to 6) to
+  :meth:`datetime.datetime.strftime`, :meth:`datetime.time.strftime`,
+  :meth:`datetime.date.strftime`, :meth:`datetime.datetime.strptime` and
+  :meth:`datetime.time.strptime`.
+  (Contributed by Stan Ulbrych and Karoline Pauls in :gh:`157290`.)
+
+
 dbm.gnu
 -------
 
@@ -576,6 +586,17 @@ pydoc
   (Contributed by Serhiy Storchaka in :gh:`153906`.)
 
 
+pickle
+------
+
+* Weak references (:class:`weakref.ref`), :class:`weakref.WeakMethod`,
+  :class:`weakref.WeakSet`, :class:`weakref.WeakValueDictionary` and
+  :class:`weakref.WeakKeyDictionary` can now be pickled if the weakly
+  referenced objects are strongly referenced elsewhere in the same pickle;
+  otherwise pickling raises :exc:`pickle.PicklingError`.
+  :class:`!WeakMethod` and :class:`!WeakSet` can now also be copied.
+  (Contributed by Serhiy Storchaka in :gh:`74876`.)
+
 re
 --
 

Lib/_pydatetime.py

@@ -237,6 +237,13 @@ def _wrap_strftime(object, format, timetuple):
                         freplace = '%06d' % getattr(object,
                                                     'microsecond', 0)
                     newformat.append(freplace)
+                elif ch in '123456' and format[i:i+1] == 'f':
+                    i += 1
+                    if freplace is None:
+                        freplace = '%06d' % getattr(object,
+                                                    'microsecond', 0)
+                    precision = int(ch)
+                    newformat.append(freplace[:precision])
                 elif ch == 'z':
                     if zreplace is None:
                         if hasattr(object, "utcoffset"):

Lib/_pyrepl/_module_completer.py

@@ -83,7 +83,11 @@ def get_completions(
         if not result:
             return None
         try:
-            return self.complete(*result, include_values=include_values)
+            names, values, action = self.complete(
+                *result, include_values=include_values
+            )
+            names = [self._remove_separated_words(line, c) for c in names]
+            return names, values, action
         except Exception:
             # Some unexpected error occurred, make it look like
             # no completions are available
@@ -343,6 +347,26 @@ def _do_import() -> str | None:
 
         return (prompt, _do_import)
 
+    def _remove_separated_words(self, line: str, completion: str) -> str:
+        """Remove completion parts imputed as separate words.
+
+        Needed because the completer insert any completion provided
+        by replacing the stem (eg. word) currently imputed, if any.
+
+        Examples:
+            - 'import foo.', 'foo.bar'    -> 'foo.bar'
+            - 'import foo  .', 'foo.bar'  -> '.bar'
+            - 'from foo import ', 'bar'   -> 'bar'
+            - 'import x.x .x.', 'x.x.x.x' -> '.x.x'
+        """
+        last_word = line.split(" ")[-1]
+        if not last_word:
+            return completion
+        index = completion.rfind(last_word)
+        if index > 0:
+            return completion[index:]
+        return completion
+
 
 class ImportParser:
     """

Lib/_strptime.py

@@ -386,6 +386,8 @@ def __init__(self, locale_time=None):
             't': r'\s*',
             '%': '%',
         }
+        for n in range(1, 7):
+            mapping[f'{n}f'] = rf"(?P<f>[0-9]{{{n}}})"
         if self.locale_time.LC_alt_digits is None:
             for d in 'dmyCHIMS':
                 mapping['O' + d] = r'(?P<%s>\d\d|\d| \d)' % d

Lib/_weakrefset.py

@@ -9,6 +9,10 @@
 
 
 class WeakSet:
+    # Slots give __getstate__() a uniform (dict, slots) shape for __reduce__.
+    # __dict__ carries user attributes; add() needs __weakref__.
+    __slots__ = '__dict__', '__weakref__', 'data', '_remove'
+
     def __init__(self, data=None):
         self.data = set()
 
@@ -40,7 +44,14 @@ def __contains__(self, item):
         return wr in self.data
 
     def __reduce__(self):
-        return self.__class__, (list(self),), self.__getstate__()
+        # Pickle the elements as weak references; data and _remove are
+        # rebuilt by the reconstructor.  Copying shares the elements.
+        dict, slots = self.__getstate__()
+        for name in ('data', '_remove'):
+            slots.pop(name, None)
+        state = (dict, slots) if dict or slots else None
+        return (_restore_weakset, (self.__class__, [ref(item) for item in self]),
+                state)
 
     def add(self, item):
         self.data.add(ref(item, self._remove))
@@ -145,3 +156,10 @@ def __repr__(self):
         return repr(self.data)
 
     __class_getitem__ = classmethod(GenericAlias)
+
+
+def _restore_weakset(cls, refs):
+    self = cls.__new__(cls)
+    WeakSet.__init__(self)
+    self.update(item for r in refs for item in [r()] if item is not None)
+    return self

Lib/bdb.py

@@ -1,12 +1,12 @@
 """Debugger basics"""
 
-import fnmatch
-import sys
-import threading
 import os
-import weakref
+import sys
 from contextlib import contextmanager
 from inspect import CO_GENERATOR, CO_COROUTINE, CO_ASYNC_GENERATOR
+lazy import fnmatch
+lazy import threading
+lazy import weakref
 
 __all__ = ["BdbQuit", "Bdb", "Breakpoint"]
 
@@ -601,6 +601,13 @@ def set_trace(self, frame=None):
 
         If frame is not specified, debugging starts from caller's frame.
         """
+
+        if self.enterframe is not None:
+            # If self.enterframe is not None, it means we are already in
+            # dispatched functions. Tracing is disabled when calling a
+            # trace function, so we just skip this set_trace() call.
+            return
+
         self.stop_trace()
         if frame is None:
             frame = sys._getframe().f_back