Merge branch 'main' into main

Author: jonathandung

Doc/c-api/list.rst

@@ -74,11 +74,25 @@ List Objects
    Like :c:func:`PyList_GetItemRef`, but returns a
    :term:`borrowed reference` instead of a :term:`strong reference`.
 
+   .. note::
+
+      In the :term:`free-threaded build`, the returned
+      :term:`borrowed reference` may become invalid if another thread modifies
+      the list concurrently. Prefer :c:func:`PyList_GetItemRef`, which returns
+      a :term:`strong reference`.
+
 
 .. c:function:: PyObject* PyList_GET_ITEM(PyObject *list, Py_ssize_t i)
 
    Similar to :c:func:`PyList_GetItem`, but without error checking.
 
+   .. note::
+
+      In the :term:`free-threaded build`, the returned
+      :term:`borrowed reference` may become invalid if another thread modifies
+      the list concurrently. Prefer :c:func:`PyList_GetItemRef`, which returns
+      a :term:`strong reference`.
+
 
 .. c:function:: int PyList_SetItem(PyObject *list, Py_ssize_t index, PyObject *item)
 
@@ -108,6 +122,14 @@ List Objects
       is being replaced; any reference in *list* at position *i* will be
       leaked.
 
+   .. note::
+
+      In the :term:`free-threaded build`, this macro has no internal
+      synchronization. It is normally only used to fill in new lists where no
+      other thread has a reference to the list. If the list may be shared,
+      use :c:func:`PyList_SetItem` instead, which uses a :term:`per-object
+      lock`.
+
 
 .. c:function:: int PyList_Insert(PyObject *list, Py_ssize_t index, PyObject *item)
 
@@ -138,6 +160,12 @@ List Objects
    Return ``0`` on success, ``-1`` on failure.  Indexing from the end of the
    list is not supported.
 
+   .. note::
+
+      In the :term:`free-threaded build`, when *itemlist* is a :class:`list`,
+      both *list* and *itemlist* are locked for the duration of the operation.
+      For other iterables (or ``NULL``), only *list* is locked.
+
 
 .. c:function:: int PyList_Extend(PyObject *list, PyObject *iterable)
 
@@ -150,6 +178,14 @@ List Objects
 
    .. versionadded:: 3.13
 
+   .. note::
+
+      In the :term:`free-threaded build`, when *iterable* is a :class:`list`,
+      :class:`set`, :class:`dict`, or dict view, both *list* and *iterable*
+      (or its underlying dict) are locked for the duration of the operation.
+      For other iterables, only *list* is locked; *iterable* may be
+      concurrently modified by another thread.
+
 
 .. c:function:: int PyList_Clear(PyObject *list)
 
@@ -168,6 +204,14 @@ List Objects
    Sort the items of *list* in place.  Return ``0`` on success, ``-1`` on
    failure.  This is equivalent to ``list.sort()``.
 
+   .. note::
+
+      In the :term:`free-threaded build`, element comparison via
+      :meth:`~object.__lt__` can execute arbitrary Python code, during which
+      the :term:`per-object lock` may be temporarily released. For built-in
+      types (:class:`str`, :class:`int`, :class:`float`), the lock is not
+      released during comparison.
+
 
 .. c:function:: int PyList_Reverse(PyObject *list)
 

Doc/data/threadsafety.dat

@@ -17,3 +17,59 @@
 PyMutex_Lock:shared:
 PyMutex_Unlock:shared:
 PyMutex_IsLocked:atomic:
+
+# List objects (Doc/c-api/list.rst)
+
+# Type checks - read ob_type pointer, always safe
+PyList_Check:atomic:
+PyList_CheckExact:atomic:
+
+# Creation - pure allocation, no shared state
+PyList_New:atomic:
+
+# Size - uses atomic load on free-threaded builds
+PyList_Size:atomic:
+PyList_GET_SIZE:atomic:
+
+# Strong-reference lookup - lock-free with atomic ops
+PyList_GetItemRef:atomic:
+
+# Borrowed-reference lookups - no locking; returned borrowed
+# reference is unsafe in free-threaded builds without
+# external synchronization
+PyList_GetItem:compatible:
+PyList_GET_ITEM:compatible:
+
+# Single-item mutations - hold per-object lock for duration;
+# appear atomic to lock-free readers
+PyList_SetItem:atomic:
+PyList_Append:atomic:
+
+# Insert - protected by per-object critical section; shifts
+# elements so lock-free readers may observe intermediate states
+PyList_Insert:shared:
+
+# Initialization macro - no synchronization; normally only used
+# to fill in new lists where there is no previous content
+PyList_SET_ITEM:compatible:
+
+# Bulk operations - hold per-object lock for duration
+PyList_GetSlice:atomic:
+PyList_AsTuple:atomic:
+PyList_Clear:atomic:
+
+# Reverse - protected by per-object critical section; swaps
+# elements so lock-free readers may observe intermediate states
+PyList_Reverse:shared:
+
+# Slice assignment - lock target list; also lock source when it
+# is a list
+PyList_SetSlice:shared:
+
+# Sort - per-object lock held; comparison callbacks may execute
+# arbitrary Python code
+PyList_Sort:shared:
+
+# Extend - lock target list; also lock source when it is a
+# list, set, or dict
+PyList_Extend:shared:

Doc/tutorial/errors.rst

@@ -549,9 +549,9 @@ caught like any other exception. ::
    >>> try:
    ...     f()
    ... except Exception as e:
-   ...     print(f'caught {type(e)}: e')
+   ...     print(f'caught {type(e)}: {e}')
    ...
-   caught <class 'ExceptionGroup'>: e
+   caught <class 'ExceptionGroup'>: there were problems (2 sub-exceptions)
    >>>
 
 By using ``except*`` instead of ``except``, we can selectively

Doc/whatsnew/3.15.rst

@@ -221,6 +221,11 @@ The following standard library modules have been updated to accept
 :func:`eval` and :func:`exec` accept :class:`!frozendict` for *globals*, and
 :func:`type` and :meth:`str.maketrans` accept :class:`!frozendict` for *dict*.
 
+Code checking for :class:`dict` type using ``isinstance(arg, dict)`` can be
+updated to ``isinstance(arg, (dict, frozendict))`` to accept also the
+:class:`!frozendict` type, or to ``isinstance(arg, collections.abc.Mapping)``
+to accept also other mapping types such as :class:`~types.MappingProxyType`.
+
 .. seealso:: :pep:`814` for the full specification and rationale.
 
 (Contributed by Victor Stinner and Donghee Na in :gh:`141510`.)

Include/internal/pycore_optimizer.h

@@ -159,10 +159,87 @@ PyAPI_FUNC(_PyExecutorObject*) _Py_GetExecutor(PyCodeObject *code, int offset);
 
 int _Py_ExecutorInit(_PyExecutorObject *, const _PyBloomFilter *);
 void _Py_ExecutorDetach(_PyExecutorObject *);
-void _Py_BloomFilter_Init(_PyBloomFilter *);
-void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj);
 PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj);
 
+/* We use a bloomfilter with k = 6, m = 256
+ * The choice of k and the following constants
+ * could do with a more rigorous analysis,
+ * but here is a simple analysis:
+ *
+ * We want to keep the false positive rate low.
+ * For n = 5 (a trace depends on 5 objects),
+ * we expect 30 bits set, giving a false positive
+ * rate of (30/256)**6 == 2.5e-6 which is plenty
+ * good enough.
+ *
+ * However with n = 10 we expect 60 bits set (worst case),
+ * giving a false positive of (60/256)**6 == 0.0001
+ *
+ * We choose k = 6, rather than a higher number as
+ * it means the false positive rate grows slower for high n.
+ *
+ * n = 5, k = 6 => fp = 2.6e-6
+ * n = 5, k = 8 => fp = 3.5e-7
+ * n = 10, k = 6 => fp = 1.6e-4
+ * n = 10, k = 8 => fp = 0.9e-4
+ * n = 15, k = 6 => fp = 0.18%
+ * n = 15, k = 8 => fp = 0.23%
+ * n = 20, k = 6 => fp = 1.1%
+ * n = 20, k = 8 => fp = 2.3%
+ *
+ * The above analysis assumes perfect hash functions,
+ * but those don't exist, so the real false positive
+ * rates may be worse.
+ */
+
+#define _Py_BLOOM_FILTER_K 6
+#define _Py_BLOOM_FILTER_SEED 20221211
+
+static inline uint64_t
+address_to_hash(void *ptr) {
+    assert(ptr != NULL);
+    uint64_t uhash = _Py_BLOOM_FILTER_SEED;
+    uintptr_t addr = (uintptr_t)ptr;
+    for (int i = 0; i < SIZEOF_VOID_P; i++) {
+        uhash ^= addr & 255;
+        uhash *= (uint64_t)PyHASH_MULTIPLIER;
+        addr >>= 8;
+    }
+    return uhash;
+}
+
+static inline void
+_Py_BloomFilter_Init(_PyBloomFilter *bloom)
+{
+    for (int i = 0; i < _Py_BLOOM_FILTER_WORDS; i++) {
+        bloom->bits[i] = 0;
+    }
+}
+
+static inline void
+_Py_BloomFilter_Add(_PyBloomFilter *bloom, void *ptr)
+{
+    uint64_t hash = address_to_hash(ptr);
+    assert(_Py_BLOOM_FILTER_K <= 8);
+    for (int i = 0; i < _Py_BLOOM_FILTER_K; i++) {
+        uint8_t bits = hash & 255;
+        bloom->bits[bits >> _Py_BLOOM_FILTER_WORD_SHIFT] |=
+            (_Py_bloom_filter_word_t)1 << (bits & (_Py_BLOOM_FILTER_BITS_PER_WORD - 1));
+        hash >>= 8;
+    }
+}
+
+static inline bool
+bloom_filter_may_contain(const _PyBloomFilter *bloom, const _PyBloomFilter *hashes)
+{
+    for (int i = 0; i < _Py_BLOOM_FILTER_WORDS; i++) {
+        if ((bloom->bits[i] & hashes->bits[i]) != hashes->bits[i]) {
+            return false;
+        }
+    }
+    return true;
+}
+
 #define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3
 #define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6
 

Include/internal/pycore_uop.h

@@ -45,10 +45,21 @@ typedef struct _PyUOpInstruction{
 
 /* Bloom filter with m = 256
  * https://en.wikipedia.org/wiki/Bloom_filter */
-#define _Py_BLOOM_FILTER_WORDS 8
+#ifdef HAVE_GCC_UINT128_T
+#define _Py_BLOOM_FILTER_WORDS 2
+typedef __uint128_t _Py_bloom_filter_word_t;
+#else
+#define _Py_BLOOM_FILTER_WORDS 4
+typedef uint64_t _Py_bloom_filter_word_t;
+#endif
+
+#define _Py_BLOOM_FILTER_BITS_PER_WORD \
+    ((int)(sizeof(_Py_bloom_filter_word_t) * 8))
+#define _Py_BLOOM_FILTER_WORD_SHIFT \
+    ((sizeof(_Py_bloom_filter_word_t) == 16) ? 7 : 6)
 
 typedef struct {
-    uint32_t bits[_Py_BLOOM_FILTER_WORDS];
+    _Py_bloom_filter_word_t bits[_Py_BLOOM_FILTER_WORDS];
 } _PyBloomFilter;
 
 #ifdef __cplusplus

Lib/argparse.py

@@ -2623,7 +2623,7 @@ def _get_nargs_pattern(self, action):
 
         # allow any number of options or arguments
         elif nargs == REMAINDER:
-            nargs_pattern = '([AO]*)' if option else '(.*)'
+            nargs_pattern = '(.*)'
 
         # allow one argument followed by any number of options or arguments
         elif nargs == PARSER:

Lib/pkgutil.py

@@ -393,6 +393,9 @@ def get_data(package, resource):
     # signature - an os.path format "filename" starting with the dirname of
     # the package's __file__
     parts = resource.split('/')
+    if os.path.isabs(resource) or '..' in parts:
+        raise ValueError("resource must be a relative path with no "
+                         "parent directory components")
     parts.insert(0, os.path.dirname(mod.__file__))
     resource_name = os.path.join(*parts)
     return loader.get_data(resource_name)

Lib/sysconfig/__init__.py

@@ -665,12 +665,10 @@ def get_platform():
 
     For other non-POSIX platforms, currently just returns :data:`sys.platform`."""
     if os.name == 'nt':
-        if 'amd64' in sys.version.lower():
-            return 'win-amd64'
-        if '(arm)' in sys.version.lower():
-            return 'win-arm32'
-        if '(arm64)' in sys.version.lower():
-            return 'win-arm64'
+        import _sysconfig
+        platform = _sysconfig.get_platform()
+        if platform:
+            return platform
         return sys.platform
 
     if os.name != "posix" or not hasattr(os, 'uname'):

Lib/test/test_argparse.py

@@ -6605,6 +6605,20 @@ def test_remainder(self):
         args = parser.parse_args(['--foo', 'a', '--', 'b', '--', 'c'])
         self.assertEqual(NS(foo='a', bar=['--', 'b', '--', 'c']), args)
 
+    def test_optional_remainder(self):
+        parser = argparse.ArgumentParser(exit_on_error=False)
+        parser.add_argument('--foo', nargs='...')
+        parser.add_argument('bar', nargs='*')
+
+        args = parser.parse_args(['--', '--foo', 'a', 'b'])
+        self.assertEqual(NS(foo=None, bar=['--foo', 'a', 'b']), args)
+        args = parser.parse_args(['--foo', '--', 'a', 'b'])
+        self.assertEqual(NS(foo=['--', 'a', 'b'], bar=[]), args)
+        args = parser.parse_args(['--foo', 'a', '--', 'b'])
+        self.assertEqual(NS(foo=['a', '--', 'b'], bar=[]), args)
+        args = parser.parse_args(['--foo', 'a', 'b', '--'])
+        self.assertEqual(NS(foo=['a', 'b', '--'], bar=[]), args)
+
     def test_subparser(self):
         parser = argparse.ArgumentParser(exit_on_error=False)
         parser.add_argument('foo')