Merge branch 'main' into dev_143414

Author: cocolato

Android/testbed/app/build.gradle.kts

@@ -92,7 +92,12 @@ android {
             }
             throw GradleException("Failed to find API level in $androidEnvFile")
         }
-        targetSdk = 35
+
+        // This controls the API level of the maxVersion managed emulator, which is used
+        // by CI and cibuildwheel. 34 takes up too much disk space (#142289), 35 has
+        // issues connecting to the internet (#142387), and 36 and later are not
+        // available as aosp_atd images yet.
+        targetSdk = 33
 
         versionCode = 1
         versionName = "1.0"

Doc/c-api/memory.rst

@@ -677,7 +677,11 @@ The pymalloc allocator
 Python has a *pymalloc* allocator optimized for small objects (smaller or equal
 to 512 bytes) with a short lifetime. It uses memory mappings called "arenas"
 with a fixed size of either 256 KiB on 32-bit platforms or 1 MiB on 64-bit
-platforms. It falls back to :c:func:`PyMem_RawMalloc` and
+platforms. When Python is configured with :option:`--with-pymalloc-hugepages`,
+the arena size on 64-bit platforms is increased to 2 MiB to match the huge page
+size, and arena allocation will attempt to use huge pages (``MAP_HUGETLB`` on
+Linux, ``MEM_LARGE_PAGES`` on Windows) with automatic fallback to regular pages.
+It falls back to :c:func:`PyMem_RawMalloc` and
 :c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes.
 
 *pymalloc* is the :ref:`default allocator <default-memory-allocators>` of the

Doc/c-api/module.rst

@@ -426,10 +426,10 @@ To retrieve the state from a given module, use the following functions:
    module state.
 
 
-.. c:function:: int PyModule_GetStateSize(PyObject *, Py_ssize_t *result)
+.. c:function:: int PyModule_GetStateSize(PyObject *module, Py_ssize_t *result)
 
-   Set *\*result* to the size of the module's state, as specified using
-   :c:macro:`Py_mod_state_size` (or :c:member:`PyModuleDef.m_size`),
+   Set *\*result* to the size of *module*'s state, as specified
+   using :c:macro:`Py_mod_state_size` (or :c:member:`PyModuleDef.m_size`),
    and return 0.
 
    On error, set *\*result* to -1, and return -1 with an exception set.
@@ -597,7 +597,7 @@ A module's token -- and the *your_token* value to use in the above code -- is:
 
 .. c:function:: int PyModule_GetToken(PyObject *module, void** result)
 
-   Set *\*result* to the module's token and return 0.
+   Set *\*result* to the module token for *module* and return 0.
 
    On error, set *\*result* to NULL, and return -1 with an exception set.
 
@@ -645,7 +645,7 @@ rather than from an extension's :ref:`export hook <extension-export-hook>`.
 
 .. c:function:: int PyModule_Exec(PyObject *module)
 
-   Execute the :c:data:`Py_mod_exec` slot(s) of the given *module*.
+   Execute the :c:data:`Py_mod_exec` slot(s) of *module*.
 
    On success, return 0.
    On error, return -1 with an exception set.

Doc/library/contextvars.rst

@@ -119,13 +119,15 @@ Context Variables
           # After the reset call the var has no value again, so
           # var.get() would raise a LookupError.
 
+      The same *token* cannot be used twice.
+
 
 .. class:: Token
 
    *Token* objects are returned by the :meth:`ContextVar.set` method.
    They can be passed to the :meth:`ContextVar.reset` method to revert
    the value of the variable to what it was before the corresponding
-   *set*.
+   *set*. A single token cannot reset a context variable more than once.
 
    Tokens support the :ref:`context manager protocol <context-managers>`
    to automatically reset context variables. See :meth:`ContextVar.set`.

Doc/library/subprocess.rst

@@ -803,14 +803,29 @@ Instances of the :class:`Popen` class have the following methods:
 
    .. note::
 
-      When the ``timeout`` parameter is not ``None``, then (on POSIX) the
-      function is implemented using a busy loop (non-blocking call and short
-      sleeps). Use the :mod:`asyncio` module for an asynchronous wait: see
+      When ``timeout`` is not ``None`` and the platform supports it, an
+      efficient event-driven mechanism is used to wait for process termination:
+
+      - Linux >= 5.3 uses :func:`os.pidfd_open` + :func:`select.poll`
+      - macOS and other BSD variants use :func:`select.kqueue` +
+        ``KQ_FILTER_PROC`` + ``KQ_NOTE_EXIT``
+      - Windows uses ``WaitForSingleObject``
+
+      If none of these mechanisms are available, the function falls back to a
+      busy loop (non-blocking call and short sleeps).
+
+   .. note::
+
+      Use the :mod:`asyncio` module for an asynchronous wait: see
       :class:`asyncio.create_subprocess_exec`.
 
    .. versionchanged:: 3.3
       *timeout* was added.
 
+   .. versionchanged:: 3.15
+      if *timeout* is not ``None``, use efficient event-driven implementation
+      on Linux >= 5.3 and macOS / BSD.
+
 .. method:: Popen.communicate(input=None, timeout=None)
 
    Interact with process: Send data to stdin.  Read data from stdout and stderr,

Doc/using/configure.rst

@@ -783,6 +783,21 @@ also be used to improve performance.
 
    See also :envvar:`PYTHONMALLOC` environment variable.
 
+.. option:: --with-pymalloc-hugepages
+
+   Enable huge page support for :ref:`pymalloc <pymalloc>` arenas (disabled by
+   default). When enabled, the arena size on 64-bit platforms is increased to
+   2 MiB and arena allocation uses ``MAP_HUGETLB`` (Linux) or
+   ``MEM_LARGE_PAGES`` (Windows) with automatic fallback to regular pages.
+
+   The configure script checks that the platform supports ``MAP_HUGETLB``
+   and emits a warning if it is not available.
+
+   On Windows, use the ``--pymalloc-hugepages`` flag with ``build.bat`` or
+   set the ``UsePymallocHugepages`` MSBuild property.
+
+   .. versionadded:: 3.15
+
 .. option:: --without-doc-strings
 
    Disable static documentation strings to reduce the memory footprint (enabled

Doc/whatsnew/3.15.rst

@@ -743,6 +743,20 @@ ssl
 
    (Contributed by Ron Frederick in :gh:`138252`.)
 
+subprocess
+----------
+
+* :meth:`subprocess.Popen.wait`: when ``timeout`` is not ``None`` and the
+  platform supports it, an efficient event-driven mechanism is used to wait for
+  process termination:
+
+  - Linux >= 5.3 uses :func:`os.pidfd_open` + :func:`select.poll`.
+  - macOS and other BSD variants use :func:`select.kqueue` + ``KQ_FILTER_PROC`` + ``KQ_NOTE_EXIT``.
+  - Windows keeps using ``WaitForSingleObject`` (unchanged).
+
+  If none of these mechanisms are available, the function falls back to the
+  traditional busy loop (non-blocking call and short sleeps).
+  (Contributed by Giampaolo Rodola in :gh:`83069`).
 
 symtable
 --------
@@ -1463,6 +1477,12 @@ Build changes
   modules that are missing or packaged separately.
   (Contributed by Stan Ulbrych and Petr Viktorin in :gh:`139707`.)
 
+* The new configure option :option:`--with-pymalloc-hugepages` enables huge
+  page support for :ref:`pymalloc <pymalloc>` arenas. When enabled, arena size
+  increases to 2 MiB and allocation uses ``MAP_HUGETLB`` (Linux) or
+  ``MEM_LARGE_PAGES`` (Windows) with automatic fallback to regular pages.
+  On Windows, use ``build.bat --pymalloc-hugepages``.
+
 * Annotating anonymous mmap usage is now supported if Linux kernel supports
   :manpage:`PR_SET_VMA_ANON_NAME <PR_SET_VMA(2const)>` (Linux 5.17 or newer).
   Annotations are visible in ``/proc/<pid>/maps`` if the kernel supports the feature

Include/internal/pycore_code.h

@@ -292,17 +292,7 @@ extern int _PyCode_SafeAddr2Line(PyCodeObject *co, int addr);
 extern void _PyCode_Clear_Executors(PyCodeObject *code);
 
 
-#ifdef Py_GIL_DISABLED
-// gh-115999 tracks progress on addressing this.
-#define ENABLE_SPECIALIZATION 0
-// Use this to enable specialization families once they are thread-safe. All
-// uses will be replaced with ENABLE_SPECIALIZATION once all families are
-// thread-safe.
-#define ENABLE_SPECIALIZATION_FT 1
-#else
 #define ENABLE_SPECIALIZATION 1
-#define ENABLE_SPECIALIZATION_FT ENABLE_SPECIALIZATION
-#endif
 
 /* Specialization functions, these are exported only for other re-generated
  * interpreters to call */

Include/internal/pycore_jit.h

@@ -26,6 +26,7 @@ typedef _Py_CODEUNIT *(*jit_func)(
 int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length);
 void _PyJIT_Free(_PyExecutorObject *executor);
 void _PyJIT_Fini(void);
+PyAPI_FUNC(int) _PyJIT_AddressInJitCode(PyInterpreterState *interp, uintptr_t addr);
 
 #endif  // _Py_JIT
 

Include/internal/pycore_obmalloc.h

@@ -208,7 +208,11 @@ typedef unsigned int pymem_uint;  /* assuming >= 16 bits */
  * mappings to reduce heap fragmentation.
  */
 #ifdef USE_LARGE_ARENAS
-#define ARENA_BITS              20                    /* 1 MiB */
+#  ifdef PYMALLOC_USE_HUGEPAGES
+#    define ARENA_BITS            21                    /* 2 MiB */
+#  else
+#    define ARENA_BITS            20                    /* 1 MiB */
+#  endif
 #else
 #define ARENA_BITS              18                    /* 256 KiB */
 #endif
@@ -469,7 +473,7 @@ nfp free pools in usable_arenas.
 */
 
 /* How many arena_objects do we initially allocate?
- * 16 = can allocate 16 arenas = 16 * ARENA_SIZE = 4MB before growing the
+ * 16 = can allocate 16 arenas = 16 * ARENA_SIZE before growing the
  * `arenas` vector.
  */
 #define INITIAL_ARENA_OBJECTS 16
@@ -512,14 +516,26 @@ struct _obmalloc_mgmt {
 
    memory address bit allocation for keys
 
-   64-bit pointers, IGNORE_BITS=0 and 2^20 arena size:
+   ARENA_BITS is configurable: 20 (1 MiB) by default on 64-bit, or
+   21 (2 MiB) when PYMALLOC_USE_HUGEPAGES is enabled.  All bit widths
+   below are derived from ARENA_BITS automatically.
+
+   64-bit pointers, IGNORE_BITS=0 and 2^20 arena size (default):
      15 -> MAP_TOP_BITS
      15 -> MAP_MID_BITS
      14 -> MAP_BOT_BITS
      20 -> ideal aligned arena
    ----
      64
 
+   64-bit pointers, IGNORE_BITS=0 and 2^21 arena size (hugepages):
+     15 -> MAP_TOP_BITS
+     15 -> MAP_MID_BITS
+     13 -> MAP_BOT_BITS
+     21 -> ideal aligned arena
+   ----
+     64
+
    64-bit pointers, IGNORE_BITS=16, and 2^20 arena size:
      16 -> IGNORE_BITS
      10 -> MAP_TOP_BITS