gh-158195: Do not hold the type lock during stop-the-world

Include/internal/pycore_interp_structs.h

@@ -588,6 +588,12 @@ struct types_state {
        where all those lower numbers are used for core static types. */
     unsigned int next_version_tag;
 
+#ifdef Py_GIL_DISABLED
+    /* Incremented on changes that can invalidate a type update that is
+       prepared under the type lock and published with the world stopped. */
+    uint64_t publish_gen;
+#endif
+
     /* Every static builtin type is initialized for each interpreter
        during its own initialization, including for the main interpreter
        during global runtime initialization.  This is done by calling

Lib/test/test_free_threading/test_threading.py

@@ -26,6 +26,30 @@ def mutate_thread():
 
 
 class TestThreadState(unittest.TestCase):
+    def test_stop_the_world(self):
+        import _testinternalcapi
+
+        _testinternalcapi.stop_the_world()
+        _testinternalcapi.stop_the_world(0)
+
+        def pause():
+            for _ in range(10):
+                _testinternalcapi.stop_the_world(0.0001)
+
+        threading_helper.run_concurrently(pause, nthreads=2)
+        for duration in (-1, -0.1, float("nan")):
+            with self.subTest(duration=duration):
+                with self.assertRaises(ValueError):
+                    _testinternalcapi.stop_the_world(duration)
+        for duration in (float("inf"), 10**100):
+            with self.subTest(duration=duration):
+                with self.assertRaises(OverflowError):
+                    _testinternalcapi.stop_the_world(duration)
+        for duration in (None, "0"):
+            with self.subTest(duration=duration):
+                with self.assertRaises(TypeError):
+                    _testinternalcapi.stop_the_world(duration)
+
     @support.requires_subprocess()
     def test_tight_stw_loop_does_not_starve_attach(self):
         script = textwrap.dedent(f"""

Lib/test/test_free_threading/test_type.py

@@ -1,5 +1,7 @@
+import gc
 import threading
 import unittest
+import weakref
 
 from concurrent.futures import ThreadPoolExecutor
 from threading import Barrier, Thread
@@ -19,6 +21,24 @@
 class A:
     attr = 1
 
+def world_stoppers():
+    """Finite workloads that contend for the stop-the-world mutex.
+
+    Waiting for the mutex releases the type lock, allowing queued type
+    updates to go stale.  Use a short pause rather than full collections
+    to prolong the contention without depending on the size of the heap.
+    """
+    def stopper():
+        for _ in range(100):
+            _testinternalcapi.stop_the_world()
+
+    def pauser():
+        for _ in range(100):
+            _testinternalcapi.stop_the_world(0.0001)
+
+    return [stopper, pauser]
+
+
 @threading_helper.requires_working_threading()
 class TestType(TestCase):
     def test_attr_cache(self):
@@ -344,6 +364,12 @@ def custom_repr(self):
         self.assertTrue(all(repr(cls()) != "custom repr"
                             for cls in subclasses))
 
+        # The slot update queue must release its references to the types,
+        # including those stored in heap-allocated chunks.
+        del subclasses
+        gc.collect()
+        self.assertEqual(Base.__subclasses__(), [])
+
     def test_concurrent_setattr_deadlock(self):
         # gh-155400: two threads assigning to a special method of the same
         # class could deadlock.  One thread held the type lock and waited for
@@ -401,6 +427,164 @@ class B(A):
         with threading_helper.start_threads(threads):
             pass
 
+    def check_stop_the_world_under_type_lock(self, setter):
+        # gh-158195: Python code that runs with the type lock held and stops
+        # the world could deadlock with a thread that held the type lock
+        # pinned while it waited for the stop-the-world mutex (__bases__
+        # assignment).  The first thread released the type lock while it was
+        # blocked on the stop-the-world mutex, was handed the mutex, and then
+        # waited for the type lock while owning the mutex.
+        def basesetter():
+            class A:
+                pass
+            class B:
+                pass
+            class C(A):
+                pass
+            for _ in range(100):
+                C.__bases__ = (B,)
+                C.__bases__ = (A,)
+
+        def stopper():
+            for _ in range(100):
+                _testinternalcapi.stop_the_world()
+
+        # The stopper keeps the stop-the-world mutex contended, which makes
+        # the setters block on it.  All workers have finite workloads so
+        # that repeated invalidation cannot indefinitely delay a setter.
+        threading_helper.run_concurrently(
+            [setter, setter, basesetter, stopper])
+
+    def test_stop_the_world_in_mro_deadlock(self):
+        # type_ready() calls a metaclass mro() with the type lock held.
+        N = 50
+
+        class Meta(type):
+            def mro(cls):
+                _testinternalcapi.stop_the_world()
+                return type.mro(cls)
+
+        def setter():
+            for _ in range(N):
+                Meta("K", (), {})
+
+        self.check_stop_the_world_under_type_lock(setter)
+
+    def test_stop_the_world_in_setattr_key_eq_deadlock(self):
+        # type_setattro() compares the attribute name with a str subclass key
+        # of the type's __dict__ (calling its __eq__) with the type lock held.
+        N = 50
+
+        class S(str):
+            __hash__ = str.__hash__
+            def __eq__(self, other):
+                _testinternalcapi.stop_the_world()
+                return str.__eq__(self, other)
+
+        C = type("C", (), {S("x"): 0})
+
+        def setter():
+            for i in range(N):
+                C.x = i
+
+        self.check_stop_the_world_under_type_lock(setter)
+
+    def test_reentrant_slot_update_consistency(self):
+        # Updating Base queues its slot update before visiting Sub.  A key
+        # comparison in Sub's dict re-enters the setter and makes that queue
+        # stale.  It must be rebuilt rather than overwriting the inner update.
+        armed = False
+
+        class Base:
+            pass
+
+        class Key(str):
+            def __hash__(self):
+                return hash("__hash__")
+
+            def __eq__(self, other):
+                nonlocal armed
+                if armed and other == "__hash__":
+                    armed = False
+                    Base.__hash__ = object.__hash__
+                return str.__eq__(self, other)
+
+        # Collide with the special-method name without defining it on Sub,
+        # so that slot updates still propagate from Base to Sub.
+        Sub = type("Sub", (Base,), {Key("trigger"): None})
+        objects = [Base(), Sub()]
+
+        # Check actual slot dispatch, not just the class dictionaries.  None
+        # and object.__hash__ use distinct C slots, unlike Python functions
+        # which share a generic wrapper that looks up the current method.
+        Base.__hash__ = None
+        for obj in objects:
+            with self.assertRaises(TypeError):
+                hash(obj)
+        Base.__hash__ = object.__hash__
+        for obj in objects:
+            self.assertEqual(hash(obj), object.__hash__(obj))
+
+        armed = True
+        Base.__hash__ = None
+        self.assertFalse(armed)
+        self.assertIs(Base.__dict__["__hash__"], object.__hash__)
+        for obj in objects:
+            self.assertEqual(hash(obj), object.__hash__(obj))
+
+        # Both applied and discarded queues must release their references.
+        base_ref = weakref.ref(Base)
+        sub_ref = weakref.ref(Sub)
+        del obj, objects, Sub, Base
+        gc.collect()
+        self.assertIsNone(base_ref())
+        self.assertIsNone(sub_ref())
+
+    def test_concurrent_bases_assignment(self):
+        # Several threads reassign __bases__ of the same class, and of its
+        # base, while others stop the world.  The validated bases and the
+        # computed MRO can go stale while the type lock is released and
+        # must be recomputed.  At the end, the MRO must agree with __bases__.
+        N = 50
+
+        class A:
+            def meth(self):
+                return "A"
+            def __repr__(self):
+                return "A"
+        class B:
+            def meth(self):
+                return "B"
+            def __repr__(self):
+                return "B"
+        class C(A):
+            pass
+        class D(C):
+            pass
+
+        def setter():
+            for _ in range(N):
+                C.__bases__ = (B,)
+                C.__bases__ = (A,)
+
+        def subsetter():
+            for _ in range(N):
+                D.__bases__ = (C, B)
+                D.__bases__ = (C,)
+
+        threading_helper.run_concurrently(
+            [setter, setter, subsetter] + world_stoppers())
+
+        self.assertEqual(C.__bases__, (A,))
+        self.assertEqual(C.__mro__, (C, A, object))
+        self.assertEqual(D.__bases__, (C,))
+        self.assertEqual(D.__mro__, (D, C, A, object))
+        self.assertEqual(D().meth(), "A")
+        self.assertEqual(repr(C()), "A")
+        self.assertEqual(repr(D()), "A")
+        self.assertIn(C, A.__subclasses__())
+        self.assertNotIn(C, B.__subclasses__())
+
 
 if __name__ == "__main__":
     unittest.main()

Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-10-00-00.gh-issue-158195.UnpinTL.rst

@@ -0,0 +1,7 @@
+Fix a deadlock in the free-threaded build between a thread that stops the
+world while running Python code under the type lock (for example a custom
+metaclass ``mro()`` or the ``__eq__`` of a :class:`str` subclass used as a
+class ``__dict__`` key) and a thread that assigns to a special method or to
+``__bases__`` of a class.  The type lock is no longer held while waiting
+for a stop-the-world; the prepared slot, MRO and bases updates are
+validated with the world stopped and recomputed if they went stale.

Modules/_testinternalcapi.c

@@ -38,6 +38,7 @@
 #include "pycore_pylifecycle.h"   // _PyInterpreterConfig_InitFromDict()
 #include "pycore_pystate.h"       // _PyThreadState_GET()
 #include "pycore_runtime_structs.h" // _PY_NSMALLPOSINTS
+#include "pycore_time.h"          // _PyTime_FromSecondsObject()
 #include "pycore_unicodeobject.h" // _PyUnicode_TransformDecimalAndSpaceToASCII()
 
 #include "clinic/_testinternalcapi.c.h"
@@ -209,6 +210,42 @@ get_stack_margin(PyObject *self, PyObject *Py_UNUSED(args))
     return PyLong_FromSize_t(_PyOS_STACK_MARGIN_BYTES);
 }
 
+static PyObject *
+stop_the_world(PyObject *self, PyObject *args)
+{
+    PyObject *duration_obj = NULL;
+    PyTime_t duration = 0;
+    if (!PyArg_ParseTuple(args, "|O:stop_the_world", &duration_obj)) {
+        return NULL;
+    }
+    if (duration_obj != NULL &&
+        _PyTime_FromSecondsObject(&duration, duration_obj,
+                                 _PyTime_ROUND_TIMEOUT) < 0) {
+        return NULL;
+    }
+    if (duration < 0) {
+        PyErr_SetString(PyExc_ValueError, "duration must be non-negative");
+        return NULL;
+    }
+#ifdef Py_GIL_DISABLED
+    PyInterpreterState *interp = _PyInterpreterState_GET();
+    _PyEval_StopTheWorld(interp);
+    if (duration != 0) {
+        PyEvent event = {0};
+        PyEvent_WaitTimed(&event, duration, /*detach=*/0);
+    }
+    _PyEval_StartTheWorld(interp);
+#endif
+    Py_RETURN_NONE;
+}
+
+PyDoc_STRVAR(stop_the_world_doc,
+"stop_the_world($module, duration=0, /)\n"
+"--\n"
+"\n"
+"Stop the world, wait for duration seconds, then restart it.\n"
+"Does nothing in GIL-enabled builds.");
+
 static PyObject *
 test_stop_the_world(PyObject *self, PyObject *Py_UNUSED(args))
 {
@@ -3345,6 +3382,7 @@ static PyMethodDef module_functions[] = {
     {"get_c_recursion_remaining", get_c_recursion_remaining, METH_NOARGS},
     {"get_stack_pointer", get_stack_pointer, METH_NOARGS},
     {"get_stack_margin", get_stack_margin, METH_NOARGS},
+    {"stop_the_world", stop_the_world, METH_VARARGS, stop_the_world_doc},
     {"test_stop_the_world", test_stop_the_world, METH_NOARGS},
     {"classify_stack_addresses", classify_stack_addresses, METH_VARARGS},
     {"get_jit_code_ranges", get_jit_code_ranges, METH_NOARGS},