From 6082c8fae282cd21d31af42ebaab3a9cb55fcd80 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Tue, 20 Oct 2026 12:48:58 -0700 Subject: [PATCH 1/5] Add external frame introspector to support deferred reference counting on free-threaded builds --- Include/cpython/pyframe.h | 49 ++++++++++ Include/internal/pycore_interpframe.h | 22 +++++ Include/internal/pycore_interpframe_structs.h | 21 ++++ Include/internal/pycore_typedefs.h | 1 + Lib/test/test_capi/test_frame.py | 45 +++++++++ Modules/_testcapi/frame.c | 80 ++++++++++++++++ Modules/_testinternalcapi.c | 95 +++++++++++++++++++ Python/frame.c | 55 +++++++++++ Python/gc_free_threading.c | 40 ++++++++ 9 files changed, 408 insertions(+) diff --git a/Include/cpython/pyframe.h b/Include/cpython/pyframe.h index 24a947de1ede1e..246663932f03a6 100644 --- a/Include/cpython/pyframe.h +++ b/Include/cpython/pyframe.h @@ -35,3 +35,52 @@ PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame /* Returns the currently executing line number, or -1 if there is no line number. * Does not raise an exception. */ PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *frame); + +/* External frame introspection. + * + * Frames with the `external` bit set were pushed by something other than + * the VM. Such frames have a `struct _PyInterpreterFrameExternal` header + * immediately before the `struct _PyInterpreterFrame` in memory. The header + * holds a strong reference to a `_PyExternalFrameIntrospector` object which + * knows how to visit external references for the garbage collector. + * + * The introspector is created with a GC visitor function. During collection + * (free-threaded build), the GC calls the visitor with the frame and a + * callback; the visitor must invoke the callback once for each PyObject* + * referenced by external frame state. + */ + +struct _PyExternalFrameIntrospector; + +/* GC visitor for external frames. + * + * `interp` is the interpreter being collected, `frame` is the external + * frame, `callback` must be invoked once for each PyObject* referenced + * by external frame state (with `arg` passed through). The callback + * follows gcvisitobjects_t conventions (return 1 to continue, 0 to stop). + * + * Returns 0 if the frame is complete (all references were visited), + * nonzero if the frame is incomplete and the GC must skip collecting + * objects with deferred references. + */ +typedef int (*PyUnstable_ExternalFrameGCVisitor)( + PyInterpreterState *interp, + struct _PyInterpreterFrame *frame, + gcvisitobjects_t callback, + void *arg); + +PyAPI_DATA(PyTypeObject) PyUnstable_ExternalFrameIntrospector_Type; + +#define PyUnstable_ExternalFrameIntrospector_Check(op) \ + Py_IS_TYPE((op), &PyUnstable_ExternalFrameIntrospector_Type) + +/* Create a new _PyExternalFrameIntrospector holding `gc_visitor`. + * Returns a new reference, or NULL with an exception set if `gc_visitor` + * is NULL or allocation fails. */ +PyAPI_FUNC(PyObject *) PyUnstable_ExternalFrameIntrospector_New( + PyUnstable_ExternalFrameGCVisitor gc_visitor); + +/* Return the gc_visitor stored in `introspector`. + * Returns NULL with a TypeError if `introspector` is not an introspector. */ +PyAPI_FUNC(PyUnstable_ExternalFrameGCVisitor) +PyUnstable_ExternalFrameIntrospector_GetGCVisitor(PyObject *introspector); diff --git a/Include/internal/pycore_interpframe.h b/Include/internal/pycore_interpframe.h index a85fa9bd32853c..0254315ba214ca 100644 --- a/Include/internal/pycore_interpframe.h +++ b/Include/internal/pycore_interpframe.h @@ -315,6 +315,28 @@ _PyThreadState_GetFrame(PyThreadState *tstate) return _PyFrame_GetFirstComplete(tstate->current_frame); } +/* Convert between a frame with the `external` bit set and its header + * which lives immediately before the frame in memory. */ +static inline _PyInterpreterFrameExternal * +_PyInterpreterFrame_GetExternal(_PyInterpreterFrame *frame) +{ + assert(frame->external); + return (_PyInterpreterFrameExternal *)((char *)frame - + sizeof(_PyInterpreterFrameExternal)); +} + +static inline _PyInterpreterFrame * +_PyInterpreterFrameExternal_GetFrame(_PyInterpreterFrameExternal *external) +{ + return (_PyInterpreterFrame *)(external + 1); +} + +static inline PyObject * +_PyInterpreterFrameExternal_GetIntrospector(_PyInterpreterFrameExternal *external) +{ + return PyStackRef_AsPyObjectBorrow(external->introspector); +} + // Update last_profiled_frame for remote profiler frame caching. // Only update if we're removing the exact frame that was last profiled. // This avoids corrupting the cache when transient frames (called and returned diff --git a/Include/internal/pycore_interpframe_structs.h b/Include/internal/pycore_interpframe_structs.h index 0d6a37cc2c1a17..12b9ba2f6e2339 100644 --- a/Include/internal/pycore_interpframe_structs.h +++ b/Include/internal/pycore_interpframe_structs.h @@ -52,6 +52,27 @@ struct _PyInterpreterFrame { _PyStackRef localsplus[1]; }; +/* Header for externally-pushed frames. + * + * When a frame's `external` bit is set, a `_PyInterpreterFrameExternal` + * header lives immediately before the `_PyInterpreterFrame` in memory. + * It holds a strong reference to a `_PyExternalFrameIntrospector` which + * knows how to visit external references for the garbage collector. + * + * Layout in memory: + * + * +-------------------------------+----------------------+ + * | _PyInterpreterFrameExternal | _PyInterpreterFrame | + * +-------------------------------+----------------------+ + * ^ ^ + * external frame + * + * Use `_PyInterpreterFrame_GetExternal()` to convert from frame to header. + */ +struct _PyInterpreterFrameExternal { + _PyStackRef introspector; /* Strong reference to _PyExternalFrameIntrospector */ +}; + /* _PyGenObject_HEAD defines the initial segment of generator and coroutine objects. */ diff --git a/Include/internal/pycore_typedefs.h b/Include/internal/pycore_typedefs.h index d330cc4dc2b8fa..5cf13907eb595c 100644 --- a/Include/internal/pycore_typedefs.h +++ b/Include/internal/pycore_typedefs.h @@ -10,6 +10,7 @@ extern "C" { #endif typedef struct _PyInterpreterFrame _PyInterpreterFrame; +typedef struct _PyInterpreterFrameExternal _PyInterpreterFrameExternal; typedef struct pyruntimestate _PyRuntimeState; #ifdef __cplusplus diff --git a/Lib/test/test_capi/test_frame.py b/Lib/test/test_capi/test_frame.py index 23cb8e3dada9d4..898772fc876d97 100644 --- a/Lib/test/test_capi/test_frame.py +++ b/Lib/test/test_capi/test_frame.py @@ -52,5 +52,50 @@ def dummy(): self.assertIsNone(frame.f_back) +class ExternalFrameTest(unittest.TestCase): + def test_introspector_new(self): + intro = _testcapi.external_frame_introspector_new() + self.assertEqual(type(intro).__name__, "external_frame_introspector") + self.assertTrue(_testcapi.external_frame_introspector_check(intro)) + self.assertFalse(_testcapi.external_frame_introspector_check(None)) + self.assertFalse(_testcapi.external_frame_introspector_check(42)) + self.assertFalse(_testcapi.external_frame_introspector_check([])) + + def test_introspector_new_null(self): + with self.assertRaises(ValueError): + _testcapi.external_frame_introspector_new_null() + + def test_introspector_roundtrip(self): + self.assertTrue(_testcapi.external_frame_introspector_roundtrip()) + + def test_introspector_get_visitor(self): + intro = _testcapi.external_frame_introspector_new() + addr = _testcapi.external_frame_introspector_get_visitor(intro) + self.assertIsInstance(addr, int) + self.assertNotEqual(addr, 0) + with self.assertRaises(TypeError): + _testcapi.external_frame_introspector_get_visitor(None) + with self.assertRaises(TypeError): + _testcapi.external_frame_introspector_get_visitor(42) + + def test_gc_visit_external_frame(self): + _testinternalcapi = import_helper.import_module('_testinternalcapi') + if sys._is_gil_enabled(): + self.skipTest("requires free-threaded build") + obj = [1, 2, 3] + calls, cb_calls = _testinternalcapi.external_frame_gc_test(obj, 0) + self.assertGreaterEqual(calls, 1) + self.assertGreaterEqual(cb_calls, 1) + + def test_gc_visit_external_frame_incomplete(self): + _testinternalcapi = import_helper.import_module('_testinternalcapi') + if sys._is_gil_enabled(): + self.skipTest("requires free-threaded build") + obj = [1, 2, 3] + calls, cb_calls = _testinternalcapi.external_frame_gc_test(obj, 1) + self.assertGreaterEqual(calls, 1) + self.assertGreaterEqual(cb_calls, 1) + + if __name__ == "__main__": unittest.main() diff --git a/Modules/_testcapi/frame.c b/Modules/_testcapi/frame.c index 5748dca948ea94..c357f8e6bddcc3 100644 --- a/Modules/_testcapi/frame.c +++ b/Modules/_testcapi/frame.c @@ -114,6 +114,76 @@ frame_getvarstring(PyObject *self, PyObject *args) } +static int +dummy_external_gc_visitor(PyInterpreterState *interp, + struct _PyInterpreterFrame *frame, + gcvisitobjects_t callback, void *arg) +{ + (void)interp; + (void)frame; + (void)callback; + (void)arg; + return 0; +} + +static PyObject * +external_frame_introspector_new(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyUnstable_ExternalFrameIntrospector_New(dummy_external_gc_visitor); +} + +static PyObject * +external_frame_introspector_new_null(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyUnstable_ExternalFrameIntrospector_New(NULL); +} + +static PyObject * +external_frame_introspector_check(PyObject *self, PyObject *obj) +{ + return PyBool_FromLong(PyUnstable_ExternalFrameIntrospector_Check(obj)); +} + +static PyObject * +external_frame_introspector_roundtrip(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *intro = PyUnstable_ExternalFrameIntrospector_New( + dummy_external_gc_visitor); + if (intro == NULL) { + return NULL; + } + if (!PyUnstable_ExternalFrameIntrospector_Check(intro)) { + Py_DECREF(intro); + PyErr_SetString(PyExc_AssertionError, "Check() failed"); + return NULL; + } + PyUnstable_ExternalFrameGCVisitor visitor = + PyUnstable_ExternalFrameIntrospector_GetGCVisitor(intro); + if (visitor == NULL) { + Py_DECREF(intro); + return NULL; + } + if (visitor != dummy_external_gc_visitor) { + Py_DECREF(intro); + PyErr_SetString(PyExc_AssertionError, "visitor mismatch"); + return NULL; + } + Py_DECREF(intro); + Py_RETURN_TRUE; +} + +static PyObject * +external_frame_introspector_get_visitor(PyObject *self, PyObject *obj) +{ + PyUnstable_ExternalFrameGCVisitor visitor = + PyUnstable_ExternalFrameIntrospector_GetGCVisitor(obj); + if (visitor == NULL) { + return NULL; + } + // Return the visitor address as an int for comparison in Python. + return PyLong_FromVoidPtr((void *)visitor); +} + static PyMethodDef test_methods[] = { {"frame_getlocals", frame_getlocals, METH_O, NULL}, {"frame_getglobals", frame_getglobals, METH_O, NULL}, @@ -123,6 +193,16 @@ static PyMethodDef test_methods[] = { {"frame_new", frame_new, METH_VARARGS, NULL}, {"frame_getvar", frame_getvar, METH_VARARGS, NULL}, {"frame_getvarstring", frame_getvarstring, METH_VARARGS, NULL}, + {"external_frame_introspector_new", external_frame_introspector_new, + METH_NOARGS, NULL}, + {"external_frame_introspector_new_null", external_frame_introspector_new_null, + METH_NOARGS, NULL}, + {"external_frame_introspector_check", external_frame_introspector_check, + METH_O, NULL}, + {"external_frame_introspector_roundtrip", + external_frame_introspector_roundtrip, METH_NOARGS, NULL}, + {"external_frame_introspector_get_visitor", + external_frame_introspector_get_visitor, METH_O, NULL}, {NULL}, }; diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 479297a996c250..648a8e4fa019f9 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1564,6 +1564,100 @@ iframe_getlasti(PyObject *self, PyObject *frame) return PyLong_FromLong(PyUnstable_InterpreterFrame_GetLasti(f)); } +static PyObject *external_test_visited_obj = NULL; +static int external_test_visitor_calls = 0; +static int external_test_callback_calls = 0; +static int external_test_visitor_incomplete = 0; + +static int +external_test_gc_visitor(PyInterpreterState *interp, + struct _PyInterpreterFrame *frame, + gcvisitobjects_t callback, void *arg) +{ + (void)interp; + (void)frame; + external_test_visitor_calls++; + if (external_test_visited_obj != NULL) { + external_test_callback_calls++; + callback(external_test_visited_obj, arg); + } + return external_test_visitor_incomplete; +} + +static PyObject * +external_frame_gc_test(PyObject *self, PyObject *args) +{ +#ifdef Py_GIL_DISABLED + PyObject *obj; + int incomplete; + if (!PyArg_ParseTuple(args, "Oi", &obj, &incomplete)) { + return NULL; + } + PyObject *intro = PyUnstable_ExternalFrameIntrospector_New( + external_test_gc_visitor); + if (intro == NULL) { + return NULL; + } + size_t ext_size = sizeof(_PyInterpreterFrameExternal); + size_t total = ext_size + sizeof(_PyInterpreterFrame); + char *alloc = (char *)PyMem_Malloc(total); + if (alloc == NULL) { + Py_DECREF(intro); + return PyErr_NoMemory(); + } + memset(alloc, 0, total); + _PyInterpreterFrameExternal *ext = + (_PyInterpreterFrameExternal *)alloc; + _PyInterpreterFrame *frame = + (_PyInterpreterFrame *)(alloc + ext_size); + + PyThreadState *tstate = PyThreadState_Get(); + _PyInterpreterFrame *old = tstate->current_frame; + frame->previous = old; + frame->owner = FRAME_OWNED_BY_THREAD; + frame->stackpointer = frame->localsplus; + frame->f_executable = PyStackRef_NULL; + frame->f_funcobj = PyStackRef_NULL; + ext->introspector = PyStackRef_FromPyObjectNew(intro); + frame->external = 1; + + if (_PyInterpreterFrame_GetExternal(frame) != ext || + _PyInterpreterFrameExternal_GetFrame(ext) != frame || + _PyInterpreterFrameExternal_GetIntrospector(ext) != intro) + { + PyStackRef_CLEAR(ext->introspector); + PyMem_Free(alloc); + Py_DECREF(intro); + PyErr_SetString(PyExc_AssertionError, "external header mismatch"); + return NULL; + } + + external_test_visited_obj = obj; + external_test_visitor_calls = 0; + external_test_callback_calls = 0; + external_test_visitor_incomplete = incomplete; + + tstate->current_frame = frame; + PyGC_Collect(); + tstate->current_frame = old; + + PyStackRef_CLEAR(ext->introspector); + frame->external = 0; + PyMem_Free(alloc); + Py_DECREF(intro); + external_test_visited_obj = NULL; + + if (external_test_visitor_calls < 1) { + PyErr_SetString(PyExc_AssertionError, "visitor was not called"); + return NULL; + } + return Py_BuildValue("(ii)", external_test_visitor_calls, + external_test_callback_calls); +#else + Py_RETURN_NONE; +#endif +} + static PyObject * code_returns_only_none(PyObject *self, PyObject *arg) { @@ -3378,6 +3472,7 @@ static PyMethodDef module_functions[] = { {"iframe_getcode", iframe_getcode, METH_O, NULL}, {"iframe_getline", iframe_getline, METH_O, NULL}, {"iframe_getlasti", iframe_getlasti, METH_O, NULL}, + {"external_frame_gc_test", external_frame_gc_test, METH_VARARGS, NULL}, {"code_returns_only_none", code_returns_only_none, METH_O, NULL}, {"get_co_framesize", get_co_framesize, METH_O, NULL}, {"get_co_localskinds", get_co_localskinds, METH_O, NULL}, diff --git a/Python/frame.c b/Python/frame.c index 79522b00b47f11..a87be64e1c9c22 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -156,3 +156,58 @@ PyUnstable_InterpreterFrame_GetLine(_PyInterpreterFrame *frame) int addr = _PyInterpreterFrame_LASTI(frame) * sizeof(_Py_CODEUNIT); return PyCode_Addr2Line(_PyFrame_GetCode(frame), addr); } + +/* External frame introspection (PyUnstable API) */ + +struct _PyExternalFrameIntrospector { + PyObject_HEAD + PyUnstable_ExternalFrameGCVisitor gc_visitor; +}; + +static void +external_frame_introspector_dealloc(PyObject *op) +{ + PyTypeObject *tp = Py_TYPE(op); + tp->tp_free(op); +} + +PyTypeObject PyUnstable_ExternalFrameIntrospector_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "external_frame_introspector", + .tp_basicsize = sizeof(struct _PyExternalFrameIntrospector), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .tp_doc = "Introspector for externally-pushed interpreter frames", + .tp_dealloc = external_frame_introspector_dealloc, + .tp_free = PyObject_Free, +}; + +PyObject * +PyUnstable_ExternalFrameIntrospector_New(PyUnstable_ExternalFrameGCVisitor gc_visitor) +{ + if (gc_visitor == NULL) { + PyErr_SetString(PyExc_ValueError, "gc_visitor must not be NULL"); + return NULL; + } + struct _PyExternalFrameIntrospector *op = PyObject_New( + struct _PyExternalFrameIntrospector, + &PyUnstable_ExternalFrameIntrospector_Type); + if (op == NULL) { + return PyErr_NoMemory(); + } + op->gc_visitor = gc_visitor; + return (PyObject *)op; +} + +PyUnstable_ExternalFrameGCVisitor +PyUnstable_ExternalFrameIntrospector_GetGCVisitor(PyObject *introspector) +{ + if (introspector == NULL || + !PyUnstable_ExternalFrameIntrospector_Check(introspector)) + { + PyErr_Format(PyExc_TypeError, + "expected external frame introspector, got %T", + introspector); + return NULL; + } + return ((struct _PyExternalFrameIntrospector *)introspector)->gc_visitor; +} diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 264afeebd54f89..5668ee5b588c87 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -443,6 +443,42 @@ gc_visit_stackref(_PyStackRef stackref) } } +static int +gc_visit_external_frame_object(PyObject *obj, void *arg) +{ + (void)arg; + if (obj != NULL && _PyObject_GC_IS_TRACKED(obj) && !gc_is_frozen(obj)) { + gc_add_refs(obj, 1); + } + return 1; +} + +static void +gc_visit_external_frame(PyInterpreterState *interp, _PyInterpreterFrame *frame, + struct collection_state *state) +{ + _PyInterpreterFrameExternal *ext = _PyInterpreterFrame_GetExternal(frame); + PyObject *introspector = + _PyInterpreterFrameExternal_GetIntrospector(ext); + if (introspector == NULL || + !PyUnstable_ExternalFrameIntrospector_Check(introspector)) + { + state->skip_deferred_objects = 1; + return; + } + PyUnstable_ExternalFrameGCVisitor visitor = + PyUnstable_ExternalFrameIntrospector_GetGCVisitor(introspector); + if (visitor == NULL) { + PyErr_Clear(); + state->skip_deferred_objects = 1; + return; + } + int res = visitor(interp, frame, gc_visit_external_frame_object, state); + if (res != 0) { + state->skip_deferred_objects = 1; + } +} + // Add 1 to the gc_refs for every deferred reference on each thread's stack. static void gc_visit_thread_stacks(PyInterpreterState *interp, struct collection_state *state) @@ -459,6 +495,10 @@ gc_visit_thread_stacks(PyInterpreterState *interp, struct collection_state *stat continue; } + if (f->external) { + gc_visit_external_frame(interp, f, state); + } + _PyStackRef *top = f->stackpointer; if (top == NULL) { // GH-129236: The stackpointer may be NULL in cases where From 553bd36beb6d833e4a98b575323f31f28dc271bb Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Tue, 20 Oct 2026 13:16:39 -0700 Subject: [PATCH 2/5] Docs and _testinternalcapi on GIL build --- Doc/c-api/frame.rst | 51 +++++++++++++++++++++++++++++++++++++ Modules/_testinternalcapi.c | 2 ++ 2 files changed, 53 insertions(+) diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 4159ff6e5965fb..b9c63c697e3982 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -243,3 +243,54 @@ Unless using :pep:`523`, you will not need this. Return the currently executing line number, or -1 if there is no line number. .. versionadded:: 3.12 + + +External frame introspectors +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Frames pushed by something other than the VM carry an external frame +introspector, which tells the :term:`free-threaded ` garbage +collector how to visit references held in external frame state. + +.. c:var:: PyTypeObject PyUnstable_ExternalFrameIntrospector_Type + + The type of external frame introspector objects. + + Instances cannot be created from Python code. Use + :c:func:`PyUnstable_ExternalFrameIntrospector_New` to create one. + + .. versionadded:: 3.16 + + +.. c:function:: int PyUnstable_ExternalFrameIntrospector_Check(PyObject *obj) + + Return non-zero if *obj* is an external frame introspector object. + + .. versionadded:: 3.16 + + +.. c:type:: int (*PyUnstable_ExternalFrameGCVisitor)(PyInterpreterState *interp, struct _PyInterpreterFrame *frame, gcvisitobjects_t callback, void *arg) + + The garbage-collector visitor held by an external frame introspector. + + The collector calls it with the interpreter being collected, the + external frame, and a *callback* to invoke once for each :c:type:`PyObject` + referenced by external frame state (passing *arg* through). The callback + follows :c:type:`gcvisitobjects_t` conventions. + + Return 0 if the frame is complete (all references were visited), or + non-zero if the frame is incomplete and the collector must skip collecting + objects with deferred references. + + .. versionadded:: 3.16 + + +.. c:function:: PyObject* PyUnstable_ExternalFrameIntrospector_New(PyUnstable_ExternalFrameGCVisitor gc_visitor) + + Create a new external frame introspector holding *gc_visitor*. + + Return a :term:`strong reference` to the new introspector on success. + Raise :exc:`ValueError` and return ``NULL`` if *gc_visitor* is ``NULL``; + raise :exc:`MemoryError` and return ``NULL`` on allocation failure. + + .. versionadded:: 3.16 diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 648a8e4fa019f9..acdd9f51e16f4d 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1569,6 +1569,7 @@ static int external_test_visitor_calls = 0; static int external_test_callback_calls = 0; static int external_test_visitor_incomplete = 0; +#ifdef Py_GIL_DISABLED static int external_test_gc_visitor(PyInterpreterState *interp, struct _PyInterpreterFrame *frame, @@ -1583,6 +1584,7 @@ external_test_gc_visitor(PyInterpreterState *interp, } return external_test_visitor_incomplete; } +#endif static PyObject * external_frame_gc_test(PyObject *self, PyObject *args) From 2eadf593f58a8e7f442f8bd45b44ca372d8741e7 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Tue, 20 Oct 2026 13:39:35 -0700 Subject: [PATCH 3/5] More GIL build ifdef --- Modules/_testinternalcapi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index acdd9f51e16f4d..9ce1e9747182b1 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1564,12 +1564,12 @@ iframe_getlasti(PyObject *self, PyObject *frame) return PyLong_FromLong(PyUnstable_InterpreterFrame_GetLasti(f)); } +#ifdef Py_GIL_DISABLED static PyObject *external_test_visited_obj = NULL; static int external_test_visitor_calls = 0; static int external_test_callback_calls = 0; static int external_test_visitor_incomplete = 0; -#ifdef Py_GIL_DISABLED static int external_test_gc_visitor(PyInterpreterState *interp, struct _PyInterpreterFrame *frame, From 1c287b139b38fb2457f5492b0e335116bc4bb3ae Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Tue, 20 Oct 2026 14:01:27 -0700 Subject: [PATCH 4/5] Update ignored --- Tools/c-analyzer/cpython/ignored.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index db9f74fdbc591f..b6491e7be4edf0 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -796,6 +796,7 @@ Objects/dictobject.c - PyFrozenDict_Type - Modules/_testcapi/unicode.c test_py_identifier PyId_hello - Modules/_testcapi/unicode.c test_py_identifier to_bytes_id - Modules/_testcapi/unicode.c test_py_identifier real_id - +Python/frame.c - PyUnstable_ExternalFrameIntrospector_Type - ## False positives Python/specialize.c - _Py_InitCleanup - From 3b36b95443b7e978bdbbe39140b632337273f63b Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Tue, 20 Oct 2026 21:22:50 -0700 Subject: [PATCH 5/5] Always set external --- Include/internal/pycore_interpframe.h | 4 ++ Lib/test/test_capi/test_frame.py | 14 +++++ Modules/_testinternalcapi.c | 70 +++++++++++++++++++++++++ Modules/_testinternalcapi/interpreter.c | 1 + Python/ceval.c | 1 + Python/pystate.c | 1 + 6 files changed, 91 insertions(+) diff --git a/Include/internal/pycore_interpframe.h b/Include/internal/pycore_interpframe.h index 0254315ba214ca..a78ff7708de6da 100644 --- a/Include/internal/pycore_interpframe.h +++ b/Include/internal/pycore_interpframe.h @@ -152,6 +152,8 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * dest->stackpointer = dest->localsplus + stacktop; // visited is GC bookkeeping for the current stack walk, not frame state. dest->visited = 0; + // Copies never carry the external header, so they are never external. + dest->external = 0; #ifdef Py_DEBUG dest->stackpointer_valid = src->stackpointer_valid; dest->lltrace = src->lltrace; @@ -208,6 +210,7 @@ _PyFrame_Initialize( frame->return_offset = 0; frame->owner = FRAME_OWNED_BY_THREAD; frame->visited = 0; + frame->external = 0; #ifdef Py_DEBUG frame->stackpointer_valid = 1; frame->lltrace = 0; @@ -463,6 +466,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int #endif frame->owner = FRAME_OWNED_BY_THREAD; frame->visited = 0; + frame->external = 0; #ifdef Py_DEBUG frame->stackpointer_valid = 1; frame->lltrace = 0; diff --git a/Lib/test/test_capi/test_frame.py b/Lib/test/test_capi/test_frame.py index 898772fc876d97..3f36416472e147 100644 --- a/Lib/test/test_capi/test_frame.py +++ b/Lib/test/test_capi/test_frame.py @@ -96,6 +96,20 @@ def test_gc_visit_external_frame_incomplete(self): self.assertGreaterEqual(calls, 1) self.assertGreaterEqual(cb_calls, 1) + def test_frame_init_clears_external_bits(self): + # VM-created frames must never carry the external bit, otherwise + # the GC would read a non-existent external header (segfault). + _testinternalcapi = import_helper.import_module('_testinternalcapi') + def func(): + pass + self.assertIsNone( + _testinternalcapi.frame_init_clears_external_bits(func)) + + def test_no_external_frames_on_stack(self): + _testinternalcapi = import_helper.import_module('_testinternalcapi') + self.assertIsNone( + _testinternalcapi.frame_no_external_frames_on_stack()) + if __name__ == "__main__": unittest.main() diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 9ce1e9747182b1..a4a8f410915598 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1564,6 +1564,72 @@ iframe_getlasti(PyObject *self, PyObject *frame) return PyLong_FromLong(PyUnstable_InterpreterFrame_GetLasti(f)); } +static PyObject * +frame_init_clears_external_bits(PyObject *self, PyObject *func) +{ + if (!PyFunction_Check(func)) { + PyErr_SetString(PyExc_TypeError, "argument must be a function"); + return NULL; + } + PyCodeObject *code = (PyCodeObject *)((PyFunctionObject *)func)->func_code; + // Pre-fill with garbage so forgotten fields are caught deterministically. + // NOTE: co_framesize is in pointer-sized words, not bytes. + size_t nbytes = (size_t)code->co_framesize * sizeof(PyObject *); + _PyInterpreterFrame *frame = PyMem_Malloc(nbytes); + _PyInterpreterFrame *copy = PyMem_Malloc(nbytes); + if (frame == NULL || copy == NULL) { + PyMem_Free(frame); + PyMem_Free(copy); + return PyErr_NoMemory(); + } + memset(frame, 0xFF, nbytes); + memset(copy, 0xFF, nbytes); + PyThreadState *tstate = PyThreadState_Get(); + _PyFrame_Initialize(tstate, frame, PyStackRef_FromPyObjectNew(func), + NULL, code, 0, NULL); + int err = 0; + if (frame->external) { + PyErr_SetString(PyExc_AssertionError, + "_PyFrame_Initialize left external bit set"); + err = 1; + } + // Release frame's owned references before copying: _PyFrame_Copy + // aliases strong references into the copy without creating new ones, + // so copying NULLs keeps teardown trivially balanced. + PyStackRef_CLOSE(frame->f_executable); + PyStackRef_CLOSE(frame->f_funcobj); + frame->f_executable = PyStackRef_NULL; + frame->f_funcobj = PyStackRef_NULL; + _PyFrame_Copy(frame, copy); + if (!err && copy->external) { + PyErr_SetString(PyExc_AssertionError, + "_PyFrame_Copy left external bit set"); + err = 1; + } + PyMem_Free(frame); + PyMem_Free(copy); + if (err) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +frame_no_external_frames_on_stack(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyThreadState *tstate = PyThreadState_Get(); + for (_PyInterpreterFrame *f = tstate->current_frame; + f != NULL; f = f->previous) + { + if (f->external) { + PyErr_SetString(PyExc_AssertionError, + "VM frame has external bit set"); + return NULL; + } + } + Py_RETURN_NONE; +} + #ifdef Py_GIL_DISABLED static PyObject *external_test_visited_obj = NULL; static int external_test_visitor_calls = 0; @@ -3474,6 +3540,10 @@ static PyMethodDef module_functions[] = { {"iframe_getcode", iframe_getcode, METH_O, NULL}, {"iframe_getline", iframe_getline, METH_O, NULL}, {"iframe_getlasti", iframe_getlasti, METH_O, NULL}, + {"frame_init_clears_external_bits", frame_init_clears_external_bits, + METH_O, NULL}, + {"frame_no_external_frames_on_stack", frame_no_external_frames_on_stack, + METH_NOARGS, NULL}, {"external_frame_gc_test", external_frame_gc_test, METH_VARARGS, NULL}, {"code_returns_only_none", code_returns_only_none, METH_O, NULL}, {"get_co_framesize", get_co_framesize, METH_O, NULL}, diff --git a/Modules/_testinternalcapi/interpreter.c b/Modules/_testinternalcapi/interpreter.c index a67ecdc211bb34..a6d70a250baa6d 100644 --- a/Modules/_testinternalcapi/interpreter.c +++ b/Modules/_testinternalcapi/interpreter.c @@ -87,6 +87,7 @@ Test_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) entry.frame.stackpointer = entry.stack; entry.frame.owner = FRAME_OWNED_BY_INTERPRETER; entry.frame.visited = 0; + entry.frame.external = 0; entry.frame.return_offset = 0; #ifdef Py_DEBUG entry.frame.lltrace = 0; diff --git a/Python/ceval.c b/Python/ceval.c index 2b8c425789c039..08b4fa9f64b9eb 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1301,6 +1301,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int entry.frame.stackpointer = entry.stack; entry.frame.owner = FRAME_OWNED_BY_INTERPRETER; entry.frame.visited = 0; + entry.frame.external = 0; entry.frame.return_offset = 0; #ifdef Py_DEBUG entry.frame.lltrace = 0; diff --git a/Python/pystate.c b/Python/pystate.c index f9debd1e20e286..00df28f8c20681 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1598,6 +1598,7 @@ init_threadstate(_PyThreadStateImpl *_tstate, _tstate->base_frame.return_offset = 0; _tstate->base_frame.owner = FRAME_OWNED_BY_INTERPRETER; _tstate->base_frame.visited = 0; + _tstate->base_frame.external = 0; #ifdef Py_DEBUG _tstate->base_frame.lltrace = 0; #endif