From 3a6fae44df9b518c2405a9957edca9d5420c30d8 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Thu, 22 Oct 2026 08:25:55 +0000 Subject: [PATCH 1/2] gh-160324: Fix io.StringIO state after MemoryError Allocate before consuming the accumulated writes in realize() and make_intermediate(), and reset the size when the contents are lost, so that later calls do not index outside the internal buffer. --- Lib/test/test_io/test_memoryio.py | 34 +++++++++++++++++ ...-10-22-08-15-56.gh-issue-160324.sIoRlz.rst | 3 ++ Modules/_io/stringio.c | 38 +++++++++++-------- 3 files changed, 59 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-22-08-15-56.gh-issue-160324.sIoRlz.rst diff --git a/Lib/test/test_io/test_memoryio.py b/Lib/test/test_io/test_memoryio.py index 90bdc791787439..5f0a407ff5f610 100644 --- a/Lib/test/test_io/test_memoryio.py +++ b/Lib/test/test_io/test_memoryio.py @@ -1136,6 +1136,40 @@ def __str__(self): memio2.write(MyStr("world")) self.assertEqual(memio2.getvalue(), "hello world") + @support.nomemtest + def test_memory_error_consistent_state(self): + # A MemoryError while copying the accumulated writes into the + # internal buffer must leave the size of the stream consistent with + # the buffer, so that using the object again does not read or write + # outside it. + piece = "0123456789" * 10 + text = piece * 50 + calls = [("readline",), ("read", 5), ("read",), ("getvalue",), + ("truncate", 3), ("write", "ab"), ("__next__",)] + for name, *args in calls: + for n in range(3): + with self.subTest(method=name, n=n): + memio = self.ioclass() + for _ in range(50): + memio.write(piece) + memio.seek(0) + method = getattr(memio, name) + try: + with support.inject_memory_error_cm(n, n + 1): + method(*args) + except MemoryError: + pass + else: + continue + # The contents are either intact or lost, never garbage. + value = memio.getvalue() + self.assertIn(value, (text, "")) + self.assertEqual(memio.seek(0, 2), len(value)) + memio.seek(0) + self.assertEqual(memio.readline(), value) + memio.write("!") + self.assertEqual(memio.getvalue(), value + "!") + class CStringIOPickleTest(PyStringIOPickleTest): UnsupportedOperation = io.UnsupportedOperation diff --git a/Misc/NEWS.d/next/Library/2026-10-22-08-15-56.gh-issue-160324.sIoRlz.rst b/Misc/NEWS.d/next/Library/2026-10-22-08-15-56.gh-issue-160324.sIoRlz.rst new file mode 100644 index 00000000000000..f711469af7445b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-22-08-15-56.gh-issue-160324.sIoRlz.rst @@ -0,0 +1,3 @@ +Fix a crash or garbage data when an :class:`io.StringIO` object is used again +after one of its methods failed with :exc:`MemoryError`. The object now keeps +its contents or becomes empty. diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index e9a46167cff596..f9425fd6a91eb1 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -146,22 +146,24 @@ resize_buffer(stringio *self, size_t size) static PyObject * make_intermediate(stringio *self) { - PyObject *intermediate = PyUnicodeWriter_Finish(self->writer); - self->writer = NULL; - self->state = STATE_REALIZED; - if (intermediate == NULL) + /* Create the new writer first: if this fails, the object keeps its + contents. */ + PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); + if (writer == NULL) return NULL; - self->writer = PyUnicodeWriter_Create(0); - if (self->writer == NULL) { - Py_DECREF(intermediate); + PyObject *intermediate = PyUnicodeWriter_Finish(self->writer); + self->writer = writer; + if (intermediate == NULL) { + /* The contents are lost, the new writer is empty. */ + self->string_size = 0; return NULL; } if (PyUnicodeWriter_WriteStr(self->writer, intermediate)) { Py_DECREF(intermediate); + self->string_size = 0; return NULL; } - self->state = STATE_ACCUMULATING; return intermediate; } @@ -174,23 +176,27 @@ realize(stringio *self) if (self->state == STATE_REALIZED) return 0; assert(self->state == STATE_ACCUMULATING); + + /* Allocate the internal buffer first: if this fails, the object stays + in the accumulating state and keeps its contents. */ + if (resize_buffer(self, self->string_size) < 0) + return -1; self->state = STATE_REALIZED; intermediate = PyUnicodeWriter_Finish(self->writer); self->writer = NULL; - if (intermediate == NULL) + if (intermediate == NULL) { + /* The contents are lost, the internal buffer contains nothing. */ + self->string_size = 0; return -1; + } - /* Append the intermediate string to the internal buffer. - The length should be equal to the current cursor position. - */ + /* Copy the intermediate string to the internal buffer. */ len = PyUnicode_GET_LENGTH(intermediate); - if (resize_buffer(self, len) < 0) { - Py_DECREF(intermediate); - return -1; - } + assert(len == self->string_size); if (!PyUnicode_AsUCS4(intermediate, self->buf, len, 0)) { Py_DECREF(intermediate); + self->string_size = 0; return -1; } From 8cfce4eaef80126468b4f1c73c01b54966fdd390 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Thu, 22 Oct 2026 19:22:08 +0000 Subject: [PATCH 2/2] gh-160324: Keep io.StringIO contents after MemoryError Shrink the buffer of the accumulating writer before finishing it, so that getvalue(), read() and the other methods leave the contents unchanged when they fail with MemoryError instead of dropping them. --- Lib/test/test_io/test_memoryio.py | 11 +++-- ...-10-22-08-15-56.gh-issue-160324.sIoRlz.rst | 2 +- Modules/_io/stringio.c | 41 ++++++++++++++++--- 3 files changed, 42 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_io/test_memoryio.py b/Lib/test/test_io/test_memoryio.py index 5f0a407ff5f610..a2f5b94a8631a4 100644 --- a/Lib/test/test_io/test_memoryio.py +++ b/Lib/test/test_io/test_memoryio.py @@ -1161,14 +1161,13 @@ def test_memory_error_consistent_state(self): pass else: continue - # The contents are either intact or lost, never garbage. - value = memio.getvalue() - self.assertIn(value, (text, "")) - self.assertEqual(memio.seek(0, 2), len(value)) + # The contents are unchanged. + self.assertEqual(memio.getvalue(), text) + self.assertEqual(memio.seek(0, 2), len(text)) memio.seek(0) - self.assertEqual(memio.readline(), value) + self.assertEqual(memio.readline(), text) memio.write("!") - self.assertEqual(memio.getvalue(), value + "!") + self.assertEqual(memio.getvalue(), text + "!") class CStringIOPickleTest(PyStringIOPickleTest): diff --git a/Misc/NEWS.d/next/Library/2026-10-22-08-15-56.gh-issue-160324.sIoRlz.rst b/Misc/NEWS.d/next/Library/2026-10-22-08-15-56.gh-issue-160324.sIoRlz.rst index f711469af7445b..d54069128e2c0e 100644 --- a/Misc/NEWS.d/next/Library/2026-10-22-08-15-56.gh-issue-160324.sIoRlz.rst +++ b/Misc/NEWS.d/next/Library/2026-10-22-08-15-56.gh-issue-160324.sIoRlz.rst @@ -1,3 +1,3 @@ Fix a crash or garbage data when an :class:`io.StringIO` object is used again after one of its methods failed with :exc:`MemoryError`. The object now keeps -its contents or becomes empty. +its contents. diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index f9425fd6a91eb1..263feb649f8d98 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -1,6 +1,7 @@ #include "Python.h" #include // offsetof() #include "pycore_object.h" +#include "pycore_unicodeobject.h" // _PyUnicode_ResizeCompact() #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "_iomodule.h" @@ -143,14 +144,39 @@ resize_buffer(stringio *self, size_t size) return -1; } +/* Shrink the buffer of the writer to its contents, so that the following + PyUnicodeWriter_Finish() has nothing to allocate and cannot fail. + If this fails, the writer is unchanged. */ +static int +shrink_writer(stringio *self) +{ + _PyUnicodeWriter *writer = (_PyUnicodeWriter *)self->writer; + if (writer->readonly || writer->buffer == NULL + || writer->pos == 0 || writer->pos == writer->size) + { + return 0; + } + PyObject *buffer = _PyUnicode_ResizeCompact(writer->buffer, writer->pos); + if (buffer == NULL) { + return -1; + } + writer->buffer = NULL; + _PyUnicodeWriter_SetBuffer(writer, buffer); + return 0; +} + static PyObject * make_intermediate(stringio *self) { - /* Create the new writer first: if this fails, the object keeps its - contents. */ + /* Create the new writer and shrink the old one first: if this fails, + the object keeps its contents. */ PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); if (writer == NULL) return NULL; + if (shrink_writer(self) < 0) { + PyUnicodeWriter_Discard(writer); + return NULL; + } PyObject *intermediate = PyUnicodeWriter_Finish(self->writer); self->writer = writer; @@ -177,10 +203,13 @@ realize(stringio *self) return 0; assert(self->state == STATE_ACCUMULATING); - /* Allocate the internal buffer first: if this fails, the object stays - in the accumulating state and keeps its contents. */ + /* Allocate the internal buffer and shrink the writer first: if this + fails, the object stays in the accumulating state and keeps its + contents. */ if (resize_buffer(self, self->string_size) < 0) return -1; + if (shrink_writer(self) < 0) + return -1; self->state = STATE_REALIZED; intermediate = PyUnicodeWriter_Finish(self->writer); @@ -364,7 +393,9 @@ _io_StringIO_read_impl(stringio *self, Py_ssize_t size) /* Optimization for seek(0); read() */ if (self->state == STATE_ACCUMULATING && self->pos == 0 && size == n) { PyObject *result = make_intermediate(self); - self->pos = self->string_size; + if (result != NULL) { + self->pos = self->string_size; + } return result; }