From 6965755b303cf7b756d6c98992cf2e9bb7c6fdd0 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Thu, 22 Oct 2026 00:35:21 +0000 Subject: [PATCH 1/2] gh-160181: Don't modify a hashed bytes buffer in place in io.BytesIO Treat the internal buffer as shared once its hash is cached, so the next modification copies it, as gh-158219 did for bytearray. --- Lib/test/test_io/test_memoryio.py | 46 +++++++++++++++++++ ...-10-22-00-32-51.gh-issue-160181.nnEeg1.rst | 3 ++ Modules/_io/bytesio.c | 17 +++++-- 3 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-22-00-32-51.gh-issue-160181.nnEeg1.rst diff --git a/Lib/test/test_io/test_memoryio.py b/Lib/test/test_io/test_memoryio.py index 90bdc791787439..fbd1d77d99565b 100644 --- a/Lib/test/test_io/test_memoryio.py +++ b/Lib/test/test_io/test_memoryio.py @@ -1054,6 +1054,52 @@ def test_cow_mutable(self): memio = self.ioclass(ba) self.assertEqual(sys.getrefcount(ba), old_rc) + def test_cow_hashed(self): + # A buffer which was exposed to Python code and hashed must not be + # modified in place, even if BytesIO holds the only reference to it: + # it would keep the cached hash of its old contents. + def write(memio): + memio.seek(0) + memio.write(b"efgh") + def getbuffer(memio): + with memio.getbuffer() as view: + view[:] = b"efgh" + def truncate(memio): + memio.truncate(2) + memio.seek(0) + memio.write(b"efgh") + def getvalue(memio): + return memio.getvalue() + def read(memio): + memio.seek(0) + return memio.read() + def peek(memio): + memio.seek(0) + return memio.peek(4) + for mutation in write, getbuffer, truncate: + for expose in getvalue, read, peek: + with self.subTest(mutation=mutation.__name__, + expose=expose.__name__): + memio = self.ioclass() + memio.write(b"abcd") + # Shrink the overallocated buffer to the exact size, + # so read() and peek() can return it without copying. + memio.getvalue() + hash(expose(memio)) + mutation(memio) + self.assertEqual(memio.getvalue(), b"efgh") + self.assertEqual(hash(memio.getvalue()), hash(b"efgh")) + self.assertIn(memio.getvalue(), {b"efgh"}) + + with self.subTest(mutation=mutation.__name__, expose="__init__"): + initial = bytes(bytearray(b"abcd")) + hash(initial) + memio = self.ioclass(initial) + del initial + mutation(memio) + self.assertEqual(memio.getvalue(), b"efgh") + self.assertEqual(hash(memio.getvalue()), hash(b"efgh")) + def test_write_with_export(self): memio = self.ioclass(b"abcd") memio.seek(2) diff --git a/Misc/NEWS.d/next/Library/2026-10-22-00-32-51.gh-issue-160181.nnEeg1.rst b/Misc/NEWS.d/next/Library/2026-10-22-00-32-51.gh-issue-160181.nnEeg1.rst new file mode 100644 index 00000000000000..5644b99cce1720 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-22-00-32-51.gh-issue-160181.nnEeg1.rst @@ -0,0 +1,3 @@ +Fix :meth:`io.BytesIO.getvalue` sometimes returning a :class:`bytes` object +with the hash of earlier contents after the buffer was written to, which made +:class:`dict` and :class:`set` lookups with it fail. diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 2374a43ef74eab..46b7e1fde80cd2 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -39,8 +39,8 @@ typedef struct { #define bytesiobuf_CAST(op) ((bytesiobuf *)(op)) /* The bytesio object can be in three states: - * Py_REFCNT(buf) == 1, exports == 0. - * Py_REFCNT(buf) > 1. exports == 0, + * Py_REFCNT(buf) == 1, the hash of buf is not cached, exports == 0. + * Py_REFCNT(buf) > 1 or the hash of buf is cached. exports == 0, first modification or export causes the internal buffer copying. * exports > 0. Any modifications are forbidden. Every exported buffer keeps a reference to buf, so it outlives closing of the bytesio object. @@ -77,10 +77,19 @@ check_exports(bytesio *self) return NULL; \ } +/* If the hash value is cached, the internal bytes object was already used in + Python somehow (ex: as a dictionary key). Modifying it in place would leave + the hash of the old contents, even if nothing else references it anymore. */ +#define HASHED_BUF(self) \ + (_PyBytes_GET_CACHED_HASH((PyBytesObject *)(self)->buf) != -1) + #ifdef Py_GIL_DISABLED -#define SHARED_BUF(self) ((self)->buf_shared || !_PyObject_IsUniquelyReferenced((self)->buf)) +#define SHARED_BUF(self) \ + ((self)->buf_shared || !_PyObject_IsUniquelyReferenced((self)->buf) \ + || HASHED_BUF(self)) #else -#define SHARED_BUF(self) (!_PyObject_IsUniquelyReferenced((self)->buf)) +#define SHARED_BUF(self) \ + (!_PyObject_IsUniquelyReferenced((self)->buf) || HASHED_BUF(self)) #endif static inline void From 69f5e85e6d0a06d00c00491f745c98f9f1f4c6cf Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Thu, 22 Oct 2026 21:55:20 +0000 Subject: [PATCH 2/2] Assert the buffer is mutable before writing to it in BytesIO --- Modules/_io/bytesio.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 46b7e1fde80cd2..6b09e0521c4b4f 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -278,6 +278,7 @@ write_bytes_lock_held(bytesio *self, PyObject *b) goto done; } } + assert(_PyBytes_IsMutable(self->buf)); if (self->pos > self->string_size) { /* In case of overseek, pad with null bytes the buffer region between