Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions Lib/test/test_io/test_memoryio.py
Original file line number Diff line number Diff line change
Expand Up @@ -1054,6 +1054,52 @@ def test_cow_mutable(self):
memio = self.ioclass(ba)
self.assertEqual(sys.getrefcount(ba), old_rc)

def test_cow_hashed(self):
# A buffer which was exposed to Python code and hashed must not be
# modified in place, even if BytesIO holds the only reference to it:
# it would keep the cached hash of its old contents.
def write(memio):
memio.seek(0)
memio.write(b"efgh")
def getbuffer(memio):
with memio.getbuffer() as view:
view[:] = b"efgh"
def truncate(memio):
memio.truncate(2)
memio.seek(0)
memio.write(b"efgh")
def getvalue(memio):
return memio.getvalue()
def read(memio):
memio.seek(0)
return memio.read()
def peek(memio):
memio.seek(0)
return memio.peek(4)
for mutation in write, getbuffer, truncate:
for expose in getvalue, read, peek:
with self.subTest(mutation=mutation.__name__,
expose=expose.__name__):
memio = self.ioclass()
memio.write(b"abcd")
# Shrink the overallocated buffer to the exact size,
# so read() and peek() can return it without copying.
memio.getvalue()
hash(expose(memio))
mutation(memio)
self.assertEqual(memio.getvalue(), b"efgh")
self.assertEqual(hash(memio.getvalue()), hash(b"efgh"))
self.assertIn(memio.getvalue(), {b"efgh"})

with self.subTest(mutation=mutation.__name__, expose="__init__"):
initial = bytes(bytearray(b"abcd"))
hash(initial)
memio = self.ioclass(initial)
del initial
mutation(memio)
self.assertEqual(memio.getvalue(), b"efgh")
self.assertEqual(hash(memio.getvalue()), hash(b"efgh"))

def test_write_with_export(self):
memio = self.ioclass(b"abcd")
memio.seek(2)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Fix :meth:`io.BytesIO.getvalue` sometimes returning a :class:`bytes` object
with the hash of earlier contents after the buffer was written to, which made
:class:`dict` and :class:`set` lookups with it fail.
18 changes: 14 additions & 4 deletions Modules/_io/bytesio.c
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ typedef struct {
#define bytesiobuf_CAST(op) ((bytesiobuf *)(op))

/* The bytesio object can be in three states:
* Py_REFCNT(buf) == 1, exports == 0.
* Py_REFCNT(buf) > 1. exports == 0,
* Py_REFCNT(buf) == 1, the hash of buf is not cached, exports == 0.
* Py_REFCNT(buf) > 1 or the hash of buf is cached. exports == 0,
first modification or export causes the internal buffer copying.
* exports > 0. Any modifications are forbidden. Every exported buffer
keeps a reference to buf, so it outlives closing of the bytesio object.
Expand Down Expand Up @@ -77,10 +77,19 @@ check_exports(bytesio *self)
return NULL; \
}

/* If the hash value is cached, the internal bytes object was already used in
Python somehow (ex: as a dictionary key). Modifying it in place would leave
the hash of the old contents, even if nothing else references it anymore. */
#define HASHED_BUF(self) \
(_PyBytes_GET_CACHED_HASH((PyBytesObject *)(self)->buf) != -1)

#ifdef Py_GIL_DISABLED
#define SHARED_BUF(self) ((self)->buf_shared || !_PyObject_IsUniquelyReferenced((self)->buf))
#define SHARED_BUF(self) \
((self)->buf_shared || !_PyObject_IsUniquelyReferenced((self)->buf) \
|| HASHED_BUF(self))
#else
#define SHARED_BUF(self) (!_PyObject_IsUniquelyReferenced((self)->buf))
#define SHARED_BUF(self) \
(!_PyObject_IsUniquelyReferenced((self)->buf) || HASHED_BUF(self))
#endif

static inline void
Expand Down Expand Up @@ -269,6 +278,7 @@ write_bytes_lock_held(bytesio *self, PyObject *b)
goto done;
}
}
assert(_PyBytes_IsMutable(self->buf));

if (self->pos > self->string_size) {
/* In case of overseek, pad with null bytes the buffer region between
Expand Down
Loading