Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions Lib/_pyio.py
Original file line number Diff line number Diff line change
Expand Up @@ -2602,6 +2602,10 @@ def _reset_encoder(position):
self._set_decoded_chars(
self._decoder.decode(input_chunk, need_eof))
self._snapshot = (dec_flags, input_chunk)
if self._decoded_chars:
self._b2cratio = len(input_chunk) / len(self._decoded_chars)
else:
self._b2cratio = 0.0

# Skip chars_to_skip of the decoded characters.
if len(self._decoded_chars) < chars_to_skip:
Expand Down
16 changes: 16 additions & 0 deletions Lib/test/test_io/test_textio.py
Original file line number Diff line number Diff line change
Expand Up @@ -705,6 +705,22 @@ def test_tell_after_readline_with_cr(self):
remaining = f.read()
self.assertEqual(remaining, "")

def test_tell_after_seek_past_cr(self):
# seek() to a position after a carriage return must not leave a
# stale bytes-per-char ratio behind for tell().
for data in (b'one\rtwo\rthree\r', 'ab\r\u20ac\u20ac\u20ac'.encode()):
expected = data.decode().replace('\r', '\n')
for i in range(len(expected)):
with self.subTest(data=data, i=i):
f = self.TextIOWrapper(self.BytesIO(data),
encoding='utf-8')
f.read(i)
pos = f.tell()
f.read(1)
f.seek(pos)
self.assertEqual(f.read(1), expected[i])
f.seek(f.tell())
self.assertEqual(f.read(), expected[i + 1:])

def test_seek_with_encoder_state(self):
f = self.open(os_helper.TESTFN, "w", encoding="euc_jis_2004")
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Fix :meth:`io.TextIOWrapper.tell` after :meth:`~io.TextIOWrapper.seek` to a
position that follows a carriage return. The bytes-per-char ratio was not
updated when the input chunk was replayed, so ``tell()`` could read past an
internal buffer and return a wrong position.
8 changes: 8 additions & 0 deletions Modules/_io/textio.c
Original file line number Diff line number Diff line change
Expand Up @@ -2812,6 +2812,7 @@ _io_TextIOWrapper_seek_impl(textio *self, PyObject *cookieObj, int whence)
goto fail;
}

Py_ssize_t nbytes = PyBytes_GET_SIZE(input_chunk);
snapshot = Py_BuildValue("iN", cookie.dec_flags, input_chunk);
if (snapshot == NULL) {
goto fail;
Expand All @@ -2824,7 +2825,14 @@ _io_TextIOWrapper_seek_impl(textio *self, PyObject *cookieObj, int whence)
if (check_decoded(decoded) < 0)
goto fail;

Py_ssize_t nchars = PyUnicode_GET_LENGTH(decoded);
textiowrapper_set_decoded_chars(self, decoded);
if (nchars > 0) {
self->b2cratio = (double) nbytes / nchars;
}
else {
self->b2cratio = 0.0;
}

/* Skip chars_to_skip of the decoded characters. */
if (PyUnicode_GetLength(self->decoded_chars) < cookie.chars_to_skip) {
Expand Down
Loading