From 98e6d6e8e0c152ea4284b64e185d06ad996ac38f Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Thu, 5 Nov 2026 15:05:55 -0800 Subject: [PATCH 1/2] gh-161242: Cache strerror() messages per errno _Py_strerror() decodes strerror_r() with the locale codec on every call, a large part of the cost of raising OSError. Cache the result for codes 0-255 in a per-interpreter dict; _locale.setlocale() clears it. PGO+LTO, main vs this change: | Benchmark | main | strerror cache | |-----------------|:-------:|:---------------------:| | os.strerror(2) | 130 ns | 21.9 ns: 5.93x faster | | stat_missing() | 935 ns | 724 ns: 1.29x faster | | open_missing() | 1.35 us | 1.15 us: 1.17x faster | | raise_oserror() | 105 ns | 107 ns: 1.02x slower | Benchmark: ```python import os import pyperf def stat_missing(): try: os.stat('/nonexistent') except OSError: pass def open_missing(): try: open('/nonexistent', 'rb') except OSError: pass def raise_oserror(): try: raise OSError(2, 'No such file or directory') except OSError: pass runner = pyperf.Runner() runner.bench_func("stat_missing()", stat_missing) runner.bench_func("open_missing()", open_missing) runner.bench_func("raise_oserror()", raise_oserror) runner.bench_func("os.strerror(2)", os.strerror, 2) ``` Co-Authored-By: Claude Opus 5.5 --- Include/internal/pycore_exceptions.h | 2 ++ Lib/test/test_exception_hierarchy.py | 12 ++++++++++ Modules/_localemodule.c | 3 +++ Objects/exceptions.c | 4 +++- Python/fileutils.c | 36 ++++++++++++++++++++++------ 5 files changed, 49 insertions(+), 8 deletions(-) diff --git a/Include/internal/pycore_exceptions.h b/Include/internal/pycore_exceptions.h index 26456d1966bbb0e..8b331308105092a 100644 --- a/Include/internal/pycore_exceptions.h +++ b/Include/internal/pycore_exceptions.h @@ -22,6 +22,8 @@ extern void _PyExc_Fini(PyInterpreterState *); struct _Py_exc_state { // The dict mapping from errno codes to OSError subclasses PyObject *errnomap; + // errno code -> _Py_strerror() message + PyObject *strerror_cache; PyBaseExceptionObject *memerrors_freelist; int memerrors_numfree; #ifdef Py_GIL_DISABLED diff --git a/Lib/test/test_exception_hierarchy.py b/Lib/test/test_exception_hierarchy.py index 3318fa8e7746f7a..d0c3a22e029d807 100644 --- a/Lib/test/test_exception_hierarchy.py +++ b/Lib/test/test_exception_hierarchy.py @@ -5,6 +5,7 @@ import unittest import errno from errno import EEXIST +from test.support import cpython_only class SubOSError(OSError): @@ -136,6 +137,17 @@ def test_posix_error(self): if os.name == "nt": self.assertEqual(e.winerror, None) + @cpython_only + @unittest.skipIf(os.name == "nt", "POSIX-specific test") + def test_strerror_cached(self): + def strerror(): + try: + os.stat('') + except FileNotFoundError as e: + return e.strerror + self.assertIs(strerror(), strerror()) + self.assertIs(strerror(), os.strerror(errno.ENOENT)) + @unittest.skipUnless(os.name == "nt", "Windows-specific test") def test_errno_translation(self): # ERROR_ALREADY_EXISTS (183) -> EEXIST diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index ba8b421c25a47cc..908b31134f45cec 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -12,6 +12,7 @@ This software comes with no warranty. Use at your own risk. #include "Python.h" #include "pycore_fileutils.h" // _Py_GetLocaleconvNumeric() #include "pycore_pymem.h" // _PyMem_Strdup() +#include "pycore_pystate.h" // _PyInterpreterState_GET() #include // setlocale() #include // strlen() @@ -169,6 +170,8 @@ _locale_setlocale_impl(PyObject *module, int category, const char *locale) "unsupported locale setting"); return NULL; } + // strerror() messages depend on LC_MESSAGES and LC_CTYPE + PyDict_Clear(_PyInterpreterState_GET()->exc_state.strerror_cache); result_object = PyUnicode_DecodeLocale(result, NULL); if (!result_object) return NULL; diff --git a/Objects/exceptions.c b/Objects/exceptions.c index e2f757c5ee9c0be..cbd19fdb88eb28d 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -4608,7 +4608,8 @@ _PyExc_InitState(PyInterpreterState *interp) /* Add exceptions to errnomap */ assert(state->errnomap == NULL); state->errnomap = PyDict_New(); - if (!state->errnomap) { + state->strerror_cache = PyDict_New(); + if (!state->errnomap || !state->strerror_cache) { return _PyStatus_NO_MEMORY(); } @@ -4699,6 +4700,7 @@ _PyExc_Fini(PyInterpreterState *interp) struct _Py_exc_state *state = &interp->exc_state; free_preallocated_memerrors(state); Py_CLEAR(state->errnomap); + Py_CLEAR(state->strerror_cache); _PyExc_FiniTypes(interp); } diff --git a/Python/fileutils.c b/Python/fileutils.c index 3ea0f59f69ad1ef..1f24f5bb95b21de 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -1,5 +1,6 @@ #include "Python.h" #include "pycore_fileutils.h" // fileutils definitions +#include "pycore_long.h" // _PyLong_FromUnsignedChar() #include "pycore_runtime.h" // _PyRuntime #include "pycore_pystate.h" // _Py_AssertHoldsTstate() #include "pycore_unicodeobject.h" // _Py_MAX_UNICODE @@ -3304,13 +3305,8 @@ _Py_IsValidFD(int fd) } -// Call strerror_r(code) if available, or use strerror() otherwise. Decode the -// result from the locale encoding using surrogateescape error handler. -// -// On success, return a Unicode string. On error, set an exception and return -// NULL. -PyObject* -_Py_strerror(int code) +static PyObject * +decode_strerror(int code) /*[clinic end generated code: output=baebf09fa02a78f2 input=75a8673d97915a91]*/ { const char *errors = "surrogateescape"; @@ -3396,3 +3392,29 @@ _Py_strerror(int code) return PyUnicode_DecodeLocale(message, errors); #endif } + + +// Call strerror_r(code) if available, or use strerror() otherwise. Decode the +// result from the locale encoding using surrogateescape error handler. +// Messages of codes 0-255 are cached until the next locale.setlocale(): +// decoding is a large part of the cost of raising OSError. +// +// On success, return a Unicode string. On error, set an exception and return +// NULL. +PyObject* +_Py_strerror(int code) +{ + PyObject *cache = _PyInterpreterState_GET()->exc_state.strerror_cache; + if (cache == NULL || code < 0 || code >= 256) { + return decode_strerror(code); + } + PyObject *key = _PyLong_FromUnsignedChar((unsigned char)code); + PyObject *message; + if (PyDict_GetItemRef(cache, key, &message) == 0) { + message = decode_strerror(code); + if (message != NULL && PyDict_SetItem(cache, key, message) < 0) { + Py_CLEAR(message); + } + } + return message; +} From 162984394811df1fb705f95554d14f203a04a2f2 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Thu, 5 Nov 2026 16:49:28 -0800 Subject: [PATCH 2/2] Use open() of a missing file in test_strerror_cached os.stat('') doesn't raise on WASI. Co-Authored-By: Claude Opus 5.5 --- Lib/test/test_exception_hierarchy.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_exception_hierarchy.py b/Lib/test/test_exception_hierarchy.py index d0c3a22e029d807..271045561555924 100644 --- a/Lib/test/test_exception_hierarchy.py +++ b/Lib/test/test_exception_hierarchy.py @@ -141,10 +141,9 @@ def test_posix_error(self): @unittest.skipIf(os.name == "nt", "POSIX-specific test") def test_strerror_cached(self): def strerror(): - try: - os.stat('') - except FileNotFoundError as e: - return e.strerror + with self.assertRaises(FileNotFoundError) as cm: + open("some_hopefully_non_existing_file") + return cm.exception.strerror self.assertIs(strerror(), strerror()) self.assertIs(strerror(), os.strerror(errno.ENOENT))