From 4ea11bd2711230b3bfb5b66c060953b3429ddc93 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Wed, 28 Oct 2026 10:35:34 -0500 Subject: [PATCH] gh-149079: Fix O(n^2) canonical ordering in unicodedata.normalize() (CVE-2026-3276) Backport of python/cpython gh-149079 (991224b1e8; 3.10: db744c0776c) to 2.7.18.x. nfd_nfkd(), which every normalization form goes through, put combining characters into canonical order with an in-place insertion sort. A long run of combining marks in alternating canonical combining class order made it quadratic. On 2.7.18.15, NFC of u"a" + u"\u0300\u0327" * n takes 0.22 s for n=20000, 1.6 s for n=50000 and 19.9 s for n=200000. Each run of non-starters is now sorted on its own: insertion sort for runs shorter than 20, otherwise a stable counting sort on the combining class through a reusable buffer, and runs already in order are skipped. The same input now takes 0.001 s, 0.004 s and 0.015 s. 2.7.18.x: the sort works on the Py_UNICODE result buffer instead of PyUnicode_READ/WRITE. On narrow (UCS-2) builds it therefore sees UTF-16 code units, as the old sort did; surrogates have combining class 0, so they end runs as before. C89 declarations. The buffer grows through PyMem_Realloc rather than upstream's PyMem_Resize, which in 2.7 overwrites its pointer argument and would leak the old buffer on failure. Tests: upstream's test_long_combining_mark_run and test_combining_mark_run_fast_paths, plus test_combining_mark_run_is_linear (a 100000-mark run through the counting sort). On 20000 random strings for each of three seeds, mostly combining marks with runs of 1-200, NFC/NFD/NFKC/NFKD from both unicodedata and ucd_3_2_0 hash identically to 2.7.18.15. Co-Authored-By: Claude Opus 5.5 --- Lib/test/test_unicodedata.py | 38 ++++++++++++ Modules/unicodedata.c | 115 +++++++++++++++++++++++++++++------ 2 files changed, 135 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 11f2cda82099868..cd406d90ce3269b 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -204,6 +204,44 @@ def test_issue10254(self): b = u'C\u0338' * 20 + u'\xC7' self.assertEqual(self.db.normalize('NFC', a), b) + def test_long_combining_mark_run(self): + # gh-149079 (CVE-2026-3276): avoid quadratic canonical ordering. + payload = u"a" + (u"\u0300\u0327" * 32) + nfd = u"a" + (u"\u0327" * 32) + (u"\u0300" * 32) + nfc = u"\u00e0" + (u"\u0327" * 32) + (u"\u0300" * 31) + + self.assertEqual(self.db.normalize("NFD", payload), nfd) + self.assertEqual(self.db.normalize("NFKD", payload), nfd) + self.assertEqual(self.db.normalize("NFC", payload), nfc) + self.assertEqual(self.db.normalize("NFKC", payload), nfc) + + def test_combining_mark_run_fast_paths(self): + # gh-149079: cover short runs and already-sorted long runs. + short_payload = u"a" + (u"\u0300\u0327" * 9) + u"\u0300" + short_nfd = u"a" + (u"\u0327" * 9) + (u"\u0300" * 10) + short_nfc = u"\u00e0" + (u"\u0327" * 9) + (u"\u0300" * 9) + long_sorted = u"a" + (u"\u0327" * 30) + (u"\u0300" * 30) + long_sorted_nfc = u"\u00e0" + (u"\u0327" * 30) + (u"\u0300" * 29) + + self.assertEqual(self.db.normalize("NFD", short_payload), short_nfd) + self.assertEqual(self.db.normalize("NFKD", short_payload), short_nfd) + self.assertEqual(self.db.normalize("NFC", short_payload), short_nfc) + self.assertEqual(self.db.normalize("NFKC", short_payload), short_nfc) + self.assertEqual(self.db.normalize("NFD", long_sorted), long_sorted) + self.assertEqual(self.db.normalize("NFKD", long_sorted), long_sorted) + self.assertEqual(self.db.normalize("NFC", long_sorted), long_sorted_nfc) + self.assertEqual(self.db.normalize("NFKC", long_sorted), long_sorted_nfc) + + def test_combining_mark_run_is_linear(self): + # gh-149079 (CVE-2026-3276): a run of 100000 combining marks in + # alternating canonical combining class order took about 1.5 s with + # the quadratic sort (and 20 s for 400000); it now takes a few + # milliseconds. This checks the counting sort on a long run. + n = 50000 + payload = u"a" + (u"\u0300\u0327" * n) + nfd = u"a" + (u"\u0327" * n) + (u"\u0300" * n) + self.assertEqual(self.db.normalize("NFD", payload), nfd) + def test_issue29456(self): # Fix #29456 u1176_str_a = u'\u1100\u1176\u11a8' diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index df6ffe343c3f1c5..aedf41f7d44f4d1 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -493,6 +493,56 @@ get_decomp_record(PyObject *self, Py_UCS4 code, int *index, int *prefix, int *co #define NCount (VCount*TCount) #define SCount (LCount*NCount) +/* Small combining runs are usually cheaper with insertion sort. */ +#define CANONICAL_ORDERING_COUNTING_SORT_THRESHOLD 20 + +static void +canonical_ordering_sort_insertion(Py_UNICODE *start, Py_UNICODE *end) +{ + Py_UNICODE *i, *j; + + for (i = start + 1; i < end; i++) { + Py_UNICODE code = *i; + unsigned char combining = _getrecord_ex(code)->combining; + + j = i; + while (j > start) { + if (_getrecord_ex(j[-1])->combining <= combining) + break; + *j = j[-1]; + j--; + } + if (j != i) + *j = code; + } +} + +static void +canonical_ordering_sort_counting(Py_UNICODE *start, Py_UNICODE *end, + Py_UNICODE *sortbuf) +{ + Py_ssize_t counts[256]; + Py_ssize_t total = 0; + Py_UNICODE *i; + int c; + + memset(counts, 0, sizeof(counts)); + for (i = start; i < end; i++) + counts[_getrecord_ex(*i)->combining]++; + + for (c = 0; c < 256; c++) { + Py_ssize_t count = counts[c]; + counts[c] = total; + total += count; + } + + /* Reuse counts[] as the next output slot for each CCC; this is a + stable sort, as canonical ordering requires. */ + for (i = start; i < end; i++) + sortbuf[counts[_getrecord_ex(*i)->combining]++] = *i; + memcpy(start, sortbuf, (end - start) * sizeof(Py_UNICODE)); +} + static PyObject* nfd_nfkd(PyObject *self, PyObject *input, int k) { @@ -503,6 +553,8 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) Py_ssize_t space, isize; int index, prefix, count, stackptr; unsigned char prev, cur; + Py_UNICODE *sortbuf = NULL; + Py_ssize_t sortbuflen = 0; stackptr = 0; isize = PyUnicode_GET_SIZE(input); @@ -581,31 +633,58 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) /* Drop overallocation. Cannot fail. */ PyUnicode_Resize(&result, PyUnicode_GET_SIZE(result) - space); - /* Sort canonically. */ + /* Sort each consecutive combining-character run canonically. The + old in-place insertion sort was O(n**2) in the run length + (gh-149079, CVE-2026-3276). */ i = PyUnicode_AS_UNICODE(result); - prev = _getrecord_ex(*i)->combining; end = i + PyUnicode_GET_SIZE(result); - for (i++; i < end; i++) { - cur = _getrecord_ex(*i)->combining; - if (prev == 0 || cur == 0 || prev <= cur) { - prev = cur; + while (i < end) { + Py_UNICODE *run_start; + Py_ssize_t run_length; + int needs_sort = 0; + + prev = _getrecord_ex(*i)->combining; + if (prev == 0) { + i++; continue; } - /* Non-canonical order. Need to switch *i with previous. */ - o = i - 1; - while (1) { - Py_UNICODE tmp = o[1]; - o[1] = o[0]; - o[0] = tmp; - o--; - if (o < PyUnicode_AS_UNICODE(result)) - break; - prev = _getrecord_ex(*o)->combining; - if (prev == 0 || prev <= cur) + + run_start = i++; + while (i < end) { + cur = _getrecord_ex(*i)->combining; + if (cur == 0) break; + if (prev > cur) + needs_sort = 1; + prev = cur; + i++; } - prev = _getrecord_ex(*i)->combining; + if (!needs_sort) + continue; + + run_length = i - run_start; + if (run_length < CANONICAL_ORDERING_COUNTING_SORT_THRESHOLD) { + canonical_ordering_sort_insertion(run_start, i); + continue; + } + + if (run_length > sortbuflen) { + Py_UNICODE *new_sortbuf = NULL; + if ((size_t)run_length <= PY_SSIZE_T_MAX / sizeof(Py_UNICODE)) + new_sortbuf = (Py_UNICODE *)PyMem_Realloc( + sortbuf, run_length * sizeof(Py_UNICODE)); + if (new_sortbuf == NULL) { + PyErr_NoMemory(); + PyMem_Free(sortbuf); + Py_DECREF(result); + return NULL; + } + sortbuf = new_sortbuf; + sortbuflen = run_length; + } + canonical_ordering_sort_counting(run_start, i, sortbuf); } + PyMem_Free(sortbuf); return result; }