diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 11f2cda82099868..cd406d90ce3269b 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -204,6 +204,44 @@ def test_issue10254(self): b = u'C\u0338' * 20 + u'\xC7' self.assertEqual(self.db.normalize('NFC', a), b) + def test_long_combining_mark_run(self): + # gh-149079 (CVE-2026-3276): avoid quadratic canonical ordering. + payload = u"a" + (u"\u0300\u0327" * 32) + nfd = u"a" + (u"\u0327" * 32) + (u"\u0300" * 32) + nfc = u"\u00e0" + (u"\u0327" * 32) + (u"\u0300" * 31) + + self.assertEqual(self.db.normalize("NFD", payload), nfd) + self.assertEqual(self.db.normalize("NFKD", payload), nfd) + self.assertEqual(self.db.normalize("NFC", payload), nfc) + self.assertEqual(self.db.normalize("NFKC", payload), nfc) + + def test_combining_mark_run_fast_paths(self): + # gh-149079: cover short runs and already-sorted long runs. + short_payload = u"a" + (u"\u0300\u0327" * 9) + u"\u0300" + short_nfd = u"a" + (u"\u0327" * 9) + (u"\u0300" * 10) + short_nfc = u"\u00e0" + (u"\u0327" * 9) + (u"\u0300" * 9) + long_sorted = u"a" + (u"\u0327" * 30) + (u"\u0300" * 30) + long_sorted_nfc = u"\u00e0" + (u"\u0327" * 30) + (u"\u0300" * 29) + + self.assertEqual(self.db.normalize("NFD", short_payload), short_nfd) + self.assertEqual(self.db.normalize("NFKD", short_payload), short_nfd) + self.assertEqual(self.db.normalize("NFC", short_payload), short_nfc) + self.assertEqual(self.db.normalize("NFKC", short_payload), short_nfc) + self.assertEqual(self.db.normalize("NFD", long_sorted), long_sorted) + self.assertEqual(self.db.normalize("NFKD", long_sorted), long_sorted) + self.assertEqual(self.db.normalize("NFC", long_sorted), long_sorted_nfc) + self.assertEqual(self.db.normalize("NFKC", long_sorted), long_sorted_nfc) + + def test_combining_mark_run_is_linear(self): + # gh-149079 (CVE-2026-3276): a run of 100000 combining marks in + # alternating canonical combining class order took about 1.5 s with + # the quadratic sort (and 20 s for 400000); it now takes a few + # milliseconds. This checks the counting sort on a long run. + n = 50000 + payload = u"a" + (u"\u0300\u0327" * n) + nfd = u"a" + (u"\u0327" * n) + (u"\u0300" * n) + self.assertEqual(self.db.normalize("NFD", payload), nfd) + def test_issue29456(self): # Fix #29456 u1176_str_a = u'\u1100\u1176\u11a8' diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index df6ffe343c3f1c5..aedf41f7d44f4d1 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -493,6 +493,56 @@ get_decomp_record(PyObject *self, Py_UCS4 code, int *index, int *prefix, int *co #define NCount (VCount*TCount) #define SCount (LCount*NCount) +/* Small combining runs are usually cheaper with insertion sort. */ +#define CANONICAL_ORDERING_COUNTING_SORT_THRESHOLD 20 + +static void +canonical_ordering_sort_insertion(Py_UNICODE *start, Py_UNICODE *end) +{ + Py_UNICODE *i, *j; + + for (i = start + 1; i < end; i++) { + Py_UNICODE code = *i; + unsigned char combining = _getrecord_ex(code)->combining; + + j = i; + while (j > start) { + if (_getrecord_ex(j[-1])->combining <= combining) + break; + *j = j[-1]; + j--; + } + if (j != i) + *j = code; + } +} + +static void +canonical_ordering_sort_counting(Py_UNICODE *start, Py_UNICODE *end, + Py_UNICODE *sortbuf) +{ + Py_ssize_t counts[256]; + Py_ssize_t total = 0; + Py_UNICODE *i; + int c; + + memset(counts, 0, sizeof(counts)); + for (i = start; i < end; i++) + counts[_getrecord_ex(*i)->combining]++; + + for (c = 0; c < 256; c++) { + Py_ssize_t count = counts[c]; + counts[c] = total; + total += count; + } + + /* Reuse counts[] as the next output slot for each CCC; this is a + stable sort, as canonical ordering requires. */ + for (i = start; i < end; i++) + sortbuf[counts[_getrecord_ex(*i)->combining]++] = *i; + memcpy(start, sortbuf, (end - start) * sizeof(Py_UNICODE)); +} + static PyObject* nfd_nfkd(PyObject *self, PyObject *input, int k) { @@ -503,6 +553,8 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) Py_ssize_t space, isize; int index, prefix, count, stackptr; unsigned char prev, cur; + Py_UNICODE *sortbuf = NULL; + Py_ssize_t sortbuflen = 0; stackptr = 0; isize = PyUnicode_GET_SIZE(input); @@ -581,31 +633,58 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) /* Drop overallocation. Cannot fail. */ PyUnicode_Resize(&result, PyUnicode_GET_SIZE(result) - space); - /* Sort canonically. */ + /* Sort each consecutive combining-character run canonically. The + old in-place insertion sort was O(n**2) in the run length + (gh-149079, CVE-2026-3276). */ i = PyUnicode_AS_UNICODE(result); - prev = _getrecord_ex(*i)->combining; end = i + PyUnicode_GET_SIZE(result); - for (i++; i < end; i++) { - cur = _getrecord_ex(*i)->combining; - if (prev == 0 || cur == 0 || prev <= cur) { - prev = cur; + while (i < end) { + Py_UNICODE *run_start; + Py_ssize_t run_length; + int needs_sort = 0; + + prev = _getrecord_ex(*i)->combining; + if (prev == 0) { + i++; continue; } - /* Non-canonical order. Need to switch *i with previous. */ - o = i - 1; - while (1) { - Py_UNICODE tmp = o[1]; - o[1] = o[0]; - o[0] = tmp; - o--; - if (o < PyUnicode_AS_UNICODE(result)) - break; - prev = _getrecord_ex(*o)->combining; - if (prev == 0 || prev <= cur) + + run_start = i++; + while (i < end) { + cur = _getrecord_ex(*i)->combining; + if (cur == 0) break; + if (prev > cur) + needs_sort = 1; + prev = cur; + i++; } - prev = _getrecord_ex(*i)->combining; + if (!needs_sort) + continue; + + run_length = i - run_start; + if (run_length < CANONICAL_ORDERING_COUNTING_SORT_THRESHOLD) { + canonical_ordering_sort_insertion(run_start, i); + continue; + } + + if (run_length > sortbuflen) { + Py_UNICODE *new_sortbuf = NULL; + if ((size_t)run_length <= PY_SSIZE_T_MAX / sizeof(Py_UNICODE)) + new_sortbuf = (Py_UNICODE *)PyMem_Realloc( + sortbuf, run_length * sizeof(Py_UNICODE)); + if (new_sortbuf == NULL) { + PyErr_NoMemory(); + PyMem_Free(sortbuf); + Py_DECREF(result); + return NULL; + } + sortbuf = new_sortbuf; + sortbuflen = run_length; + } + canonical_ordering_sort_counting(run_start, i, sortbuf); } + PyMem_Free(sortbuf); return result; }