From 337d3ddd1d262fbc9a8e5851402b164526978d38 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Wed, 28 Oct 2026 00:50:27 -0500 Subject: [PATCH 01/23] gh-151981: Make tarfile._Stream.seek break at EOF (CVE-2026-11972) Backport of python/cpython GH-151982 (3.13 3f031d431f). In streaming mode (mode="r|"), _Stream.seek() skipped forward by reading the declared number of blocks even after the stream hit EOF. A member header that declares a huge size (e.g. 1 << 64 via a GNU/PAX header) made getmembers() spin for an effectively unbounded time. seek() now stops reading once read() returns no data, and the following header read raises ReadError("unexpected end of data"). 2.7.18.x: xrange() loop; the test loops over GNU_FORMAT and PAX_FORMAT instead of using support.subTests, and lives in StreamReadTest. Co-Authored-By: Claude Opus 5.5 --- Lib/tarfile.py | 4 +++- Lib/test/test_tarfile.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/Lib/tarfile.py b/Lib/tarfile.py index d1cb4fe15ae27d7..223b573f45122c3 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -556,7 +556,9 @@ def seek(self, pos=0): if pos - self.pos >= 0: blocks, remainder = divmod(pos - self.pos, self.bufsize) for i in xrange(blocks): - self.read(self.bufsize) + data = self.read(self.bufsize) + if not data: + break self.read(remainder) else: raise StreamError("seeking backwards is not allowed") diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index fe66218eaa826fd..8fa499f55bda64c 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -592,6 +592,22 @@ def test_provoke_stream_error(self): f = self.tar.extractfile(tarinfos[0]) # read the first member self.assertRaises(tarfile.StreamError, f.read) + def test_getmembers_big_size(self): + # gh-151981: A loop in seek() for streaming files tried to read the + # declared number of blocks even at EOF + for format in (tarfile.GNU_FORMAT, tarfile.PAX_FORMAT): + tinfo = tarfile.TarInfo("huge-file") + tinfo.size = 1 << 64 + bio = io.BytesIO() + # Write header without data + bio.write(tinfo.tobuf(format)) + + # Reset & try to get contents + bio.seek(0) + with tarfile.open(fileobj=bio, mode="r|") as tar: + with self.assertRaises(tarfile.ReadError): + tar.getmembers() + def test_compare_members(self): tar1 = tarfile.open(tarname, encoding="iso8859-1") try: From b54342dfaaa01cf5ef838abcbfd54911f0bc6147 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Wed, 28 Oct 2026 00:52:19 -0500 Subject: [PATCH 02/23] gh-98820: Fix quadratic time in csv.Sniffer for quoted fields (CVE-2026-18503) Backport of python/cpython gh-98820 (3.12 063d4555c9). Sniffer._guess_quote_and_delimiter() matched a quoted field body with a lazy ".*?", which scans to the end of the sample from every start position: quadratic in the sample size. An attacker-controlled sample passed to Sniffer.sniff() could burn significant CPU. The body of a quoted field now ends at the first quote that is not doubled, as it does for a csv reader. 2.7.18.x: Python 2.7's re module has no possessive quantifiers (*+ is 3.11+), so the body is matched atomically with the equivalent lookahead + backreference idiom (?=(?P...*))(?P=body). A differential fuzz of 20000 random samples under Python 3.12 found the 2.7 pattern and upstream's possessive pattern produce identical matches. assertRaisesRegexp instead of assertRaisesRegex in the test. Co-Authored-By: Claude Opus 5.5 --- Lib/csv.py | 16 +++++++++++----- Lib/test/test_csv.py | 7 +++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/Lib/csv.py b/Lib/csv.py index 70c53ae7791e34a..4ad565f5bff38db 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -214,12 +214,18 @@ def _guess_quote_and_delimiter(self, data, delimiters): this way. """ + # The body of a quoted field ends at the first quote which is + # not doubled, as it does for a reader. A lazy ".*?" scans to + # the end of the sample instead, from every start: quadratically. + # 2.7's re has no possessive "*+", so the body is matched + # atomically with a lookahead and a backreference. + body = r'(?=(?P(?:(?P=quote){2}|(?!(?P=quote)).)*))(?P=body)' matches = [] - for restr in ('(?P[^\w\n"\'])(?P ?)(?P["\']).*?(?P=quote)(?P=delim)', # ,".*?", - '(?:^|\n)(?P["\']).*?(?P=quote)(?P[^\w\n"\'])(?P ?)', # ".*?", - '(?P[^\w\n"\'])(?P ?)(?P["\']).*?(?P=quote)(?:$|\n)', # ,".*?" - '(?:^|\n)(?P["\']).*?(?P=quote)(?:$|\n)'): # ".*?" (no delim, no space) - regexp = re.compile(restr, re.DOTALL | re.MULTILINE) + for restr in (r'(?P[^\w\n"\'])(?P ?)(?P["\'])%s(?P=quote)(?P=delim)', # ,"...", + r'(?:^|\n)(?P["\'])%s(?P=quote)(?P[^\w\n"\'])(?P ?)', # "...", + r'(?P[^\w\n"\'])(?P ?)(?P["\'])%s(?P=quote)(?:$|\n)', # ,"..." + r'(?:^|\n)(?P["\'])%s(?P=quote)(?:$|\n)'): # "..." (no delim, no space) + regexp = re.compile(restr % body, re.DOTALL | re.MULTILINE) matches = regexp.findall(data) if matches: break diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 88d5af7c130d5db..e4c18ffe0a3a11e 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -1096,6 +1096,13 @@ def test_doublequote(self): dialect = sniffer.sniff(self.sample9) self.assertTrue(dialect.doublequote) + def test_sniff_quoted_single_column(self): + # gh-98820: this sample used to take minutes. + sniffer = csv.Sniffer() + sample = '"abcdefghijklmnopqrstuvwxyz"\n' * 10000 + with self.assertRaisesRegexp(csv.Error, "Could not determine delimiter"): + sniffer.sniff(sample, delimiters=',:|\t') + class NUL: def write(s, *args): pass From 0f0b14a1a076171f7b20d5e0e24034c0efca11b8 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Wed, 28 Oct 2026 00:53:07 -0500 Subject: [PATCH 03/23] gh-87451: Apply CVE-2021-4189 PASV fix to ftplib.ftpcp() (CVE-2026-8328) Backport of python/cpython gh-87451 (3.15 5dadc64673). 2.7.18.x already carries the CVE-2021-4189 fix in FTP.makepasv(): it ignores the IPv4 address a server advertises in its PASV reply and uses the control connection's peer address unless trust_server_pasv_ipv4_address is set. ftpcp() still called parse227() directly and passed the server-supplied host to target.sendport(), so a malicious source server could point the target's data connection at an arbitrary host:port (SSRF). ftpcp() now applies the same rule. 2.7.18.x: no unittest.mock in 2.7, so TestFtpcpSecurity uses small fake FTP/socket classes; % formatting instead of f-strings; the class is added to test_main(). Co-Authored-By: Claude Opus 5.5 --- Lib/ftplib.py | 11 +++++++- Lib/test/test_ftplib.py | 59 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/Lib/ftplib.py b/Lib/ftplib.py index 7c772e6ee5ee16d..cd9e87cb6267b88 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -939,7 +939,16 @@ def ftpcp(source, sourcename, target, targetname = '', type = 'I'): type = 'TYPE ' + type source.voidcmd(type) target.voidcmd(type) - sourcehost, sourceport = parse227(source.sendcmd('PASV')) + # Don't trust the IPv4 address the source server advertises in its PASV + # reply: a malicious source could otherwise point the target's data + # connection at an arbitrary host (SSRF). A caller that needs the old + # behavior can set trust_server_pasv_ipv4_address on the source FTP + # object. See FTP.makepasv(), which applies the same rule. + untrusted_host, sourceport = parse227(source.sendcmd('PASV')) + if source.trust_server_pasv_ipv4_address: + sourcehost = untrusted_host + else: + sourcehost = source.sock.getpeername()[0] target.sendport(sourcehost, sourceport) # RFC 959: the user must "listen" [...] BEFORE sending the # transfer request. diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index c7def634d6b77c7..dd42483c3f81247 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -980,10 +980,67 @@ def testTimeoutDirectAccess(self): ftp.close() +class TestFtpcpSecurity(TestCase): + """ftpcp() must not trust the host a source server advertises in PASV. + + A malicious source server can otherwise redirect the target server's + data connection to an arbitrary host:port (SSRF), so ftpcp() uses the + source server's actual peer address instead, the same as FTP.makepasv(). + """ + + class _FakeSock: + def __init__(self, peer): + self.peer = peer + + def getpeername(self): + return self.peer + + class _FakeFTP: + def __init__(self, replies, peer=None, trust=False): + self.replies = replies + self.sock = TestFtpcpSecurity._FakeSock(peer) + self.trust_server_pasv_ipv4_address = trust + self.sendport_calls = [] + + def voidcmd(self, cmd): + return '200 ok' + + def voidresp(self): + return '226 ok' + + def sendcmd(self, cmd): + return self.replies.get(cmd.split()[0], '150 ok') + + def sendport(self, host, port): + self.sendport_calls.append((host, port)) + + def _make_pair(self, advertised_host, real_host, trust=False): + # PASV replies give the host as comma-separated octets, not dotted. + advertised = advertised_host.replace('.', ',') + source = self._FakeFTP( + {'PASV': '227 Entering Passive Mode (%s,1,2).' % advertised}, + peer=(real_host, 21), trust=trust) + target = self._FakeFTP({}) + return source, target + + def test_ftpcp_ignores_untrusted_pasv_host(self): + source, target = self._make_pair(advertised_host='10.0.0.5', + real_host='198.51.100.7') + ftplib.ftpcp(source, 'a', target, 'b') + self.assertEqual(target.sendport_calls, [('198.51.100.7', 258)]) + + def test_ftpcp_trust_server_pasv_ipv4_address(self): + source, target = self._make_pair(advertised_host='10.0.0.5', + real_host='198.51.100.7', trust=True) + ftplib.ftpcp(source, 'a', target, 'b') + self.assertEqual(target.sendport_calls, [('10.0.0.5', 258)]) + + def test_main(): tests = [TestFTPClass, TestTimeouts, TestIPv6Environment, - TestTLS_FTPClassMixin, TestTLS_FTPClass] + TestTLS_FTPClassMixin, TestTLS_FTPClass, + TestFtpcpSecurity] thread_info = test_support.threading_setup() try: From 2f517c33de7a907ec83d5201c649be2e24e81e36 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Wed, 28 Oct 2026 00:54:08 -0500 Subject: [PATCH 04/23] gh-150751: validate httplib Content-Length and chunk-size (CVE-2026-103995) Backport of python/cpython GH-150752 and GH-159156 (gh-150751; the 3.10.22.1 ports are 833a27a452d and dbefda311d5). HTTPResponse parsed Content-Length with int() and chunk sizes with int(_, 16). Both accept input the RFC 9112 grammar forbids: a sign ("+5", "-5"), surrounding whitespace, and for base 16 an "0x" prefix. A peer that disagrees with httplib about message framing can use this for response splitting / desynchronisation. Values are now matched against Content-Length = 1*DIGIT and chunk-size = 1*HEXDIG first; an invalid Content-Length is ignored (the body is read to close) and an invalid chunk size closes the connection with IncompleteRead. Optional whitespace (SP/HTAB) around the Content-Length value is still allowed, per RFC 9112 section 5.1. 2.7.18.x: re has no fullmatch, so the patterns are anchored with \Z and used with match(). 2.7's int() never accepted underscores, but the grammar check covers them anyway. Tests use plain loops instead of subTest and also cover "0x5" as a Content-Length. Co-Authored-By: Claude Opus 5.5 --- Lib/httplib.py | 22 ++++++++++++++-------- Lib/test/test_httplib.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/Lib/httplib.py b/Lib/httplib.py index 9c8c5eedccb04a3..6271c76f741dcaf 100644 --- a/Lib/httplib.py +++ b/Lib/httplib.py @@ -261,6 +261,11 @@ # to prevent http header injection. _contains_disallowed_method_pchar_re = re.compile('[\x00-\x1f]') +# RFC 9112: Content-Length = 1*DIGIT and chunk-size = 1*HEXDIG. +# int() is more permissive, so we match against the grammar before calling it. +_is_legal_content_length = re.compile(r'[0-9]+\Z').match +_is_legal_chunk_size = re.compile(r'[0-9a-fA-F]+\Z').match + # We always set the Content-Length header for these methods because some # servers will otherwise respond with a 411 _METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'} @@ -503,14 +508,12 @@ def begin(self): # do we have a Content-Length? # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked" length = self.msg.getheader('content-length') - if length and not self.chunked: - try: - self.length = int(length) - except ValueError: - self.length = None - else: - if self.length < 0: # ignore nonsensical negative lengths - self.length = None + if length is not None: + # RFC 9112, section 5.1: optional whitespace (SP / HTAB) around + # the field value is not part of the value. + length = length.strip(' \t') + if length and not self.chunked and _is_legal_content_length(length): + self.length = int(length) else: self.length = None @@ -632,7 +635,10 @@ def _read_chunked(self, amt): i = line.find(';') if i >= 0: line = line[:i] # strip chunk-extensions + line = line.rstrip() try: + if not _is_legal_chunk_size(line): + raise ValueError("invalid chunk size") chunk_left = int(line, 16) except ValueError: # close the connection as protocol synchronisation is diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index e7ee76b0226d096..dfd0e79075eed07 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -622,6 +622,46 @@ def test_negative_content_length(self): self.assertEqual(resp.read(), 'Hello\r\n') self.assertTrue(resp.isclosed()) + def test_malformed_content_length(self): + # RFC 9112: Content-Length = 1*DIGIT. Values that int() accepts but + # the grammar forbids must not be used to frame the body. + for value in ('+5', '5_0', '5 0', '0x5'): + sock = FakeSocket( + 'HTTP/1.1 200 OK\r\nContent-Length: %s\r\n\r\nHello\r\n' % value) + resp = httplib.HTTPResponse(sock, method="GET") + resp.begin() + self.assertIsNone(resp.length, value) + self.assertEqual(resp.read(), 'Hello\r\n') + resp.close() + + def test_content_length_with_whitespace(self): + # RFC 9112, section 5.1: optional whitespace (SP / HTAB) around + # the field value is not part of the value, so it must not + # prevent the Content-Length from being used to frame the body. + for value in ('7 ', '7\t', '7 \t ', ' 7 '): + sock = FakeSocket( + 'HTTP/1.1 200 OK\r\nContent-Length: %s\r\n\r\n' + 'Hello\r\nextra' % value) + resp = httplib.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.length, 7, repr(value)) + self.assertEqual(resp.read(), 'Hello\r\n') + self.assertTrue(resp.isclosed()) + resp.close() + + def test_malformed_chunk_size(self): + # RFC 9112: chunk-size = 1*HEXDIG. Reject sizes that int(_, 16) accepts + # but the grammar forbids (a sign, an "0x" prefix or leading + # whitespace). + start = 'HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n' + for size in ('-5', '+5', '0x5', ' 5'): + sock = FakeSocket(start + '%s\r\nHELLO\r\n0\r\n\r\n' % size) + resp = httplib.HTTPResponse(sock, method="GET") + resp.begin() + self.assertRaises(httplib.IncompleteRead, resp.read) + self.assertTrue(resp.isclosed(), size) + resp.close() + def test_incomplete_read(self): sock = FakeSocket('HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nHello\r\n') resp = httplib.HTTPResponse(sock, method="GET") From 27175c6df8af4c7045a704710784d96dba9f9963 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Wed, 28 Oct 2026 00:54:55 -0500 Subject: [PATCH 05/23] bpo-44022: limit the headers skipped after a 100 Continue (CVE-2021-3737) Backport of python/cpython bpo-44022 (GH-25933, 078b146f062) to 2.7.18.x. HTTPResponse.begin() skips the header block of each 100 Continue response. That loop limited the length of each line (_MAXLINE) but not the number of lines, so a malicious server could keep a client reading header lines forever (an infinite loop / DoS). The skipped lines are now counted and HTTPException("got more than 100 headers") is raised past _MAXHEADERS, the same limit already applied to a final response's headers. 2.7.18.x: upstream refactored the loop into _read_headers(); 2.7 parses headers with HTTPMessage.readheaders(), so the counter is added to the existing skip loop instead. The upstream test would pass on unpatched 2.7 (EOF after the 'r' lines ends in BadStatusLine, itself an HTTPException), so the 2.7 test appends a valid 200 response and checks the message; a second test checks a normal 100 Continue is still skipped. Co-Authored-By: Claude Opus 5.5 --- Lib/httplib.py | 4 ++++ Lib/test/test_httplib.py | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/Lib/httplib.py b/Lib/httplib.py index 9c8c5eedccb04a3..bdc6cb22fcae371 100644 --- a/Lib/httplib.py +++ b/Lib/httplib.py @@ -453,6 +453,7 @@ def begin(self): if status != CONTINUE: break # skip the header from the 100 response + skipped = 0 while True: skip = self.fp.readline(_MAXLINE + 1) if len(skip) > _MAXLINE: @@ -460,6 +461,9 @@ def begin(self): skip = skip.strip() if not skip: break + skipped += 1 + if skipped > _MAXHEADERS: + raise HTTPException("got more than %d headers" % _MAXHEADERS) if self.debuglevel > 0: print "header:", skip diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index e7ee76b0226d096..8f39725acaaabcd 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -677,6 +677,30 @@ def test_overflowing_header_line(self): resp = httplib.HTTPResponse(FakeSocket(body)) self.assertRaises(httplib.LineTooLong, resp.begin) + def test_overflowing_header_limit_after_100(self): + # CVE-2021-3737: the headers of a 100 Continue response used to be + # skipped without any limit on their number. + body = ( + 'HTTP/1.1 100 OK\r\n' + + 'r\n' * 32768 + + '\r\nHTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n' + ) + resp = httplib.HTTPResponse(FakeSocket(body)) + with self.assertRaisesRegexp(httplib.HTTPException, + "got more than 100 headers"): + resp.begin() + + def test_headers_after_100(self): + # A 100 Continue response with a few headers is still skipped. + body = ( + 'HTTP/1.1 100 Continue\r\nX-A: 1\r\nX-B: 2\r\n\r\n' + 'HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nHello' + ) + resp = httplib.HTTPResponse(FakeSocket(body)) + resp.begin() + self.assertEqual(resp.status, 200) + self.assertEqual(resp.read(), 'Hello') + def test_overflowing_chunked_line(self): body = ( 'HTTP/1.1 200 OK\r\n' From 3b8050a369b835823baebfbcc078ba129a3dd4c0 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Wed, 28 Oct 2026 00:55:39 -0500 Subject: [PATCH 06/23] gh-87389: Fix an open redirect vulnerability in SimpleHTTPServer (CVE-2021-28861) Backport of python/cpython gh-87389 (GH-93879, 4dc2cae3abd) to 2.7.18.x. A request path starting with "//" that resolves to a directory made SimpleHTTPRequestHandler.send_head() answer with a 301 whose Location header also started with "//". Browsers treat "//host/path" as a scheme-relative URL, so a crafted link to the server redirected the user to an attacker-chosen host. BaseHTTPRequestHandler.parse_request() now reduces a leading run of slashes in the path to a single "/". Verified on 2.7.18.15: the "//python.org/..." request answers "Location: //python.org/..."; with this change it is "/python.org/...". 2.7.18.x: the fix lands in BaseHTTPServer.parse_request(); the test uses string concatenation and the literal 301 status instead of f-strings and HTTPStatus. Co-Authored-By: Claude Opus 5.5 --- Lib/BaseHTTPServer.py | 7 ++++++ Lib/test/test_httpservers.py | 49 ++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/Lib/BaseHTTPServer.py b/Lib/BaseHTTPServer.py index 3df3323a97eae69..b580de0681f0de7 100644 --- a/Lib/BaseHTTPServer.py +++ b/Lib/BaseHTTPServer.py @@ -287,6 +287,13 @@ def parse_request(self): return False self.command, self.path, self.request_version = command, path, version + # gh-87389: The purpose of replacing '//' with '/' is to protect + # against open redirect attacks possibly triggered if the path starts + # with '//' because http clients treat //path as an absolute URI + # without scheme (similar to http://path) rather than a path. + if self.path.startswith('//'): + self.path = '/' + self.path.lstrip('/') # Reduce to a single / + # Examine the headers and look for a Connection directive self.headers = self.MessageClass(self.rfile, 0) diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 25236c6ef41ad2e..9aa1b1ac43f3d7d 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -351,6 +351,55 @@ def check_status_and_reason(self, response, status, data=None): if data: self.assertEqual(data, body) + def test_get_dir_redirect_location_domain_injection_bug(self): + """Ensure //evil.co/..%2f../../X does not put //evil.co/ in Location. + + //netloc/ in a Location header is a redirect to a new host. + https://github.com/python/cpython/issues/87389 + + This checks that a path resolving to a directory on our server cannot + resolve into a redirect to another server. + """ + os.mkdir(os.path.join(self.tempdir, 'existing_directory')) + url = ('/python.org/..%2f..%2f..%2f..%2f..%2f../%0a%0d/../' + + self.tempdir_name + '/existing_directory') + expected_location = url + '/' # single slash prefix, trailing slash + # Canonicalizes to /tmp/tempdir_name/existing_directory which does + # exist and is a dir, triggering the 301 redirect logic. + response = self.request(url) + self.check_status_and_reason(response, 301) + location = response.getheader('Location') + self.assertEqual(location, expected_location, msg='non-attack failed!') + + # //python.org... multi-slash prefix, no trailing slash + attack_url = '/' + url + response = self.request(attack_url) + self.check_status_and_reason(response, 301) + location = response.getheader('Location') + self.assertFalse(location.startswith('//'), msg=location) + self.assertEqual(location, expected_location, + msg='Expected Location header to start with a single / and ' + 'end with a / as this is a directory redirect.') + + # ///python.org... triple-slash prefix, no trailing slash + attack3_url = '//' + url + response = self.request(attack3_url) + self.check_status_and_reason(response, 301) + self.assertEqual(response.getheader('Location'), expected_location) + + # If the second word in the http request (Request-URI for the http + # method) is a full URI, we don't worry about it, as that'll be parsed + # and reassembled as a full URI within BaseHTTPRequestHandler.send_head + # so no errant scheme-less //netloc//evil.co/ domain mixup can happen. + attack_scheme_netloc_2slash_url = 'https://pypi.org/' + url + response = self.request(attack_scheme_netloc_2slash_url) + self.check_status_and_reason(response, 301) + location = response.getheader('Location') + # We're just ensuring that the scheme and domain make it through, if + # there are or aren't multiple slashes at the start of the path that + # follows that isn't important in this Location: header. + self.assertTrue(location.startswith('https://pypi.org/'), msg=location) + def test_get(self): #constructs the path relative to the root directory of the HTTPServer response = self.request(self.base_url + '/test') From 5bd9f306f25a676781dc5d6071657d32ba02f99e Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Wed, 28 Oct 2026 00:56:42 -0500 Subject: [PATCH 07/23] gh-159428: Don't read mimetypes.knownfiles by default on Windows (CVE-2024-3220) Backport of python/cpython gh-159428 (GH-159445; the 3.10.22.1 port is f3c18820409) to 2.7.18.x. mimetypes.init() reads the registry on Windows and then every file in knownfiles. Those are Unix paths ("/etc/mime.types", ...), which on Windows resolve to the root of the current drive (C:\etc\mime.types). Unprivileged users can usually create such files, and so could register arbitrary MIME types for every Python process on the machine, e.g. to change what SimpleHTTPServer serves a file as. knownfiles is now empty on Windows; only the registry is used by default. Callers can still pass files to init() or add to knownfiles themselves. 2.7.18.x: test_mimetypes empties mimetypes.knownfiles at import for isolation, so the test keeps a reference to the default list taken before that line; the non-Windows check lives in a new KnownfilesTestCase registered in test_main(). Co-Authored-By: Claude Opus 5.5 --- Lib/mimetypes.py | 28 +++++++++++++++++----------- Lib/test/test_mimetypes.py | 13 +++++++++++++ 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index afc053f135ddfba..92b0a414c2e5fef 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -37,17 +37,23 @@ "add_type","read_mime_types","init" ] -knownfiles = [ - "/etc/mime.types", - "/etc/httpd/mime.types", # Mac OS X - "/etc/httpd/conf/mime.types", # Apache - "/etc/apache/mime.types", # Apache 1 - "/etc/apache2/mime.types", # Apache 2 - "/usr/local/etc/httpd/conf/mime.types", - "/usr/local/lib/netscape/mime.types", - "/usr/local/etc/httpd/conf/mime.types", # Apache 1.2 - "/usr/local/etc/mime.types", # Apache 1.3 - ] +if sys.platform == "win32": + # The default paths would resolve to the root of the current drive on Windows + # (e.g. C:\etc\mime.types), which unprivileged users can usually create. + # Only the registry is used by default. + knownfiles = [] +else: + knownfiles = [ + "/etc/mime.types", + "/etc/httpd/mime.types", # Mac OS X + "/etc/httpd/conf/mime.types", # Apache + "/etc/apache/mime.types", # Apache 1 + "/etc/apache2/mime.types", # Apache 2 + "/usr/local/etc/httpd/conf/mime.types", + "/usr/local/lib/netscape/mime.types", + "/usr/local/etc/httpd/conf/mime.types", # Apache 1.2 + "/usr/local/etc/mime.types", # Apache 1.3 + ] inited = False _db = None diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index e9a7216d1c6c214..2c8b7fd2137dee6 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -8,6 +8,8 @@ from test import test_support # Tell it we don't know about external files: +# The default list, before it is emptied below for test isolation. +knownfiles = mimetypes.knownfiles mimetypes.knownfiles = [] mimetypes.inited = False mimetypes._default_mime_types() @@ -67,6 +69,12 @@ def test_guess_all_types(self): eq(all, []) +class KnownfilesTestCase(unittest.TestCase): + @unittest.skipIf(sys.platform == "win32", "non-Windows only") + def test_default_knownfiles(self): + self.assertIn("/etc/mime.types", knownfiles) + + @unittest.skipUnless(sys.platform.startswith("win"), "Windows only") class Win32MimeTypesTestCase(unittest.TestCase): def setUp(self): @@ -90,6 +98,10 @@ def test_registry_parsing(self): eq(db.guess_type("image.jpg"), ("image/jpeg", None)) eq(db.guess_type("image.png"), ("image/png", None)) + def test_default_knownfiles(self): + # CVE-2024-3220: the Unix paths would resolve to e.g. C:\etc\mime.types + self.assertEqual(knownfiles, []) + def test_non_latin_extension(self): import _winreg @@ -163,6 +175,7 @@ def __getattr__(self, name): def test_main(): test_support.run_unittest(MimeTypesTestCase, + KnownfilesTestCase, Win32MimeTypesTestCase ) From 2109f04b7d1ac298a56c64c61a666b9bc2afa25a Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Wed, 28 Oct 2026 00:58:09 -0500 Subject: [PATCH 08/23] gh-145986: Avoid unbounded C recursion in pyexpat conv_content_model (CVE-2026-4224) Backport of python/cpython gh-145986 (eb0e8be3a7) to 2.7.18.x. When a parser has an ElementDeclHandler, pyexpat converts each element declaration's content model to nested tuples with the recursive conv_content_model(). An inline DTD with a deeply nested content model (e.g. 500000 levels of "(a, ") overflowed the C stack: on 2.7.18.15 the reproducer segfaults. conv_content_model() now goes through Py_EnterRecursiveCall()/Py_LeaveRecursiveCall(), so deep nesting raises RuntimeError("maximum recursion depth exceeded in conv_content_model") instead. As upstream, models nested deeper than the recursion limit (1000 by default) now raise rather than being converted. 2.7.18.x: the public Py_EnterRecursiveCall() replaces the internal _Py_EnterRecursiveCall(); C89 declarations; 2.7 raises RuntimeError, not RecursionError. The tests go into a new ElementDeclHandlerTest class registered in test_main(), with a second test that a normal nested model still converts. The crash is in pyexpat itself; it reproduces and is fixed with the bundled expat 2.6.4. Co-Authored-By: Claude Opus 5.5 --- Lib/test/test_pyexpat.py | 30 +++++++++++++++++++++++++++++- Modules/pyexpat.c | 11 +++++++++-- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index 72840bd31280893..f4877ffeb334a64 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -666,6 +666,33 @@ def resolve_entity(context, base, system_id, public_id): self.assertEqual(handler_call_args, [("bar", "baz")]) +class ElementDeclHandlerTest(unittest.TestCase): + def test_deeply_nested_content_model(self): + # This should raise a RuntimeError (recursion limit) and not crash. + # See https://github.com/python/cpython/issues/145986. + N = 500000 + data = ( + '\n]>\n\n' + ) + + parser = expat.ParserCreate() + parser.ElementDeclHandler = lambda _1, _2: None + with self.assertRaises(RuntimeError): + parser.Parse(data, True) + + def test_nested_content_model(self): + # A modestly nested content model is still converted. + models = [] + parser = expat.ParserCreate() + parser.ElementDeclHandler = lambda name, model: models.append(model) + parser.Parse('\n' + '\n]>\n\n', True) + self.assertEqual(len(models), 2) + self.assertEqual(len(models[0][3]), 2) + + def test_main(): run_unittest(SetAttributeTest, ParseTest, @@ -677,7 +704,8 @@ def test_main(): sf1296433Test, ChardataBufferTest, MalformedInputText, - ForeignDTDTests) + ForeignDTDTests, + ElementDeclHandlerTest) if __name__ == "__main__": test_main() diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index a1a870d418e8b3b..4bea10e252a4858 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -692,9 +692,14 @@ conv_content_model(XML_Content * const model, PyObject *(*conv_string)(const XML_Char *)) { PyObject *result = NULL; - PyObject *children = PyTuple_New(model->numchildren); + PyObject *children; int i; + if (Py_EnterRecursiveCall(" in conv_content_model")) { + return NULL; + } + + children = PyTuple_New(model->numchildren); if (children != NULL) { assert(model->numchildren < INT_MAX); for (i = 0; i < (int)model->numchildren; ++i) { @@ -702,7 +707,7 @@ conv_content_model(XML_Content * const model, conv_string); if (child == NULL) { Py_XDECREF(children); - return NULL; + goto done; } PyTuple_SET_ITEM(children, i, child); } @@ -710,6 +715,8 @@ conv_content_model(XML_Content * const model, model->type, model->quant, conv_string,model->name, children); } +done: + Py_LeaveRecursiveCall(); return result; } From ec2f6f5fd7cf80aa744621c995c7d8e6b237e99f Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Wed, 28 Oct 2026 00:59:29 -0500 Subject: [PATCH 09/23] gh-150599: Prevent bz2 decompressor reuse after errors (CVE-2026-9669) Backport of python/cpython gh-150599 (3.14 157a5df8cb) to 2.7.18.x. After BZ2_bzDecompress() reported an error, BZ2Decompressor.decompress() raised but left the object usable. An application that caught the error and called decompress() again resumed libbz2 from an invalid internal state; with crafted input that can write out of bounds of a stack buffer. On 2.7.18.15 the upstream reproducer's second call goes back into libbz2 and returns ''. The decompressor now records the error and raises ValueError("Decompressor is unusable after a previous error") on any later call. 2.7.18.x: 2.7 has its own BZ2Decompressor implementation (no needs_input or eof attributes), so the guard is a new bzerror field checked next to the existing "end of stream was already found" check. 2.7 also has a second path upstream's 3.x code doesn't: when growing the output buffer fails, it calls BZ2_bzDecompressEnd() but left the object usable; that path now marks the decompressor unusable too. The test expects IOError ("invalid data stream"), as 2.7 raises, and uses binascii.unhexlify. Co-Authored-By: Claude Opus 5.5 --- Lib/test/test_bz2.py | 15 +++++++++++++++ Modules/bz2module.c | 12 ++++++++++++ 2 files changed, 27 insertions(+) diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index bf5eb2055bea36d..1077da4d721da3a 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -2,6 +2,7 @@ from test.test_support import TESTFN, _4G, bigmemtest, import_module, findfile import unittest +import binascii from cStringIO import StringIO import os import subprocess @@ -417,6 +418,20 @@ def testEOFError(self): self.assertRaises(EOFError, bz2d.decompress, "anything") self.assertRaises(EOFError, bz2d.decompress, "") + def test_decompress_after_data_error(self): + # gh-150599: re-entering BZ2_bzDecompress() after an error could + # write out of bounds; the decompressor is now unusable instead. + data = binascii.unhexlify( + "425a6839314159265359000000000000007fffff000000000000000000000000" + "00000000000000000000000000000000000000e0370000000000000000000000" + "000000000000000000000000000000000000000000000000000083f3" + ) + bzd = BZ2Decompressor() + with self.assertRaisesRegexp(IOError, "invalid data stream"): + bzd.decompress(data) + with self.assertRaisesRegexp(ValueError, "previous error"): + bzd.decompress(b'\x00' * 18) + @bigmemtest(_4G, memuse=1.25) def testBigmem(self, size): # Issue #14398: decompression fails when output data is >=2GB. diff --git a/Modules/bz2module.c b/Modules/bz2module.c index 81f168827b100a1..c68d84783ad14d8 100644 --- a/Modules/bz2module.c +++ b/Modules/bz2module.c @@ -120,6 +120,7 @@ typedef struct { PyObject_HEAD bz_stream bzs; int running; + int bzerror; PyObject *unused_data; #ifdef WITH_THREAD PyThread_type_lock lock; @@ -1942,6 +1943,13 @@ BZ2Decomp_decompress(BZ2DecompObject *self, PyObject *args) "already found"); goto error; } + if (self->bzerror) { + /* Re-entering BZ2_bzDecompress() after an error can write out of + bounds. */ + PyErr_SetString(PyExc_ValueError, + "Decompressor is unusable after a previous error"); + goto error; + } ret = PyString_FromStringAndSize(NULL, SMALLCHUNK); if (!ret) @@ -1975,6 +1983,7 @@ BZ2Decomp_decompress(BZ2DecompObject *self, PyObject *args) break; } if (bzerror != BZ_OK) { + self->bzerror = bzerror; Util_CatchBZ2Error(bzerror); goto error; } @@ -1989,6 +1998,8 @@ BZ2Decomp_decompress(BZ2DecompObject *self, PyObject *args) if (buffer_left == 0) { if (Util_GrowBuffer(&ret) < 0) { BZ2_bzDecompressEnd(bzs); + /* The stream state is freed: don't reuse it. */ + self->bzerror = BZ_MEM_ERROR; goto error; } bzs->next_out = BUF(ret) + output_size; @@ -2050,6 +2061,7 @@ BZ2Decomp_init(BZ2DecompObject *self, PyObject *args, PyObject *kwargs) } self->running = 1; + self->bzerror = 0; return 0; From 97b9b4b9091126e22445c8eb567b8b90ccb05760 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Wed, 28 Oct 2026 01:00:24 -0500 Subject: [PATCH 10/23] gh-152674: Avoid quadratic behavior in ElementPath index predicates (CVE-2026-6879) Backport of python/cpython gh-152674 (2ffab08378) to 2.7.18.x. For an index predicate ([1], [last()], [last()-N]) ElementPath called parent.findall(elem.tag) once per candidate element, so findall() and a fully consumed iterfind() over n same-tag siblings did O(n^2) work. A crafted document plus a query like ".//item[last()]" could burn CPU. select() now caches the selected sibling per (parent, tag) pair. 2.7.18.x: cElementTree's find()/findall() also go through this Python ElementPath module, so the fix covers both implementations. The test counts findall() calls through an Element subclass; cElementTree's Element is a factory function in 2.7 and cannot be subclassed, so the test is skipped there. super(Class, self) and %-formatting replace the 3.x spellings. Without the fix, findall() calls grow as 1025/2049/4097 for 1024/2048/4096 siblings; with it they are constant. Co-Authored-By: Claude Opus 5.5 --- Lib/test/test_xml_etree.py | 34 ++++++++++++++++++++++++++++++++++ Lib/xml/etree/ElementPath.py | 15 +++++++++++---- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index fed93f1d907f0bf..119f87c99576199 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1906,6 +1906,40 @@ def test_find_xpath(self): self.assertEqual(e.find('./tag[last()-1]').attrib['class'], 'c') self.assertEqual(e.find('./tag[last()-2]').attrib['class'], 'b') + def test_find_xpath_index_no_quadratic_complexity(self): + if not isinstance(ET.Element, type): + self.skipTest("Element is not subclassable in this implementation") + + class CountingElement(ET.Element): + findall_calls = 0 + def findall(self, *args, **kwargs): + type(self).findall_calls += 1 + return super(CountingElement, self).findall(*args, **kwargs) + + def work(n, pattern): + root = CountingElement("root") + for _ in range(n): + ET.SubElement(root, "a") + CountingElement.findall_calls = 0 + root.findall(pattern) + return CountingElement.findall_calls + + for pattern in [".//a[1]", ".//a[last()]"]: + w1 = work(1024, pattern) + w2 = work(2048, pattern) + w3 = work(4096, pattern) + + self.assertGreater(w1, 0) + r1 = float(w2) / w1 + r2 = float(w3) / w2 + # Doubling N must not ~double the parent.findall calls. + # Linear-in-N call counts indicate the cache is missing. + self.assertLess( + max(r1, r2), 1.5, + msg="Possible quadratic behavior on %r: calls=%r ratios=%r" + % (pattern, (w1, w2, w3), (r1, r2)), + ) + def test_findall(self): e = ET.XML(SAMPLE_XML) e[2] = ET.XML(SAMPLE_SECTION) diff --git a/Lib/xml/etree/ElementPath.py b/Lib/xml/etree/ElementPath.py index 4a626d799c827d8..efc05f3748bb7bd 100644 --- a/Lib/xml/etree/ElementPath.py +++ b/Lib/xml/etree/ElementPath.py @@ -211,15 +211,22 @@ def select(context, result): index = -1 def select(context, result): parent_map = get_parent_map(context) + cache = {} for elem in result: try: parent = parent_map[elem] + except KeyError: + continue + key = (parent, elem.tag) + if key not in cache: # FIXME: what if the selector is "*" ? elems = list(parent.findall(elem.tag)) - if elems[index] is elem: - yield elem - except (IndexError, KeyError): - pass + try: + cache[key] = elems[index] + except IndexError: + cache[key] = None + if cache[key] is elem: + yield elem return select raise SyntaxError("invalid predicate") From bb28a49bf4bb9457f89f3a8f5f0bc56875031d9d Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Wed, 28 Oct 2026 01:01:16 -0500 Subject: [PATCH 11/23] gh-155694: Scope HTTPPasswordMgr credentials by URL scheme (CVE-2026-15806) Backport of python/cpython gh-155694 (3.14 a0d023fbd2) to 2.7.18.x. HTTPPasswordMgr (and HTTPPasswordMgrWithDefaultRealm) reduced URIs to (authority, path) and dropped the scheme, so credentials added for https://example.com/ were also returned for http://example.com/. An attacker who can redirect or downgrade a client to plain HTTP could then capture Basic/Digest credentials meant only for TLS. Stored and requested URIs now keep their scheme, and a stored URI with a scheme only matches requests with the same scheme. A stored bare authority ("example.com") or network-path reference ("//example.com/") still matches any scheme, as before; reduce_uri() and is_suburi() keep their public behavior. 2.7.18.x: 2.7 has no HTTPPasswordMgrWithPriorAuth, so only add_password()/find_user_password() change. (scheme,) + tuple replaces the 3.x star-unpacking. 2.7's password-manager tests are doctests, so the new tests go in a PasswordManagerSchemeTests class registered in test_main(), with an extra HTTPPasswordMgrWithDefaultRealm case. Co-Authored-By: Claude Opus 5.5 --- Lib/test/test_urllib2.py | 57 ++++++++++++++++++++++++++++++++++++++++ Lib/urllib2.py | 21 ++++++++++++--- 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 20a0f581436d640..a289bf60dc86d29 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1265,6 +1265,62 @@ def _test_basic_auth(self, opener, auth_handler, auth_header, self.assertEqual(len(http_handler.requests), 1) self.assertFalse(http_handler.requests[0].has_header(auth_header)) +class PasswordManagerSchemeTests(unittest.TestCase): + + def test_password_manager_scheme(self): + mgr = urllib2.HTTPPasswordMgr() + mgr.add_password( + "realm", "https://example.com/", "user", "password") + + self.assertEqual( + mgr.find_user_password("realm", "https://example.com/"), + ("user", "password")) + self.assertEqual( + mgr.find_user_password("realm", "http://example.com/"), + (None, None)) + # Support an authority without a scheme. + self.assertEqual( + mgr.find_user_password("realm", "example.com"), + ("user", "password")) + # An authority without a scheme continues to match any scheme. + mgr.add_password( + "realm", "schemeless.example.com", "user", "password") + for scheme in "http", "https": + self.assertEqual( + mgr.find_user_password( + "realm", "%s://schemeless.example.com/" % scheme), + ("user", "password")) + + # A network-path reference also has no scheme. + mgr.add_password( + "realm", "//network-path.example.com/", "user", "password") + self.assertEqual( + mgr.find_user_password( + "realm", "https://network-path.example.com/"), + ("user", "password")) + + def test_password_manager_default_realm_scheme(self): + mgr = urllib2.HTTPPasswordMgrWithDefaultRealm() + mgr.add_password(None, "https://example.com/", "user", "password") + self.assertEqual( + mgr.find_user_password("realm", "https://example.com/x"), + ("user", "password")) + self.assertEqual( + mgr.find_user_password("realm", "http://example.com/x"), + (None, None)) + + def test_password_manager_reduced_uri(self): + mgr = urllib2.HTTPPasswordMgr() + + self.assertEqual( + mgr.reduce_uri("http://example.com/path"), + ("example.com:80", "/path")) + self.assertTrue( + mgr.is_suburi( + ("example.com", "/path"), + ("example.com", "/path/subpath"))) + + class MiscTests(unittest.TestCase, FakeHTTPMixin): def test_build_opener(self): @@ -1485,6 +1541,7 @@ def test_main(verbose=None): OpenerDirectorTests, HandlerTests, MiscTests, + PasswordManagerSchemeTests, RequestTests) test_support.run_unittest(*tests) diff --git a/Lib/urllib2.py b/Lib/urllib2.py index 71e05bfd24eea72..9548f959004952c 100644 --- a/Lib/urllib2.py +++ b/Lib/urllib2.py @@ -788,16 +788,17 @@ def add_password(self, realm, uri, user, passwd): self.passwd[realm] = {} for default_port in True, False: reduced_uri = tuple( - [self.reduce_uri(u, default_port) for u in uri]) + [self._reduce_uri_with_scheme(u, default_port) for u in uri]) self.passwd[realm][reduced_uri] = (user, passwd) def find_user_password(self, realm, authuri): domains = self.passwd.get(realm, {}) for default_port in True, False: - reduced_authuri = self.reduce_uri(authuri, default_port) + reduced_authuri = self._reduce_uri_with_scheme( + authuri, default_port) for uris, authinfo in domains.iteritems(): for uri in uris: - if self.is_suburi(uri, reduced_authuri): + if self._is_suburi_with_scheme(uri, reduced_authuri): return authinfo return None, None @@ -824,6 +825,20 @@ def reduce_uri(self, uri, default_port=True): authority = "%s:%d" % (host, dport) return authority, path + def _reduce_uri_with_scheme(self, uri, default_port=True): + # Like reduce_uri(), but keep the scheme (None for a bare authority + # or a network-path reference) so that credentials added for one + # scheme are not sent over another, e.g. https -> http. + parts = urlparse.urlsplit(uri) + scheme = parts[0] if parts[1] else None + return (scheme or None,) + self.reduce_uri(uri, default_port) + + def _is_suburi_with_scheme(self, base, test): + if (base[0] is not None and test[0] is not None and + base[0] != test[0]): + return False + return self.is_suburi(base[1:], test[1:]) + def is_suburi(self, base, test): """Check if test is below base in a URI tree From a5889ecb81fb5636e3b7f8c67ab8762991d809d5 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Wed, 28 Oct 2026 01:03:14 -0500 Subject: [PATCH 12/23] gh-153030: Fix quadratic complexity in incremental HTMLParser feeding (CVE-2026-15308) Backport of python/cpython gh-153030 (3.14 07efb08123) to 2.7.18.x. HTMLParser.feed() concatenated each chunk onto rawdata and rescanned the whole unparsed buffer. While a construct stays unterminated (a comment, processing instruction, doctype, CDATA section, start tag or RAWTEXT element such as ") # RAWTEXT element + + def test_incremental_events_preserved(self): + # Data fed while a construct is unterminated is still reported in + # full, and parsing resumes on each feed() once it makes progress. + source = "

a

c" + expected = [("starttag", "p", []), ("data", "a"), ("endtag", "p"), + ("comment", " " + "x" * 1000 + " "), + ("starttag", "b", []), ("data", "c"), ("endtag", "b")] + parser = EventCollector() + for ch in source: + parser.feed(ch) + parser.close() + self.assertEqual(parser.get_events(), expected) + # Small feeds after an unterminated construct may be held back until + # the buffer doubles, but close() always delivers them. + parser = EventCollector() + parser.feed("") + parser.feed("") + parser.close() + self.assertEqual(parser.get_events(), + [("comment", " " + "y" * 100 + " "), + ("starttag", "i", [])]) + + class AttributesTestCase(TestCaseBase): def test_attr_syntax(self): @@ -649,7 +695,8 @@ def test_broken_condcoms(self): def test_main(): - test_support.run_unittest(HTMLParserTestCase, AttributesTestCase) + test_support.run_unittest(HTMLParserTestCase, AttributesTestCase, + IncrementalFeedTestCase) if __name__ == "__main__": From 987278299cef7963cfc147e9960ad4de10eb7aa0 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Wed, 28 Oct 2026 10:25:42 -0500 Subject: [PATCH 13/23] gh-158876: Update bundled Expat to 2.9.0 Update the vendored Expat in Modules/expat from 2.6.4 to 2.9.0. The files are taken from 3.10.22.x (95dadd7c4a5, the 3.10 backport of gh-158876), which fixes these Expat CVEs: CVE-2024-8176 CVE-2025-59375 CVE-2026-24515 CVE-2026-25210 CVE-2026-32776 CVE-2026-32777 CVE-2026-32778 CVE-2026-41080 CVE-2026-45186 CVE-2026-50219 CVE-2026-56131 CVE-2026-56132 CVE-2026-56403 CVE-2026-56404 CVE-2026-56405 CVE-2026-56406 CVE-2026-56407 CVE-2026-56408 CVE-2026-56409 CVE-2026-56410 CVE-2026-56411 CVE-2026-56412 CVE-2026-66046 CVE-2026-72522 CVE-2026-76641 CVE-2026-76956 CVE-2026-76957 CVE-2026-77214 CVE-2026-93990 CVE-2026-102633 Expat 2.9.0 adds xcs.c (and the new headers fallthrough.h, hash_table.h, memory_sanitizer.h and xcs.h). xcs.c is added to setup.py's expat_sources, to Modules/Setup.dist and to PCbuild/_elementtree.vcxproj, which on Windows compiles the vendored sources. The stale macconfig.h entry in that project is replaced, as upstream did. 2.7.18.x keeps the two local changes it already had on top of 2.6.4: - expat_external.h does not include pyexpatns.h. On Windows, pyexpat.vcxproj compiles pyexpat.c against these headers but links the ActiveState libexpat.lib, so the symbols must keep their XML_* names. - xmlparse.c's PREFIX type is renamed to PPREFIX, because PC/pyconfig.h defines a PREFIX macro. The Windows pyexpat therefore builds against 2.9.0 headers. The external expat it links must be 2.9.0 too. pyexpat.c is unchanged. Expat 2.9.0 deprecates the XML_Size position getters, XML_GetInputContext and XML_SetHashSalt, so gcc now warns about them. They still work, and switching pyexpat.c to the *64 replacements would stop it from linking against an Expat older than 2.9.0. test_pyexpat gains test_deeply_nested_entities, which uses a chain of 200000 nested general entities (CVE-2024-8176). It segfaults with the bundled 2.6.4 and passes with 2.9.0. Co-Authored-By: Claude Opus 5.5 --- Lib/test/test_pyexpat.py | 12 + Modules/Setup.dist | 2 +- Modules/expat/COPYING | 2 +- Modules/expat/ascii.h | 2 + Modules/expat/asciitab.h | 2 + Modules/expat/expat.h | 173 +- Modules/expat/expat_config.h | 10 +- Modules/expat/expat_external.h | 144 +- Modules/expat/fallthrough.h | 51 + Modules/expat/hash_table.h | 78 + Modules/expat/iasciitab.h | 2 + Modules/expat/internal.h | 162 +- Modules/expat/latin1tab.h | 2 + Modules/expat/memory_sanitizer.h | 53 + Modules/expat/nametab.h | 2 + Modules/expat/pyexpatns.h | 21 +- Modules/expat/siphash.h | 17 +- Modules/expat/utf8tab.h | 2 + Modules/expat/winconfig.h | 2 + Modules/expat/xcs.c | 90 + Modules/expat/xcs.h | 44 + Modules/expat/xmlparse.c | 3595 +++++++++++++++++--------- Modules/expat/xmlrole.c | 117 +- Modules/expat/xmlrole.h | 42 +- Modules/expat/xmltok.c | 424 ++- Modules/expat/xmltok.h | 288 +-- Modules/expat/xmltok_impl.c | 88 +- Modules/expat/xmltok_impl.h | 2 + Modules/expat/xmltok_ns.c | 13 +- PCbuild/_elementtree.vcxproj | 6 +- PCbuild/_elementtree.vcxproj.filters | 14 +- setup.py | 4 +- 32 files changed, 3489 insertions(+), 1977 deletions(-) create mode 100644 Modules/expat/fallthrough.h create mode 100644 Modules/expat/hash_table.h create mode 100644 Modules/expat/memory_sanitizer.h create mode 100644 Modules/expat/xcs.c create mode 100644 Modules/expat/xcs.h diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index 72840bd31280893..0fe1c6575d4c9de 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -255,6 +255,18 @@ def test_parse_again(self): self.assertEqual(expat.ErrorString(cm.exception.code), expat.errors.XML_ERROR_FINISHED) + def test_deeply_nested_entities(self): + # CVE-2024-8176: Expat < 2.7.0 expanded nested general entities + # recursively and overflowed the C stack on a long enough chain. + n = 200000 + decl = ''.join('' % (i, i + 1) for i in range(n)) + doc = ']>&e0;' % (decl, n) + out = [] + parser = expat.ParserCreate() + parser.CharacterDataHandler = out.append + parser.Parse(doc, True) + self.assertEqual(''.join(out), 'x') + class NamespaceSeparatorTest(unittest.TestCase): def test_legal(self): # Tests that make sure we get errors when the namespace_separator value diff --git a/Modules/Setup.dist b/Modules/Setup.dist index bbc9222129641bb..6fed5557b5906ba 100644 --- a/Modules/Setup.dist +++ b/Modules/Setup.dist @@ -469,7 +469,7 @@ GLHACK=-Dclear=__GLclear # Interface to the Expat XML parser # More information on Expat can be found at www.libexpat.org. # -#pyexpat expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DXML_POOR_ENTROPY=1 -DUSE_PYEXPAT_CAPI +#pyexpat expat/xcs.c expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DXML_POOR_ENTROPY=1 -DUSE_PYEXPAT_CAPI # Hye-Shik Chang's CJKCodecs diff --git a/Modules/expat/COPYING b/Modules/expat/COPYING index ce9e5939291e45f..c6d184a8aae845c 100644 --- a/Modules/expat/COPYING +++ b/Modules/expat/COPYING @@ -1,5 +1,5 @@ Copyright (c) 1998-2000 Thai Open Source Software Center Ltd and Clark Cooper -Copyright (c) 2001-2022 Expat maintainers +Copyright (c) 2001-2025 Expat maintainers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/Modules/expat/ascii.h b/Modules/expat/ascii.h index 1f594d2e54b4d29..1d9cf70bf695da1 100644 --- a/Modules/expat/ascii.h +++ b/Modules/expat/ascii.h @@ -31,6 +31,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #define ASCII_A 0x41 diff --git a/Modules/expat/asciitab.h b/Modules/expat/asciitab.h index af766fb24785ea3..43af0876ccd3f8a 100644 --- a/Modules/expat/asciitab.h +++ b/Modules/expat/asciitab.h @@ -30,6 +30,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ /* 0x00 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML, diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 523b37d8d5787d8..1b2852dd76cb487 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -11,7 +11,7 @@ Copyright (c) 2000-2005 Fred L. Drake, Jr. Copyright (c) 2001-2002 Greg Stein Copyright (c) 2002-2016 Karl Waclawek - Copyright (c) 2016-2024 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2016 Cristian Rodríguez Copyright (c) 2016 Thomas Beutlich Copyright (c) 2017 Rhodri James @@ -19,6 +19,8 @@ Copyright (c) 2023 Hanno Böck Copyright (c) 2023 Sony Corporation / Snild Dolkow Copyright (c) 2024 Taichi Haradaguchi <20001722@ymail.ne.jp> + Copyright (c) 2025 Matthew Fernandez + Copyright (c) 2026 Braian Plaku Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -39,24 +41,27 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifndef Expat_INCLUDED -#define Expat_INCLUDED 1 +# define Expat_INCLUDED 1 -#include -#include "expat_external.h" +# include // for int64_t, uint8_t, uint64_t +# include +# include "expat_external.h" -#ifdef __cplusplus +# ifdef __cplusplus extern "C" { -#endif +# endif struct XML_ParserStruct; typedef struct XML_ParserStruct *XML_Parser; typedef unsigned char XML_Bool; -#define XML_TRUE ((XML_Bool)1) -#define XML_FALSE ((XML_Bool)0) +# define XML_TRUE ((XML_Bool)1) +# define XML_FALSE ((XML_Bool)0) /* The XML_Status enum gives the possible return values for several API functions. The preprocessor #defines are included so this @@ -73,11 +78,11 @@ typedef unsigned char XML_Bool; */ enum XML_Status { XML_STATUS_ERROR = 0, -#define XML_STATUS_ERROR XML_STATUS_ERROR +# define XML_STATUS_ERROR XML_STATUS_ERROR XML_STATUS_OK = 1, -#define XML_STATUS_OK XML_STATUS_OK +# define XML_STATUS_OK XML_STATUS_OK XML_STATUS_SUSPENDED = 2 -#define XML_STATUS_SUSPENDED XML_STATUS_SUSPENDED +# define XML_STATUS_SUSPENDED XML_STATUS_SUSPENDED }; enum XML_Error { @@ -135,6 +140,26 @@ enum XML_Error { XML_ERROR_NOT_STARTED, }; +/* Added in 2.9.0. */ +enum XML_Prop_Error { + XML_PROP_ERROR_NONE = 0, // i.e. success + + XML_PROP_ERROR_INVALID_KEY = 1, // i.e. the property is not known at all + XML_PROP_ERROR_INVALID_TYPE = 2, // i.e. the property is known but expects a + // different type (so a different + // getter/setter would need to be called) + XML_PROP_ERROR_INVALID_VALUE = 3, // i.e. the value is invalid (with a setter + // function) or a NULL-pointer (with a + // getter function) + + XML_PROP_ERROR_PARSER_NULL = 4, // i.e. the parser argument is NULL + XML_PROP_ERROR_PARSER_NOT_ROOT = 5, // i.e. the parser is not a root parser + // but a subparser (and a root parser is + // needed) + + /* potentially more error codes upcoming here */ +}; + enum XML_Content_Type { XML_CTYPE_EMPTY = 1, XML_CTYPE_ANY, @@ -276,7 +301,7 @@ XML_ParserCreate_MM(const XML_Char *encoding, /* Prepare a parser object to be reused. This is particularly valuable when memory allocation overhead is disproportionately high, - such as when a large number of small documnents need to be parsed. + such as when a large number of small documents need to be parsed. All handlers are cleared from the parser, except for the unknownEncodingHandler. The parser's external state is re-initialized except for the values of ns and ns_triplets. @@ -680,7 +705,7 @@ XMLPARSEAPI(void) XML_SetUserData(XML_Parser parser, void *userData); /* Returns the last value set by XML_SetUserData or NULL. */ -#define XML_GetUserData(parser) (*(void **)(parser)) +# define XML_GetUserData(parser) (*(void **)(parser)) /* This is equivalent to supplying an encoding argument to XML_ParserCreate. On success XML_SetEncoding returns non-zero, @@ -752,29 +777,6 @@ XML_GetSpecifiedAttributeCount(XML_Parser parser); XMLPARSEAPI(int) XML_GetIdAttributeIndex(XML_Parser parser); -#ifdef XML_ATTR_INFO -/* Source file byte offsets for the start and end of attribute names and values. - The value indices are exclusive of surrounding quotes; thus in a UTF-8 source - file an attribute value of "blah" will yield: - info->valueEnd - info->valueStart = 4 bytes. -*/ -typedef struct { - XML_Index nameStart; /* Offset to beginning of the attribute name. */ - XML_Index nameEnd; /* Offset after the attribute name's last byte. */ - XML_Index valueStart; /* Offset to beginning of the attribute value. */ - XML_Index valueEnd; /* Offset after the attribute value's last byte. */ -} XML_AttrInfo; - -/* Returns an array of XML_AttrInfo structures for the attribute/value pairs - passed in last call to the XML_StartElementHandler that were specified - in the start-tag rather than defaulted. Each attribute/value pair counts - as 1; thus the number of entries in the array is - XML_GetSpecifiedAttributeCount(parser) / 2. -*/ -XMLPARSEAPI(const XML_AttrInfo *) -XML_GetAttributeInfo(XML_Parser parser); -#endif - /* Parses some input. Returns XML_STATUS_ERROR if a fatal error is detected. The last call to XML_Parse must have isFinal true; len may be zero for this call (or any other). @@ -916,10 +918,23 @@ XML_SetParamEntityParsing(XML_Parser parser, function behavior. This must be called before parsing is started. Returns 1 if successful, 0 when called after parsing has started. Note: If parser == NULL, the function will do nothing and return 0. + DEPRECATED since Expat 2.8.0. + Please use XML_SetHashSalt16Bytes instead. */ +XML_ATTR_DEPRECATED("please use XML_SetHashSalt16Bytes instead") XMLPARSEAPI(int) XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt); +/* Sets the hash salt to use for internal hash calculations. + Helps in preventing DoS attacks based on predicting hash function behavior. + This must be called before parsing is started. + Returns XML_TRUE if successful, XML_FALSE when called after parsing has + started or when parser is NULL. + Added in Expat 2.8.0. +*/ +XMLPARSEAPI(XML_Bool) +XML_SetHashSalt16Bytes(XML_Parser parser, const uint8_t entropy[16]); + /* If XML_Parse or XML_ParseBuffer have returned XML_STATUS_ERROR, then XML_GetErrorCode returns information about the error. */ @@ -946,15 +961,24 @@ XML_GetErrorCode(XML_Parser parser); return 0 to indicate an error. Note: XML_GetCurrentByteIndex returns -1 to indicate an error. */ +XML_ATTR_DEPRECATED("please use XML_GetCurrentLineNumber64 instead") XMLPARSEAPI(XML_Size) XML_GetCurrentLineNumber(XML_Parser parser); +XMLPARSEAPI(uint64_t) XML_GetCurrentLineNumber64(XML_Parser parser); +XML_ATTR_DEPRECATED("please use XML_GetCurrentColumnNumber64 instead") XMLPARSEAPI(XML_Size) XML_GetCurrentColumnNumber(XML_Parser parser); +XMLPARSEAPI(uint64_t) XML_GetCurrentColumnNumber64(XML_Parser parser); +XML_ATTR_DEPRECATED("please use XML_GetCurrentByteIndex64 instead") XMLPARSEAPI(XML_Index) XML_GetCurrentByteIndex(XML_Parser parser); +XMLPARSEAPI(int64_t) XML_GetCurrentByteIndex64(XML_Parser parser); /* Return the number of bytes in the current event. Returns 0 if the event is in an internal entity. */ +XML_ATTR_DEPRECATED("please use XML_GetCurrentByteCount64 instead") XMLPARSEAPI(int) XML_GetCurrentByteCount(XML_Parser parser); +XMLPARSEAPI(uint64_t) +XML_GetCurrentByteCount64(XML_Parser parser); /* If XML_CONTEXT_BYTES is >=1, returns the input buffer, sets the integer pointed to by offset to the offset within this buffer @@ -966,13 +990,16 @@ XML_GetCurrentByteCount(XML_Parser parser); NOTE: The character pointer returned should not be used outside the handler that makes the call. */ +XML_ATTR_DEPRECATED("please use XML_GetInputContext64 instead") XMLPARSEAPI(const char *) XML_GetInputContext(XML_Parser parser, int *offset, int *size); +XMLPARSEAPI(const char *) +XML_GetInputContext64(XML_Parser parser, int64_t *offset, uint64_t *size); /* For backwards compatibility with previous versions. */ -#define XML_GetErrorLineNumber XML_GetCurrentLineNumber -#define XML_GetErrorColumnNumber XML_GetCurrentColumnNumber -#define XML_GetErrorByteIndex XML_GetCurrentByteIndex +# define XML_GetErrorLineNumber XML_GetCurrentLineNumber +# define XML_GetErrorColumnNumber XML_GetCurrentColumnNumber +# define XML_GetErrorByteIndex XML_GetCurrentByteIndex /* Frees the content model passed to the element declaration handler */ XMLPARSEAPI(void) @@ -1022,17 +1049,24 @@ enum XML_FeatureEnum { XML_FEATURE_UNICODE_WCHAR_T, XML_FEATURE_DTD, XML_FEATURE_CONTEXT_BYTES, + /* Added in Expat 1.95.5, removed in Expat 2.9.0. */ XML_FEATURE_MIN_SIZE, XML_FEATURE_SIZEOF_XML_CHAR, XML_FEATURE_SIZEOF_XML_LCHAR, + /* Added in Expat 2.0.0. */ XML_FEATURE_NS, + /* Added in Expat 2.0.1. */ XML_FEATURE_LARGE_SIZE, + /* Added in Expat 2.1.0, removed in Expat 2.9.0. */ XML_FEATURE_ATTR_INFO, /* Added in Expat 2.4.0. */ XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT, XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT, /* Added in Expat 2.6.0. */ - XML_FEATURE_GE + XML_FEATURE_GE, + /* Added in Expat 2.7.2. */ + XML_FEATURE_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT, + XML_FEATURE_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT, /* Additional features must be added to the end of this enum. */ }; @@ -1045,7 +1079,42 @@ typedef struct { XMLPARSEAPI(const XML_Feature *) XML_GetFeatureList(void); -#if defined(XML_DTD) || (defined(XML_GE) && XML_GE == 1) +/* Added in Expat 2.9.0. */ +enum XML_Parser_Property { +# if defined(XML_TESTING) + XML_PROP_INVALID = 0, +# endif +# if XML_GE == 1 + XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD = 1, // of type `uint64_t` + XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION = 2, // of type `double` + XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD = 3, // of type `uint64_t` + XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION = 4, // of type `double` +# endif + XML_PROP_REPARSE_DEFERRAL_ENABLED = 5, // of type `XML_Bool` + /* more properties upcoming here */ +}; + +/* Added in Expat 2.9.0. */ +XMLPARSEAPI(enum XML_Prop_Error) +XML_GetPropertyBool(XML_Parser parser, enum XML_Parser_Property property, + XML_Bool *value); +XMLPARSEAPI(enum XML_Prop_Error) +XML_GetPropertyDouble(XML_Parser parser, enum XML_Parser_Property property, + double *value); +XMLPARSEAPI(enum XML_Prop_Error) +XML_GetPropertyUInt64(XML_Parser parser, enum XML_Parser_Property property, + uint64_t *value); +XMLPARSEAPI(enum XML_Prop_Error) +XML_SetPropertyBool(XML_Parser parser, enum XML_Parser_Property property, + XML_Bool value); +XMLPARSEAPI(enum XML_Prop_Error) +XML_SetPropertyDouble(XML_Parser parser, enum XML_Parser_Property property, + double value); +XMLPARSEAPI(enum XML_Prop_Error) +XML_SetPropertyUInt64(XML_Parser parser, enum XML_Parser_Property property, + uint64_t value); + +# if defined(XML_DTD) || (defined(XML_GE) && XML_GE == 1) /* Added in Expat 2.4.0 for XML_DTD defined and * added in Expat 2.6.0 for XML_GE == 1. */ XMLPARSEAPI(XML_Bool) @@ -1057,7 +1126,17 @@ XML_SetBillionLaughsAttackProtectionMaximumAmplification( XMLPARSEAPI(XML_Bool) XML_SetBillionLaughsAttackProtectionActivationThreshold( XML_Parser parser, unsigned long long activationThresholdBytes); -#endif + +/* Added in Expat 2.7.2. */ +XMLPARSEAPI(XML_Bool) +XML_SetAllocTrackerMaximumAmplification(XML_Parser parser, + float maximumAmplificationFactor); + +/* Added in Expat 2.7.2. */ +XMLPARSEAPI(XML_Bool) +XML_SetAllocTrackerActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes); +# endif /* Added in Expat 2.6.0. */ XMLPARSEAPI(XML_Bool) @@ -1066,12 +1145,12 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); /* Expat follows the semantic versioning convention. See https://semver.org */ -#define XML_MAJOR_VERSION 2 -#define XML_MINOR_VERSION 6 -#define XML_MICRO_VERSION 4 +# define XML_MAJOR_VERSION 2 +# define XML_MINOR_VERSION 9 +# define XML_MICRO_VERSION 0 -#ifdef __cplusplus +# ifdef __cplusplus } -#endif +# endif #endif /* not Expat_INCLUDED */ diff --git a/Modules/expat/expat_config.h b/Modules/expat/expat_config.h index c3967f03cec52ef..f8448f5bb43f57f 100644 --- a/Modules/expat/expat_config.h +++ b/Modules/expat/expat_config.h @@ -3,7 +3,7 @@ * distribution. */ #ifndef EXPAT_CONFIG_H -#define EXPAT_CONFIG_H +#define EXPAT_CONFIG_H 1 #include #ifdef WORDS_BIGENDIAN @@ -12,9 +12,17 @@ #define BYTEORDER 1234 #endif +#define HAVE_MEMMOVE 1 + #define XML_NS 1 #define XML_DTD 1 #define XML_GE 1 #define XML_CONTEXT_BYTES 1024 +#undef HAVE_ARC4RANDOM +#undef HAVE_ARC4RANDOM_BUF +#undef HAVE_GETENTROPY +#undef HAVE_GETRANDOM +#undef HAVE_SYSCALL_GETRANDOM + #endif /* EXPAT_CONFIG_H */ diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h index 8829f77091047a4..0620fe47ed3cd26 100644 --- a/Modules/expat/expat_external.h +++ b/Modules/expat/expat_external.h @@ -12,9 +12,11 @@ Copyright (c) 2001-2002 Greg Stein Copyright (c) 2002-2006 Karl Waclawek Copyright (c) 2016 Cristian Rodríguez - Copyright (c) 2016-2019 Sebastian Pipping + Copyright (c) 2016-2025 Sebastian Pipping Copyright (c) 2017 Rhodri James Copyright (c) 2018 Yury Gribov + Copyright (c) 2026 Matthew Fernandez + Copyright (c) 2026 Braian Plaku Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -35,17 +37,19 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifndef Expat_External_INCLUDED -#define Expat_External_INCLUDED 1 +# define Expat_External_INCLUDED 1 /* External API definitions */ /* Expat tries very hard to make the API boundary very specifically defined. There are two macros defined to control this boundary; each of these can be defined before including this header to - achieve some different behavior, but doing so it not recommended or + achieve some different behavior, but doing so is not recommended or tested frequently. XMLCALL - The calling convention to use for all calls across the @@ -64,12 +68,12 @@ compiled with the cdecl calling convention as the default since system headers may assume the cdecl convention. */ -#ifndef XMLCALL -# if defined(_MSC_VER) -# define XMLCALL __cdecl -# elif defined(__GNUC__) && defined(__i386) && ! defined(__INTEL_COMPILER) -# define XMLCALL __attribute__((cdecl)) -# else +# ifndef XMLCALL +# if defined(_MSC_VER) +# define XMLCALL __cdecl +# elif defined(__GNUC__) && defined(__i386) && ! defined(__INTEL_COMPILER) +# define XMLCALL __attribute__((cdecl)) +# else /* For any platform which uses this definition and supports more than one calling convention, we need to extend this definition to declare the convention used on that platform, if it's possible to @@ -80,86 +84,100 @@ pre-processor and how to specify the same calling convention as the platform's malloc() implementation. */ -# define XMLCALL -# endif -#endif /* not defined XMLCALL */ +# define XMLCALL +# endif +# endif /* not defined XMLCALL */ -#if ! defined(XML_STATIC) && ! defined(XMLIMPORT) -# ifndef XML_BUILDING_EXPAT +# if ! defined(XML_STATIC) && ! defined(XMLIMPORT) +# ifndef XML_BUILDING_EXPAT /* using Expat from an application */ -# if defined(_MSC_EXTENSIONS) && ! defined(__BEOS__) && ! defined(__CYGWIN__) -# define XMLIMPORT __declspec(dllimport) +# if defined(_MSC_VER) && ! defined(__BEOS__) && ! defined(__CYGWIN__) +# define XMLIMPORT __declspec(dllimport) +# endif + # endif +# endif /* not defined XML_STATIC */ +# ifndef XML_ENABLE_VISIBILITY +# define XML_ENABLE_VISIBILITY 0 # endif -#endif /* not defined XML_STATIC */ - -#ifndef XML_ENABLE_VISIBILITY -# define XML_ENABLE_VISIBILITY 0 -#endif -#if ! defined(XMLIMPORT) && XML_ENABLE_VISIBILITY -# define XMLIMPORT __attribute__((visibility("default"))) -#endif +# if ! defined(XMLIMPORT) && XML_ENABLE_VISIBILITY +# define XMLIMPORT __attribute__((visibility("default"))) +# endif /* If we didn't define it above, define it away: */ -#ifndef XMLIMPORT -# define XMLIMPORT -#endif - -#if defined(__GNUC__) \ - && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96)) -# define XML_ATTR_MALLOC __attribute__((__malloc__)) -#else -# define XML_ATTR_MALLOC -#endif - -#if defined(__GNUC__) \ - && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) -# define XML_ATTR_ALLOC_SIZE(x) __attribute__((__alloc_size__(x))) -#else -# define XML_ATTR_ALLOC_SIZE(x) -#endif - -#define XMLPARSEAPI(type) XMLIMPORT type XMLCALL - -#ifdef __cplusplus -extern "C" { -#endif +# ifndef XMLIMPORT +# define XMLIMPORT +# endif -#ifdef XML_UNICODE_WCHAR_T -# ifndef XML_UNICODE -# define XML_UNICODE +# if defined(__GNUC__) \ + && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96)) +# define XML_ATTR_MALLOC __attribute__((__malloc__)) +# else +# define XML_ATTR_MALLOC # endif -# if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2) -# error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc" + +# if defined(__GNUC__) \ + && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) +# define XML_ATTR_ALLOC_SIZE(x) __attribute__((__alloc_size__(x))) +# else +# define XML_ATTR_ALLOC_SIZE(x) +# endif + +/* Marks a function that Expat still provides but that callers should move off + of. */ +# if defined(__clang__) || defined(__GNUC__) +# define XML_ATTR_DEPRECATED(message) \ + __attribute__((__deprecated__(message))) +# elif defined(_MSC_VER) +# define XML_ATTR_DEPRECATED(message) __declspec(deprecated(message)) +# else +# define XML_ATTR_DEPRECATED(message) // empty i.e. no deprecation +# endif + +# define XMLPARSEAPI(type) XMLIMPORT type XMLCALL + +# ifdef __cplusplus +extern "C" { # endif -#endif -#ifdef XML_UNICODE /* Information is UTF-16 encoded. */ # ifdef XML_UNICODE_WCHAR_T +# ifndef XML_UNICODE +# define XML_UNICODE +# endif +# if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2) +# error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc" +# endif +# endif + +# ifdef XML_UNICODE /* Information is UTF-16 encoded. */ +# ifdef XML_UNICODE_WCHAR_T typedef wchar_t XML_Char; typedef wchar_t XML_LChar; -# else +# else typedef unsigned short XML_Char; typedef char XML_LChar; -# endif /* XML_UNICODE_WCHAR_T */ -#else /* Information is UTF-8 encoded. */ +# endif /* XML_UNICODE_WCHAR_T */ +# else /* Information is UTF-8 encoded. */ typedef char XML_Char; typedef char XML_LChar; -#endif /* XML_UNICODE */ +# endif /* XML_UNICODE */ -#ifdef XML_LARGE_SIZE /* Use large integers for file/stream positions. */ +# ifdef XML_LARGE_SIZE /* Use large integers for file/stream positions. */ +# if defined(__clang__) || defined(__GNUC__) +# warning Macro XML_LARGE_SIZE is deprecated, please use the 64bit location API functions instead. +# endif typedef long long XML_Index; typedef unsigned long long XML_Size; -#else +# else typedef long XML_Index; typedef unsigned long XML_Size; -#endif /* XML_LARGE_SIZE */ +# endif /* XML_LARGE_SIZE */ -#ifdef __cplusplus +# ifdef __cplusplus } -#endif +# endif #endif /* not Expat_External_INCLUDED */ diff --git a/Modules/expat/fallthrough.h b/Modules/expat/fallthrough.h new file mode 100644 index 000000000000000..0152d1bdc5de8ff --- /dev/null +++ b/Modules/expat/fallthrough.h @@ -0,0 +1,51 @@ +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2026 Nick Begg + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT +*/ + +#ifndef FALLTHROUGH_H +# define FALLTHROUGH_H 1 + +// Explicit fallthrough in switch case to avoid warnings +// with compiler flag -Wimplicit-fallthrough. + +# define EXPAT_FALLTHROUGH \ + do { \ + } while (0) + +# if defined(__has_attribute) +# if __has_attribute(fallthrough) +# undef EXPAT_FALLTHROUGH +# define EXPAT_FALLTHROUGH __attribute__((fallthrough)) +# endif +# endif + +#endif // FALLTHROUGH_H diff --git a/Modules/expat/hash_table.h b/Modules/expat/hash_table.h new file mode 100644 index 000000000000000..f290aad18693feb --- /dev/null +++ b/Modules/expat/hash_table.h @@ -0,0 +1,78 @@ +/* Hash table related internal API + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2026 Sebastian Pipping + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT +*/ + +#if ! defined(HASH_TABLE_H) +# define HASH_TABLE_H 1 + +# include "expat.h" // for XML_Bool, XML_Parser +# include "internal.h" // for XML_NONTESTING_STATIC + +# include // for size_t + +typedef const XML_Char *KEY; + +typedef struct { + KEY name; +} NAMED; + +typedef struct { + NAMED **v; + unsigned char power; + size_t size; + size_t used; + XML_Parser parser; +} HASH_TABLE; + +typedef struct { + NAMED **p; + NAMED **end; +} HASH_TABLE_ITER; + +XML_NONTESTING_STATIC NAMED *lookupWithLength(XML_Parser parser, + HASH_TABLE *table, KEY name, + size_t nameLen, + size_t createSize); +XML_NONTESTING_STATIC NAMED *lookup(XML_Parser parser, HASH_TABLE *table, + KEY name, size_t createSize); + +XML_NONTESTING_STATIC void hashTableInit(HASH_TABLE *table, XML_Parser parser); +XML_NONTESTING_STATIC void hashTableClear(HASH_TABLE *table); +XML_NONTESTING_STATIC void hashTableDestroy(HASH_TABLE *table); +XML_NONTESTING_STATIC void hashTableIterInit(HASH_TABLE_ITER *iter, + const HASH_TABLE *table); +XML_NONTESTING_STATIC NAMED *hashTableIterNext(HASH_TABLE_ITER *iter); + +XML_NONTESTING_STATIC XML_Bool keyeq(KEY s1, size_t s1len, KEY s2); +XML_NONTESTING_STATIC size_t keylen(KEY s); + +#endif // ! defined(HASH_TABLE_H) diff --git a/Modules/expat/iasciitab.h b/Modules/expat/iasciitab.h index 5d8646f2a318b8a..1de8d519c8b7bad 100644 --- a/Modules/expat/iasciitab.h +++ b/Modules/expat/iasciitab.h @@ -30,6 +30,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ /* Like asciitab.h, except that 0xD has code BT_S rather than BT_CR */ diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h index 167ec36804a43b0..de34cd7dfc55c58 100644 --- a/Modules/expat/internal.h +++ b/Modules/expat/internal.h @@ -6,13 +6,6 @@ The following calling convention macros are defined for frequently called functions: - FASTCALL - Used for those internal functions that have a simple - body and a low number of arguments and local variables. - - PTRCALL - Used for functions called though function pointers. - - PTRFASTCALL - Like PTRCALL, but for low number of arguments. - inline - Used for selected internal functions for which inlining may improve performance on some platforms. @@ -28,11 +21,14 @@ Copyright (c) 2002-2003 Fred L. Drake, Jr. Copyright (c) 2002-2006 Karl Waclawek Copyright (c) 2003 Greg Stein - Copyright (c) 2016-2024 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2018 Yury Gribov Copyright (c) 2019 David Loffredo Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow Copyright (c) 2024 Taichi Haradaguchi <20001722@ymail.ne.jp> + Copyright (c) 2026 Matthew Wozniczka + Copyright (c) 2026 Braian Plaku + Copyright (c) 2026 Florian Schmaus Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -53,124 +49,106 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ - -#if defined(__GNUC__) && defined(__i386__) && ! defined(__MINGW32__) -/* We'll use this version by default only where we know it helps. - - regparm() generates warnings on Solaris boxes. See SF bug #692878. - Instability reported with egcs on a RedHat Linux 7.3. - Let's comment out: - #define FASTCALL __attribute__((stdcall, regparm(3))) - and let's try this: + SPDX-License-Identifier: MIT */ -# define FASTCALL __attribute__((regparm(3))) -# define PTRFASTCALL __attribute__((regparm(3))) -#endif - -/* Using __fastcall seems to have an unexpected negative effect under - MS VC++, especially for function pointers, so we won't use it for - now on that platform. It may be reconsidered for a future release - if it can be made more effective. - Likely reason: __fastcall on Windows is like stdcall, therefore - the compiler cannot perform stack optimizations for call clusters. -*/ - -/* Make sure all of these are defined if they aren't already. */ - -#ifndef FASTCALL -# define FASTCALL -#endif -#ifndef PTRCALL -# define PTRCALL -#endif +#if ! defined(INTERNAL_H) +# define INTERNAL_H 1 -#ifndef PTRFASTCALL -# define PTRFASTCALL -#endif - -#ifndef XML_MIN_SIZE -# if ! defined(__cplusplus) && ! defined(inline) +# if ! defined(inline) # ifdef __GNUC__ # define inline __inline # endif /* __GNUC__ */ # endif -#endif /* XML_MIN_SIZE */ -#ifdef __cplusplus -# define inline inline -#else # ifndef inline # define inline # endif -#endif - -#include // ULONG_MAX -#if defined(_WIN32) \ - && (! defined(__USE_MINGW_ANSI_STDIO) \ - || (1 - __USE_MINGW_ANSI_STDIO - 1 == 0)) -# define EXPAT_FMT_ULL(midpart) "%" midpart "I64u" -# if defined(_WIN64) // Note: modifiers "td" and "zu" do not work for MinGW -# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "I64d" -# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "I64u" -# else -# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "d" -# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u" +# if ! defined(XML_NONTESTING_STATIC) +# if defined(XML_TESTING) +# define XML_NONTESTING_STATIC // empty i.e. not static +# else +# define XML_NONTESTING_STATIC static +# endif # endif -#else -# define EXPAT_FMT_ULL(midpart) "%" midpart "llu" -# if ! defined(ULONG_MAX) -# error Compiler did not define ULONG_MAX for us -# elif ULONG_MAX == 18446744073709551615u // 2^64-1 -# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "ld" -# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "lu" + +# include // ULONG_MAX +# include // size_t + +# if defined(_WIN32) \ + && (! defined(__USE_MINGW_ANSI_STDIO) \ + || (1 - __USE_MINGW_ANSI_STDIO - 1 == 0)) +# define EXPAT_FMT_LLX(midpart) "%" midpart "I64x" +# define EXPAT_FMT_ULL(midpart) "%" midpart "I64u" +# if defined(_WIN64) // Note: modifiers "td" and "zu" do not work for MinGW +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "I64d" +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "I64u" +# else +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "d" +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u" +# endif # else -# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "d" -# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u" +# include // PRIdPTR, PRIuPTR +# define EXPAT_FMT_LLX(midpart) "%" midpart "llx" +# define EXPAT_FMT_ULL(midpart) "%" midpart "llu" +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart PRIdPTR +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart PRIuPTR # endif -#endif -#ifndef UNUSED_P -# define UNUSED_P(p) (void)p -#endif +# ifndef UNUSED_P +# define UNUSED_P(p) (void)p +# endif /* NOTE BEGIN If you ever patch these defaults to greater values for non-attack XML payload in your environment, please file a bug report with libexpat. Thank you! */ -#define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT \ - 100.0f -#define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT \ - 8388608 // 8 MiB, 2^23 -/* NOTE END */ +# define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT \ + 100.0f +# define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT \ + 8388608 // 8 MiB, 2^23 + +# define EXPAT_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT 100.0f +# define EXPAT_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT \ + 67108864 // 64 MiB, 2^26 + +// NOTE: If function expat_alloc was user facing, EXPAT_MALLOC_ALIGNMENT would +// have to take sizeof(long double) into account +union expat_align { + long long l; + void *p; +}; +# define EXPAT_MALLOC_ALIGNMENT sizeof(union expat_align) +# define EXPAT_MALLOC_PADDING ((EXPAT_MALLOC_ALIGNMENT) - sizeof(size_t)) -#include "expat.h" // so we can use type XML_Parser below +/* NOTE END */ -#ifdef __cplusplus -extern "C" { -#endif +# include "expat.h" // so we can use type XML_Parser below void _INTERNAL_trim_to_complete_utf8_characters(const char *from, const char **fromLimRef); -#if defined(XML_GE) && XML_GE == 1 +# if defined(XML_GE) && XML_GE == 1 unsigned long long testingAccountingGetCountBytesDirect(XML_Parser parser); unsigned long long testingAccountingGetCountBytesIndirect(XML_Parser parser); const char *unsignedCharToPrintable(unsigned char c); -#endif +# endif extern -#if ! defined(XML_TESTING) +# if ! defined(XML_TESTING) const -#endif +# endif XML_Bool g_reparseDeferralEnabledDefault; // written ONLY in runtests.c -#if defined(XML_TESTING) +# if defined(XML_TESTING) + +int xmlSetHashSalt(XML_Parser parser, unsigned long hash_salt); + +void *expat_malloc(XML_Parser parser, size_t size, int sourceLine); +void expat_free(XML_Parser parser, void *ptr, int sourceLine); +void *expat_realloc(XML_Parser parser, void *ptr, size_t size, int sourceLine); extern unsigned int g_bytesScanned; // used for testing only -#endif +# endif -#ifdef __cplusplus -} -#endif +#endif // not defined INTERNAL_H diff --git a/Modules/expat/latin1tab.h b/Modules/expat/latin1tab.h index b681d278af6569b..3793f4f3cc7841d 100644 --- a/Modules/expat/latin1tab.h +++ b/Modules/expat/latin1tab.h @@ -30,6 +30,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ /* 0x80 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER, diff --git a/Modules/expat/memory_sanitizer.h b/Modules/expat/memory_sanitizer.h new file mode 100644 index 000000000000000..f739b88f304871b --- /dev/null +++ b/Modules/expat/memory_sanitizer.h @@ -0,0 +1,53 @@ +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2026 Matthew Fernandez + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT +*/ + +#if ! defined(MEMORY_SANITIZER_H) +# define MEMORY_SANITIZER_H 1 + +# if defined(__has_feature) +# if __has_feature(memory_sanitizer) +# include + +// inform Memory Sanitizer that [base, base + extent) is now initialized +# define MSAN_UNPOISON(base, extent) __msan_unpoison((base), (extent)) + +# endif +# endif + +# if ! defined(MSAN_UNPOISON) +# define MSAN_UNPOISON(base, extent) \ + do { \ + } while (0) +# endif + +#endif // ! defined(MEMORY_SANITIZER_H) diff --git a/Modules/expat/nametab.h b/Modules/expat/nametab.h index 63485446b967272..2385851562e7dcd 100644 --- a/Modules/expat/nametab.h +++ b/Modules/expat/nametab.h @@ -28,6 +28,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ static const unsigned namingBitmap[] = { diff --git a/Modules/expat/pyexpatns.h b/Modules/expat/pyexpatns.h index 999c5c730abd7a2..4c550b7d0131819 100644 --- a/Modules/expat/pyexpatns.h +++ b/Modules/expat/pyexpatns.h @@ -26,7 +26,7 @@ * http://lxr.mozilla.org/seamonkey/source/modules/libimg/png/mozpngconf.h#115 * * The list of relevant exported symbols can be had with this command: - * + * nm pyexpat.so \ | grep -v " [a-zBUA] " \ | grep -v "_fini\|_init\|initpyexpat" @@ -38,6 +38,9 @@ #ifndef PYEXPATNS_H #define PYEXPATNS_H +#define testingAccountingGetCountBytesDirect PyExpat_testingAccountingGetCountBytesDirect +#define testingAccountingGetCountBytesIndirect PyExpat_testingAccountingGetCountBytesIndirect +#define unsignedCharToPrintable PyExpat_unsignedCharToPrintable #define XML_DefaultCurrent PyExpat_XML_DefaultCurrent #define XML_ErrorString PyExpat_XML_ErrorString #define XML_ExpatVersion PyExpat_XML_ExpatVersion @@ -47,14 +50,22 @@ #define XML_GetBase PyExpat_XML_GetBase #define XML_GetBuffer PyExpat_XML_GetBuffer #define XML_GetCurrentByteCount PyExpat_XML_GetCurrentByteCount +#define XML_GetCurrentByteCount64 PyExpat_XML_GetCurrentByteCount64 #define XML_GetCurrentByteIndex PyExpat_XML_GetCurrentByteIndex +#define XML_GetCurrentByteIndex64 PyExpat_XML_GetCurrentByteIndex64 #define XML_GetCurrentColumnNumber PyExpat_XML_GetCurrentColumnNumber +#define XML_GetCurrentColumnNumber64 PyExpat_XML_GetCurrentColumnNumber64 #define XML_GetCurrentLineNumber PyExpat_XML_GetCurrentLineNumber +#define XML_GetCurrentLineNumber64 PyExpat_XML_GetCurrentLineNumber64 #define XML_GetErrorCode PyExpat_XML_GetErrorCode #define XML_GetFeatureList PyExpat_XML_GetFeatureList #define XML_GetIdAttributeIndex PyExpat_XML_GetIdAttributeIndex #define XML_GetInputContext PyExpat_XML_GetInputContext +#define XML_GetInputContext64 PyExpat_XML_GetInputContext64 #define XML_GetParsingStatus PyExpat_XML_GetParsingStatus +#define XML_GetPropertyBool PyExpat_XML_GetPropertyBool +#define XML_GetPropertyDouble PyExpat_XML_GetPropertyDouble +#define XML_GetPropertyUInt64 PyExpat_XML_GetPropertyUInt64 #define XML_GetSpecifiedAttributeCount PyExpat_XML_GetSpecifiedAttributeCount #define XmlGetUtf16InternalEncoding PyExpat_XmlGetUtf16InternalEncoding #define XmlGetUtf16InternalEncodingNS PyExpat_XmlGetUtf16InternalEncodingNS @@ -79,8 +90,12 @@ #define XmlPrologStateInit PyExpat_XmlPrologStateInit #define XmlPrologStateInitExternalEntity PyExpat_XmlPrologStateInitExternalEntity #define XML_ResumeParser PyExpat_XML_ResumeParser +#define XML_SetAllocTrackerActivationThreshold PyExpat_XML_SetAllocTrackerActivationThreshold +#define XML_SetAllocTrackerMaximumAmplification PyExpat_XML_SetAllocTrackerMaximumAmplification #define XML_SetAttlistDeclHandler PyExpat_XML_SetAttlistDeclHandler #define XML_SetBase PyExpat_XML_SetBase +#define XML_SetBillionLaughsAttackProtectionActivationThreshold PyExpat_XML_SetBillionLaughsAttackProtectionActivationThreshold +#define XML_SetBillionLaughsAttackProtectionMaximumAmplification PyExpat_XML_SetBillionLaughsAttackProtectionMaximumAmplification #define XML_SetCdataSectionHandler PyExpat_XML_SetCdataSectionHandler #define XML_SetCharacterDataHandler PyExpat_XML_SetCharacterDataHandler #define XML_SetCommentHandler PyExpat_XML_SetCommentHandler @@ -103,6 +118,10 @@ #define XML_SetNotStandaloneHandler PyExpat_XML_SetNotStandaloneHandler #define XML_SetParamEntityParsing PyExpat_XML_SetParamEntityParsing #define XML_SetProcessingInstructionHandler PyExpat_XML_SetProcessingInstructionHandler +#define XML_SetPropertyBool PyExpat_XML_SetPropertyBool +#define XML_SetPropertyDouble PyExpat_XML_SetPropertyDouble +#define XML_SetPropertyUInt64 PyExpat_XML_SetPropertyUInt64 +#define XML_SetReparseDeferralEnabled PyExpat_XML_SetReparseDeferralEnabled #define XML_SetReturnNSTriplet PyExpat_XML_SetReturnNSTriplet #define XML_SetSkippedEntityHandler PyExpat_XML_SetSkippedEntityHandler #define XML_SetStartCdataSectionHandler PyExpat_XML_SetStartCdataSectionHandler diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h index 04f6f74585b5a2f..ac1fbfaf0bde81d 100644 --- a/Modules/expat/siphash.h +++ b/Modules/expat/siphash.h @@ -8,6 +8,8 @@ * * 1. https://www.131002.net/siphash/siphash24.c * 2. https://www.131002.net/siphash/ + * + * SPDX-License-Identifier: CC0-1.0 * -------------------------------------------------------------------------- * HISTORY: * @@ -100,6 +102,7 @@ #include /* size_t */ #include /* uint64_t uint32_t uint8_t */ +#include "fallthrough.h" /* * Workaround to not require a C++11 compiler for using ULL suffix @@ -234,25 +237,25 @@ sip24_final(struct siphash *H) { switch (left) { case 7: b |= (uint64_t)H->buf[6] << 48; - /* fall through */ + EXPAT_FALLTHROUGH; case 6: b |= (uint64_t)H->buf[5] << 40; - /* fall through */ + EXPAT_FALLTHROUGH; case 5: b |= (uint64_t)H->buf[4] << 32; - /* fall through */ + EXPAT_FALLTHROUGH; case 4: b |= (uint64_t)H->buf[3] << 24; - /* fall through */ + EXPAT_FALLTHROUGH; case 3: b |= (uint64_t)H->buf[2] << 16; - /* fall through */ + EXPAT_FALLTHROUGH; case 2: b |= (uint64_t)H->buf[1] << 8; - /* fall through */ + EXPAT_FALLTHROUGH; case 1: b |= (uint64_t)H->buf[0] << 0; - /* fall through */ + EXPAT_FALLTHROUGH; case 0: break; } diff --git a/Modules/expat/utf8tab.h b/Modules/expat/utf8tab.h index 88efcf91cc16a6c..73732d1527f1420 100644 --- a/Modules/expat/utf8tab.h +++ b/Modules/expat/utf8tab.h @@ -30,6 +30,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ /* 0x80 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL, diff --git a/Modules/expat/winconfig.h b/Modules/expat/winconfig.h index 05805514ec7fa21..9e7e8bdc22ca36c 100644 --- a/Modules/expat/winconfig.h +++ b/Modules/expat/winconfig.h @@ -31,6 +31,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifndef WINCONFIG_H diff --git a/Modules/expat/xcs.c b/Modules/expat/xcs.c new file mode 100644 index 000000000000000..ae5e042d778fffa --- /dev/null +++ b/Modules/expat/xcs.c @@ -0,0 +1,90 @@ +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2022-2026 Sebastian Pipping + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT +*/ + +#include "xcs.h" + +#if defined(XML_UNICODE) +# if defined(XML_UNICODE_WCHAR_T) +# include // for wcscmp, wcslen, wcsncmp +# endif +#else +# include // for strcmp, strlen, strncmp +#endif + +size_t +xcslen(const XML_Char *s) { +#ifdef XML_UNICODE +# ifdef XML_UNICODE_WCHAR_T + return wcslen(s); +# else + // XML_Char is unsigned short + size_t len = 0; + while (s[len]) { + len++; + } + return len; +# endif +#else + return strlen(s); +#endif +} + +int +xcscmp(const XML_Char *a, const XML_Char *b) { +#if defined(XML_UNICODE) +# if defined(XML_UNICODE_WCHAR_T) + return wcscmp(a, b); +# else + for (; a[0] && b[0] && a[0] == b[0]; a++, b++) + ; + return a[0] - b[0]; +# endif +#else + return strcmp(a, b); +#endif +} + +int +xcsncmp(const XML_Char *a, const XML_Char *b, size_t len) { +#if defined(XML_UNICODE) +# if defined(XML_UNICODE_WCHAR_T) + return wcsncmp(a, b, len); +# else + for (; len > 0 && a[0] && b[0] && a[0] == b[0]; len--, a++, b++) { + } + return (len == 0) ? 0 : (a[0] - b[0]); +# endif +#else + return strncmp(a, b, len); +#endif +} diff --git a/Modules/expat/xcs.h b/Modules/expat/xcs.h new file mode 100644 index 000000000000000..1b942647d333093 --- /dev/null +++ b/Modules/expat/xcs.h @@ -0,0 +1,44 @@ +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2026 Sebastian Pipping + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT +*/ + +#if ! defined(EXPAT_XCS_H) + +# include // size_t +# include "expat_external.h" // for XML_Char + +extern size_t xcslen(const XML_Char *s); + +extern int xcscmp(const XML_Char *a, const XML_Char *b); +extern int xcsncmp(const XML_Char *a, const XML_Char *b, size_t len); + +#endif // ! defined(EXPAT_XCS_H) diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index 836db6c0b97f143..f1dd54cb605900b 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* c5625880f4bf417c1463deee4eb92d86ff413f802048621c57e25fe483eb59e4 (2.6.4+) +/* e3ca845466942140c9af531ac57808c4650116df5d6dbeda3b6546d2548a586e (2.9.0+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -13,7 +13,7 @@ Copyright (c) 2002-2016 Karl Waclawek Copyright (c) 2005-2009 Steven Solie Copyright (c) 2016 Eric Rahm - Copyright (c) 2016-2024 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2016 Gaurav Copyright (c) 2016 Thomas Beutlich Copyright (c) 2016 Gustavo Grieco @@ -39,8 +39,25 @@ Copyright (c) 2022 Sean McBride Copyright (c) 2023 Owain Davies Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow - Copyright (c) 2024 Berkay Eren Ürün + Copyright (c) 2024-2025 Berkay Eren Ürün Copyright (c) 2024 Hanno Böck + Copyright (c) 2025-2026 Matthew Fernandez + Copyright (c) 2025 Atrem Borovik + Copyright (c) 2025 Alfonso Gregory + Copyright (c) 2026 Rosen Penev + Copyright (c) 2026 Francesco Bertolaccini + Copyright (c) 2026 Christian Ng + Copyright (c) 2026 Nick Begg + Copyright (c) 2026 Kartik Kenchi + Copyright (c) 2026 Haris Hussain + Copyright (c) 2026 Evgeny Kotkov + Copyright (c) 2026 Darren Carreras + Copyright (c) 2026 Alberto Maschietto + Copyright (c) 2026 Zeyou Liu + Copyright (c) 2026 Stan Ulbrych + Copyright (c) 2026 Braian Plaku + Copyright (c) 2026 Filippo Tedeschi + Copyright (c) 2026 Junki Lee Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -61,6 +78,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #define XML_BUILDING_EXPAT 1 @@ -80,28 +99,17 @@ # error XML_CONTEXT_BYTES must be defined, non-empty and >=0 (0 to disable, >=1 to enable; 1024 is a common default) #endif -#if defined(HAVE_SYSCALL_GETRANDOM) -# if ! defined(_GNU_SOURCE) -# define _GNU_SOURCE 1 /* syscall prototype */ -# endif -#endif - -#ifdef _WIN32 -/* force stdlib to define rand_s() */ -# if ! defined(_CRT_RAND_S) -# define _CRT_RAND_S -# endif -#endif - #include #include #include /* memset(), memcpy() */ #include -#include /* UINT_MAX */ +#include /* INT_MAX, UINT_MAX */ #include /* fprintf */ -#include /* getenv, rand_s */ -#include /* uintptr_t */ +#include /* getenv */ +#include /* SIZE_MAX, UINT64_MAX, uint64_t, uintptr_t */ #include /* isnan */ +#include +#include /* wcsncmp */ #ifdef _WIN32 # define getpid GetCurrentProcessId @@ -120,32 +128,36 @@ #include "ascii.h" #include "expat.h" #include "siphash.h" +#include "xcs.h" + +#if defined(HAVE_ARC4RANDOM) +# include "random_arc4random.h" +#endif /* defined(HAVE_ARC4RANDOM) */ + +#if defined(HAVE_ARC4RANDOM_BUF) +# include "random_arc4random_buf.h" +#endif // defined(HAVE_ARC4RANDOM_BUF) + +#if defined(XML_DEV_URANDOM) +# include "random_dev_urandom.h" +#endif /* defined(XML_DEV_URANDOM) */ + +#if defined(HAVE_GETENTROPY) +# include "random_getentropy.h" +#endif // defined(HAVE_GETENTROPY) #if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) -# if defined(HAVE_GETRANDOM) -# include /* getrandom */ -# else -# include /* syscall */ -# include /* SYS_getrandom */ -# endif -# if ! defined(GRND_NONBLOCK) -# define GRND_NONBLOCK 0x0001 -# endif /* defined(GRND_NONBLOCK) */ -#endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ - -#if defined(HAVE_LIBBSD) \ - && (defined(HAVE_ARC4RANDOM_BUF) || defined(HAVE_ARC4RANDOM)) -# include -#endif +# include "random_getrandom.h" +#endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ -#if defined(_WIN32) && ! defined(LOAD_LIBRARY_SEARCH_SYSTEM32) -# define LOAD_LIBRARY_SEARCH_SYSTEM32 0x00000800 -#endif +#if defined(_WIN32) && ! defined(XML_POOR_ENTROPY) +# include "random_rand_s.h" +#endif /* defined(_WIN32) && ! defined(XML_POOR_ENTROPY) */ #if ! defined(HAVE_GETRANDOM) && ! defined(HAVE_SYSCALL_GETRANDOM) \ && ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) \ - && ! defined(XML_DEV_URANDOM) && ! defined(_WIN32) \ - && ! defined(XML_POOR_ENTROPY) + && ! defined(HAVE_GETENTROPY) && ! defined(XML_DEV_URANDOM) \ + && ! defined(_WIN32) && ! defined(XML_POOR_ENTROPY) # error You do not have support for any sources of high quality entropy \ enabled. For end user security, that is probably not what you want. \ \ @@ -154,13 +166,12 @@ * Linux >=3.17 + glibc (including <2.25) (syscall SYS_getrandom): HAVE_SYSCALL_GETRANDOM, \ * BSD / macOS >=10.7 / glibc >=2.36 (arc4random_buf): HAVE_ARC4RANDOM_BUF, \ * BSD / macOS (including <10.7) / glibc >=2.36 (arc4random): HAVE_ARC4RANDOM, \ - * libbsd (arc4random_buf): HAVE_ARC4RANDOM_BUF + HAVE_LIBBSD, \ - * libbsd (arc4random): HAVE_ARC4RANDOM + HAVE_LIBBSD, \ + * BSD / macOS >=10.12 / glibc >=2.25 (getentropy): HAVE_GETENTROPY, \ * Linux (including <3.17) / BSD / macOS (including <10.7) / Solaris >=8 (/dev/urandom): XML_DEV_URANDOM, \ * Windows >=Vista (rand_s): _WIN32. \ \ - If insist on not using any of these, bypass this error by defining \ - XML_POOR_ENTROPY; you have been warned. \ + If you insist on not using any of these, bypass this error by defining \ + XML_POOR_ENTROPY and be vulnerable to hash flooding; you have been warned. \ \ If you have reasons to patch this detection code away or need changes \ to the build system, please open a bug. Thank you! @@ -220,25 +231,10 @@ typedef char ICHAR; #define EXPAT_MIN(a, b) (((a) < (b)) ? (a) : (b)) #include "internal.h" +#include "hash_table.h" #include "xmltok.h" #include "xmlrole.h" -typedef const XML_Char *KEY; - -typedef struct { - KEY name; -} NAMED; - -typedef struct { - NAMED **v; - unsigned char power; - size_t size; - size_t used; - const XML_Memory_Handling_Suite *mem; -} HASH_TABLE; - -static size_t keylen(KEY s); - static void copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key); /* For probing (after a collision) we need a step size relative prime @@ -254,11 +250,6 @@ static void copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key); #define PROBE_STEP(hash, mask, power) \ ((unsigned char)((SECOND_HASH(hash, mask, power)) | 1)) -typedef struct { - NAMED **p; - NAMED **end; -} HASH_TABLE_ITER; - #define INIT_TAG_BUF_SIZE 32 /* must be a multiple of sizeof(XML_Char) */ #define INIT_DATA_BUF_SIZE 1024 #define INIT_ATTS_SIZE 16 @@ -274,8 +265,8 @@ typedef struct binding { struct binding *prevPrefixBinding; const struct attribute_id *attId; XML_Char *uri; - int uriLen; - int uriAlloc; + size_t uriLen; + size_t uriAlloc; } BINDING; typedef struct prefix { @@ -287,9 +278,9 @@ typedef struct { const XML_Char *str; const XML_Char *localPart; const XML_Char *prefix; - int strLen; - int uriLen; - int prefixLen; + size_t strLen; + size_t uriLen; + size_t prefixLen; } TAG_NAME; /* TAG represents an open element. @@ -310,8 +301,11 @@ typedef struct tag { const char *rawName; /* tagName in the original encoding */ int rawNameLength; TAG_NAME name; /* tagName in the API encoding */ - char *buf; /* buffer for name components */ - char *bufEnd; /* end of the buffer */ + union { + char *raw; /* for byte-level access (rawName storage) */ + XML_Char *str; /* for character-level access (converted name) */ + } buf; /* buffer for name components */ + char *bufEnd; /* end of the buffer */ BINDING *bindings; } TAG; @@ -324,9 +318,13 @@ typedef struct { const XML_Char *base; const XML_Char *publicId; const XML_Char *notation; - XML_Bool open; - XML_Bool is_param; - XML_Bool is_internal; /* true if declared in internal subset outside PE */ + bool open; + bool hasMore; /* true if entity has not been completely processed */ + /* An entity can be open while being already completely processed (!hasMore). + The reason is the delayed closing of entities until their inner + entities are processed and closed */ + bool is_param; + bool is_internal; /* true if declared in internal subset outside PE */ } ENTITY; typedef struct { @@ -344,7 +342,7 @@ typedef struct { typedef struct block { struct block *next; int size; - XML_Char s[1]; + XML_Char s[]; } BLOCK; typedef struct { @@ -353,7 +351,7 @@ typedef struct { const XML_Char *end; XML_Char *ptr; XML_Char *start; - const XML_Memory_Handling_Suite *mem; + XML_Parser parser; } STRING_POOL; /* The XML_Char before the name is used to determine whether @@ -361,8 +359,8 @@ typedef struct { typedef struct attribute_id { XML_Char *name; PPREFIX *prefix; - XML_Bool maybeTokenized; - XML_Bool xmlns; + bool maybeTokenized; + bool xmlns; } ATTRIBUTE_ID; typedef struct { @@ -371,6 +369,22 @@ typedef struct { const XML_Char *value; } DEFAULT_ATTRIBUTE; +// This structure allows mapping attribute names to instances of +// `DEFAULT_ATTRIBUTE`. +typedef struct { + // Member `name` goes first to make this structure compatible with structure + // `NAMED` (further up), which is needed to support use of structure + // `NAME_AND_DEFAULT_ATTRIBUTE` in a hash table as implemented by function + // `lookup` (further down). + const XML_Char *name; + // We would store a `DEFAULT_ATTRIBUTE *` here but the backing array + // can be reallocated which would invalidate the pointer. Using an index + // into the array instead, avoids that problem. + size_t attIndex; + // This is set to `false` by function `lookup`. + bool initialized; +} NAME_AND_DEFAULT_ATTRIBUTE; + typedef struct { unsigned long version; unsigned long hash; @@ -381,9 +395,10 @@ typedef struct { const XML_Char *name; PPREFIX *prefix; const ATTRIBUTE_ID *idAtt; - int nDefaultAtts; - int allocDefaultAtts; + size_t nDefaultAtts; + size_t allocDefaultAtts; DEFAULT_ATTRIBUTE *defaultAtts; + HASH_TABLE defaultAttForName; } ELEMENT_TYPE; typedef struct { @@ -401,27 +416,35 @@ typedef struct { XML_Bool standalone; #ifdef XML_DTD /* indicates if external PE has been read */ - XML_Bool paramEntityRead; + bool paramEntityRead; HASH_TABLE paramEntities; #endif /* XML_DTD */ PPREFIX defaultPrefix; /* === scaffolding for building content model === */ - XML_Bool in_eldecl; + bool in_eldecl; CONTENT_SCAFFOLD *scaffold; unsigned contentStringLen; unsigned scaffSize; unsigned scaffCount; int scaffLevel; int *scaffIndex; + size_t scaffIndexSize; } DTD; +enum EntityType { + ENTITY_INTERNAL, + ENTITY_ATTRIBUTE, + ENTITY_VALUE, +}; + typedef struct open_internal_entity { const char *internalEventPtr; const char *internalEventEndPtr; struct open_internal_entity *next; ENTITY *entity; int startTagLevel; - XML_Bool betweenDecl; /* WFC: PE Between Declarations */ + bool betweenDecl; /* WFC: PE Between Declarations */ + enum EntityType type; } OPEN_INTERNAL_ENTITY; enum XML_Account { @@ -441,6 +464,14 @@ typedef struct accounting { unsigned long long activationThresholdBytes; } ACCOUNTING; +typedef struct MALLOC_TRACKER { + size_t bytesAllocated; + size_t peakBytesAllocated; // updated live only for debug level >=2 + unsigned long debugLevel; + float maximumAmplificationFactor; // >=1.0 + XmlBigCount activationThresholdBytes; +} MALLOC_TRACKER; + typedef struct entity_stats { unsigned int countEverOpened; unsigned int currentDepth; @@ -449,8 +480,8 @@ typedef struct entity_stats { } ENTITY_STATS; #endif /* XML_GE == 1 */ -typedef enum XML_Error PTRCALL Processor(XML_Parser parser, const char *start, - const char *end, const char **endPtr); +typedef enum XML_Error Processor(XML_Parser parser, const char *start, + const char *end, const char **endPtr); static Processor prologProcessor; static Processor prologInitProcessor; @@ -481,8 +512,8 @@ static enum XML_Error doProlog(XML_Parser parser, const ENCODING *enc, const char *next, const char **nextPtr, XML_Bool haveMore, XML_Bool allowClosingDoctype, enum XML_Account account); -static enum XML_Error processInternalEntity(XML_Parser parser, ENTITY *entity, - XML_Bool betweenDecl); +static enum XML_Error processEntity(XML_Parser parser, ENTITY *entity, + bool betweenDecl, enum EntityType type); static enum XML_Error doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, const char *start, const char *end, const char **endPtr, @@ -513,18 +544,22 @@ static enum XML_Error storeAttributeValue(XML_Parser parser, const char *ptr, const char *end, STRING_POOL *pool, enum XML_Account account); -static enum XML_Error appendAttributeValue(XML_Parser parser, - const ENCODING *enc, - XML_Bool isCdata, const char *ptr, - const char *end, STRING_POOL *pool, - enum XML_Account account); +static enum XML_Error +appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, + const char *ptr, const char *end, STRING_POOL *pool, + enum XML_Account account, const char **nextPtr); static ATTRIBUTE_ID *getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start, const char *end); static int setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *elementType); #if XML_GE == 1 static enum XML_Error storeEntityValue(XML_Parser parser, const ENCODING *enc, const char *start, const char *end, - enum XML_Account account); + enum XML_Account account, + const char **nextPtr); +static enum XML_Error callStoreEntityValue(XML_Parser parser, + const ENCODING *enc, + const char *start, const char *end, + enum XML_Account account); #else static enum XML_Error storeSelfEntityValue(XML_Parser parser, ENTITY *entity); #endif @@ -538,57 +573,47 @@ static void reportDefault(XML_Parser parser, const ENCODING *enc, static const XML_Char *getContext(XML_Parser parser); static XML_Bool setContext(XML_Parser parser, const XML_Char *context); -static void FASTCALL normalizePublicId(XML_Char *s); +static void normalizePublicId(XML_Char *s); -static DTD *dtdCreate(const XML_Memory_Handling_Suite *ms); +static DTD *dtdCreate(XML_Parser parser); /* do not call if m_parentParser != NULL */ -static void dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms); -static void dtdDestroy(DTD *p, XML_Bool isDocEntity, - const XML_Memory_Handling_Suite *ms); +static void dtdReset(DTD *p, XML_Parser parser); +static void dtdDestroy(DTD *p, XML_Bool isDocEntity, XML_Parser parser); static int dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, - const XML_Memory_Handling_Suite *ms); + XML_Parser parser); static int copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, STRING_POOL *newPool, const HASH_TABLE *oldTable); -static NAMED *lookup(XML_Parser parser, HASH_TABLE *table, KEY name, - size_t createSize); -static void FASTCALL hashTableInit(HASH_TABLE *table, - const XML_Memory_Handling_Suite *ms); -static void FASTCALL hashTableClear(HASH_TABLE *table); -static void FASTCALL hashTableDestroy(HASH_TABLE *table); -static void FASTCALL hashTableIterInit(HASH_TABLE_ITER *iter, - const HASH_TABLE *table); -static NAMED *FASTCALL hashTableIterNext(HASH_TABLE_ITER *iter); - -static void FASTCALL poolInit(STRING_POOL *pool, - const XML_Memory_Handling_Suite *ms); -static void FASTCALL poolClear(STRING_POOL *pool); -static void FASTCALL poolDestroy(STRING_POOL *pool); + +static void poolInit(STRING_POOL *pool, XML_Parser parser); +static void poolClear(STRING_POOL *pool); +static void poolDestroy(STRING_POOL *pool); static XML_Char *poolAppend(STRING_POOL *pool, const ENCODING *enc, const char *ptr, const char *end); static XML_Char *poolStoreString(STRING_POOL *pool, const ENCODING *enc, const char *ptr, const char *end); -static XML_Bool FASTCALL poolGrow(STRING_POOL *pool); -static const XML_Char *FASTCALL poolCopyString(STRING_POOL *pool, - const XML_Char *s); +static XML_Bool poolGrow(STRING_POOL *pool); +static bool poolGrowUntil(STRING_POOL *pool, size_t needed); +static const XML_Char *poolCopyString(STRING_POOL *pool, const XML_Char *s); +static const XML_Char *poolCopyStringNoFinish(STRING_POOL *pool, + const XML_Char *s); static const XML_Char *poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n); -static const XML_Char *FASTCALL poolAppendString(STRING_POOL *pool, - const XML_Char *s); +static const XML_Char *poolAppendString(STRING_POOL *pool, const XML_Char *s); -static int FASTCALL nextScaffoldPart(XML_Parser parser); +static int nextScaffoldPart(XML_Parser parser); static XML_Content *build_model(XML_Parser parser); static ELEMENT_TYPE *getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, const char *end); -static XML_Char *copyString(const XML_Char *s, - const XML_Memory_Handling_Suite *memsuite); +static XML_Char *copyString(const XML_Char *s, XML_Parser parser); -static unsigned long generate_hash_secret_salt(XML_Parser parser); +static struct sipkey generate_hash_secret_salt(void); static XML_Bool startParsing(XML_Parser parser); static XML_Parser parserCreate(const XML_Char *encodingName, const XML_Memory_Handling_Suite *memsuite, - const XML_Char *nameSep, DTD *dtd); + const XML_Char *nameSep, DTD *dtd, + XML_Parser parentParser); static void parserInit(XML_Parser parser, const XML_Char *encodingName); @@ -612,24 +637,67 @@ static void entityTrackingOnOpen(XML_Parser parser, ENTITY *entity, int sourceLine); static void entityTrackingOnClose(XML_Parser parser, ENTITY *entity, int sourceLine); +#endif /* XML_GE == 1 */ static XML_Parser getRootParserOf(XML_Parser parser, unsigned int *outLevelDiff); -#endif /* XML_GE == 1 */ static unsigned long getDebugLevel(const char *variableName, unsigned long defaultDebugLevel); +static bool poolAppendChar(STRING_POOL *pool, XML_Char c); + +static bool poolAppendChars(STRING_POOL *pool, const XML_Char *s, size_t len); + +#if XML_GE == 1 +static enum XML_Prop_Error setBillionLaughsAttackProtectionMaximumAmplification( + XML_Parser parser, float maximumAmplificationFactor); + +static enum XML_Prop_Error setBillionLaughsAttackProtectionActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes); + +static enum XML_Prop_Error +setAllocTrackerMaximumAmplification(XML_Parser parser, + float maximumAmplificationFactor); + +static enum XML_Prop_Error +setAllocTrackerActivationThreshold(XML_Parser parser, + unsigned long long activationThresholdBytes); +#endif /* XML_GE == 1 */ + +static enum XML_Prop_Error setReparseDeferralEnabled(XML_Parser parser, + XML_Bool enabled); + #define poolStart(pool) ((pool)->start) #define poolLength(pool) ((pool)->ptr - (pool)->start) #define poolChop(pool) ((void)--(pool->ptr)) #define poolLastChar(pool) (((pool)->ptr)[-1]) #define poolDiscard(pool) ((pool)->ptr = (pool)->start) #define poolFinish(pool) ((pool)->start = (pool)->ptr) -#define poolAppendChar(pool, c) \ - (((pool)->ptr == (pool)->end && ! poolGrow(pool)) \ - ? 0 \ - : ((*((pool)->ptr)++ = c), 1)) + +bool +poolAppendChar(STRING_POOL *pool, XML_Char c) { + if (pool->ptr == pool->end && ! poolGrow(pool)) + return false; + + *(pool->ptr)++ = c; + return true; +} + +bool +poolAppendChars(STRING_POOL *pool, const XML_Char *s, size_t len) { + // Detect and prevent integer overflow + if (len > SIZE_MAX / sizeof(XML_Char)) + return false; + + if (! poolGrowUntil(pool, len)) + return false; + + memcpy(pool->ptr, s, len * sizeof(XML_Char)); + pool->ptr += len; + + return true; +} #if ! defined(XML_TESTING) const @@ -661,7 +729,7 @@ struct XML_ParserStruct { char *m_bufferEnd; // past last character to be parsed const char *m_bufferLim; // allocated end of m_buffer - XML_Index m_parseEndByteIndex; + uint64_t m_parseEndByteIndex; const char *m_parseEndPtr; size_t m_partialTokenBytesBefore; /* used in heuristic to avoid O(n^2) */ XML_Bool m_reparseDeferralEnabled; @@ -700,6 +768,8 @@ struct XML_ParserStruct { void *m_unknownEncodingMem; void *m_unknownEncodingData; void *m_unknownEncodingHandlerData; + // Application callback invoked by callUnknownEncodingConvert. + int(XMLCALL *m_unknownEncodingConvert)(void *, const char *); void(XMLCALL *m_unknownEncodingRelease)(void *); PROLOG_STATE m_prologState; Processor *m_processor; @@ -708,7 +778,9 @@ struct XML_ParserStruct { const char *m_eventEndPtr; const char *m_positionPtr; OPEN_INTERNAL_ENTITY *m_openInternalEntities; - OPEN_INTERNAL_ENTITY *m_freeInternalEntities; + OPEN_INTERNAL_ENTITY *m_openAttributeEntities; + OPEN_INTERNAL_ENTITY *m_openValueEntities; + OPEN_INTERNAL_ENTITY *m_freeEntities; XML_Bool m_defaultExpandInternalEntities; int m_tagLevel; ENTITY *m_declEntity; @@ -728,21 +800,18 @@ struct XML_ParserStruct { TAG *m_freeTagList; BINDING *m_inheritedBindings; BINDING *m_freeBindingList; - int m_attsSize; + size_t m_attsSize; int m_nSpecifiedAtts; int m_idAttIndex; ATTRIBUTE *m_atts; NS_ATT *m_nsAtts; unsigned long m_nsAttsVersion; unsigned char m_nsAttsPower; -#ifdef XML_ATTR_INFO - XML_AttrInfo *m_attInfo; -#endif POSITION m_position; STRING_POOL m_tempPool; STRING_POOL m_temp2Pool; char *m_groupConnector; - unsigned int m_groupSize; + size_t m_groupSize; XML_Char m_namespaceSeparator; XML_Parser m_parentParser; XML_ParsingStatus m_parsingStatus; @@ -751,165 +820,265 @@ struct XML_ParserStruct { XML_Bool m_useForeignDTD; enum XML_ParamEntityParsing m_paramEntityParsing; #endif - unsigned long m_hash_secret_salt; + struct sipkey m_hash_secret_salt_128; + XML_Bool m_hash_secret_salt_set; #if XML_GE == 1 ACCOUNTING m_accounting; + MALLOC_TRACKER m_alloc_tracker; ENTITY_STATS m_entity_stats; #endif + XML_Bool m_reenter; + unsigned m_handlerCallDepth; }; -#define MALLOC(parser, s) (parser->m_mem.malloc_fcn((s))) -#define REALLOC(parser, p, s) (parser->m_mem.realloc_fcn((p), (s))) -#define FREE(parser, p) (parser->m_mem.free_fcn((p))) - -XML_Parser XMLCALL -XML_ParserCreate(const XML_Char *encodingName) { - return XML_ParserCreate_MM(encodingName, NULL, NULL); -} +#if XML_GE == 1 +# define MALLOC(parser, s) (expat_malloc((parser), (s), __LINE__)) +# define REALLOC(parser, p, s) (expat_realloc((parser), (p), (s), __LINE__)) +# define FREE(parser, p) (expat_free((parser), (p), __LINE__)) +#else +# define MALLOC(parser, s) (parser->m_mem.malloc_fcn((s))) +# define REALLOC(parser, p, s) (parser->m_mem.realloc_fcn((p), (s))) +# define FREE(parser, p) (parser->m_mem.free_fcn((p))) +#endif -XML_Parser XMLCALL -XML_ParserCreateNS(const XML_Char *encodingName, XML_Char nsSep) { - XML_Char tmp[2] = {nsSep, 0}; - return XML_ParserCreate_MM(encodingName, NULL, tmp); +#if XML_GE == 1 +static void +expat_heap_stat(XML_Parser rootParser, char operator, size_t absDiff, + size_t newTotal, size_t peakTotal, int sourceLine) { + // NOTE: This can be +infinity or -nan + const float amplification + = (float)newTotal / (float)rootParser->m_accounting.countBytesDirect; + fprintf( + stderr, + "expat: Allocations(%p): Direct " EXPAT_FMT_ULL("10") ", allocated %c" EXPAT_FMT_SIZE_T( + "10") " to " EXPAT_FMT_SIZE_T("10") " (" EXPAT_FMT_SIZE_T("10") " peak), amplification %8.2f (xmlparse.c:%d)\n", + (void *)rootParser, rootParser->m_accounting.countBytesDirect, operator, + absDiff, newTotal, peakTotal, (double)amplification, sourceLine); } -// "xml=http://www.w3.org/XML/1998/namespace" -static const XML_Char implicitContext[] - = {ASCII_x, ASCII_m, ASCII_l, ASCII_EQUALS, ASCII_h, - ASCII_t, ASCII_t, ASCII_p, ASCII_COLON, ASCII_SLASH, - ASCII_SLASH, ASCII_w, ASCII_w, ASCII_w, ASCII_PERIOD, - ASCII_w, ASCII_3, ASCII_PERIOD, ASCII_o, ASCII_r, - ASCII_g, ASCII_SLASH, ASCII_X, ASCII_M, ASCII_L, - ASCII_SLASH, ASCII_1, ASCII_9, ASCII_9, ASCII_8, - ASCII_SLASH, ASCII_n, ASCII_a, ASCII_m, ASCII_e, - ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e, - '\0'}; +static bool +expat_heap_increase_tolerable(XML_Parser rootParser, size_t increase, + int sourceLine) { + assert(rootParser != NULL); + assert(increase > 0); -/* To avoid warnings about unused functions: */ -#if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) + size_t newTotal = 0; + bool tolerable = true; -# if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) + // Detect integer overflow + if (SIZE_MAX - rootParser->m_alloc_tracker.bytesAllocated < increase) { + tolerable = false; + } else { + newTotal = rootParser->m_alloc_tracker.bytesAllocated + increase; + + if (newTotal >= rootParser->m_alloc_tracker.activationThresholdBytes) { + assert(newTotal > 0); + // NOTE: This can be +infinity when dividing by zero but not -nan + const float amplification + = (float)newTotal / (float)rootParser->m_accounting.countBytesDirect; + if (amplification + > rootParser->m_alloc_tracker.maximumAmplificationFactor) { + tolerable = false; + } + } + } -/* Obtain entropy on Linux 3.17+ */ -static int -writeRandomBytes_getrandom_nonblock(void *target, size_t count) { - int success = 0; /* full count bytes written? */ - size_t bytesWrittenTotal = 0; - const unsigned int getrandomFlags = GRND_NONBLOCK; + if (! tolerable && (rootParser->m_alloc_tracker.debugLevel >= 1)) { + expat_heap_stat(rootParser, '+', increase, newTotal, newTotal, sourceLine); + } - do { - void *const currentTarget = (void *)((char *)target + bytesWrittenTotal); - const size_t bytesToWrite = count - bytesWrittenTotal; + return tolerable; +} - const int bytesWrittenMore = -# if defined(HAVE_GETRANDOM) - getrandom(currentTarget, bytesToWrite, getrandomFlags); -# else - syscall(SYS_getrandom, currentTarget, bytesToWrite, getrandomFlags); -# endif +# if defined(XML_TESTING) +void * +# else +static void * +# endif +expat_malloc(XML_Parser parser, size_t size, int sourceLine) { + // Detect integer overflow + if (SIZE_MAX - size < sizeof(size_t) + EXPAT_MALLOC_PADDING) { + return NULL; + } - if (bytesWrittenMore > 0) { - bytesWrittenTotal += bytesWrittenMore; - if (bytesWrittenTotal >= count) - success = 1; - } - } while (! success && (errno == EINTR)); + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(rootParser->m_parentParser == NULL); - return success; -} + const size_t bytesToAllocate = sizeof(size_t) + EXPAT_MALLOC_PADDING + size; -# endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ + if (SIZE_MAX - rootParser->m_alloc_tracker.bytesAllocated < bytesToAllocate) { + return NULL; // i.e. signal integer overflow as out-of-memory + } -# if ! defined(_WIN32) && defined(XML_DEV_URANDOM) + if (! expat_heap_increase_tolerable(rootParser, bytesToAllocate, + sourceLine)) { + return NULL; // i.e. signal violation as out-of-memory + } -/* Extract entropy from /dev/urandom */ -static int -writeRandomBytes_dev_urandom(void *target, size_t count) { - int success = 0; /* full count bytes written? */ - size_t bytesWrittenTotal = 0; + // Actually allocate + void *const mallocedPtr = parser->m_mem.malloc_fcn(bytesToAllocate); - const int fd = open("/dev/urandom", O_RDONLY); - if (fd < 0) { - return 0; + if (mallocedPtr == NULL) { + return NULL; } - do { - void *const currentTarget = (void *)((char *)target + bytesWrittenTotal); - const size_t bytesToWrite = count - bytesWrittenTotal; + // Update in-block recorded size + *(size_t *)mallocedPtr = size; - const ssize_t bytesWrittenMore = read(fd, currentTarget, bytesToWrite); + // Update accounting + rootParser->m_alloc_tracker.bytesAllocated += bytesToAllocate; - if (bytesWrittenMore > 0) { - bytesWrittenTotal += bytesWrittenMore; - if (bytesWrittenTotal >= count) - success = 1; + // Report as needed + if (rootParser->m_alloc_tracker.debugLevel >= 2) { + if (rootParser->m_alloc_tracker.bytesAllocated + > rootParser->m_alloc_tracker.peakBytesAllocated) { + rootParser->m_alloc_tracker.peakBytesAllocated + = rootParser->m_alloc_tracker.bytesAllocated; } - } while (! success && (errno == EINTR)); + expat_heap_stat(rootParser, '+', bytesToAllocate, + rootParser->m_alloc_tracker.bytesAllocated, + rootParser->m_alloc_tracker.peakBytesAllocated, sourceLine); + } - close(fd); - return success; + return (char *)mallocedPtr + sizeof(size_t) + EXPAT_MALLOC_PADDING; } -# endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */ +# if defined(XML_TESTING) +void +# else +static void +# endif +expat_free(XML_Parser parser, void *ptr, int sourceLine) { + assert(parser != NULL); -#endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */ + if (ptr == NULL) { + return; + } -#if defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF) + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(rootParser->m_parentParser == NULL); -static void -writeRandomBytes_arc4random(void *target, size_t count) { - size_t bytesWrittenTotal = 0; + // Extract size (to the eyes of malloc_fcn/realloc_fcn) and + // the original pointer returned by malloc/realloc + void *const mallocedPtr = (char *)ptr - EXPAT_MALLOC_PADDING - sizeof(size_t); + const size_t bytesAllocated + = sizeof(size_t) + EXPAT_MALLOC_PADDING + *(size_t *)mallocedPtr; - while (bytesWrittenTotal < count) { - const uint32_t random32 = arc4random(); - size_t i = 0; + // Update accounting + assert(rootParser->m_alloc_tracker.bytesAllocated >= bytesAllocated); + rootParser->m_alloc_tracker.bytesAllocated -= bytesAllocated; - for (; (i < sizeof(random32)) && (bytesWrittenTotal < count); - i++, bytesWrittenTotal++) { - const uint8_t random8 = (uint8_t)(random32 >> (i * 8)); - ((uint8_t *)target)[bytesWrittenTotal] = random8; - } + // Report as needed + if (rootParser->m_alloc_tracker.debugLevel >= 2) { + expat_heap_stat(rootParser, '-', bytesAllocated, + rootParser->m_alloc_tracker.bytesAllocated, + rootParser->m_alloc_tracker.peakBytesAllocated, sourceLine); } + + // NOTE: This may be freeing rootParser, so freeing has to come last + parser->m_mem.free_fcn(mallocedPtr); } -#endif /* defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF) */ +# if defined(XML_TESTING) +void * +# else +static void * +# endif +expat_realloc(XML_Parser parser, void *ptr, size_t size, int sourceLine) { + assert(parser != NULL); -#ifdef _WIN32 + if (ptr == NULL) { + return expat_malloc(parser, size, sourceLine); + } -/* Provide declaration of rand_s() for MinGW-32 (not 64, which has it), - as it didn't declare it in its header prior to version 5.3.0 of its - runtime package (mingwrt, containing stdlib.h). The upstream fix - was introduced at https://osdn.net/projects/mingw/ticket/39658 . */ -# if defined(__MINGW32__) && defined(__MINGW32_VERSION) \ - && __MINGW32_VERSION < 5003000L && ! defined(__MINGW64_VERSION_MAJOR) -__declspec(dllimport) int rand_s(unsigned int *); -# endif + if (size == 0) { + expat_free(parser, ptr, sourceLine); + return NULL; + } -/* Obtain entropy on Windows using the rand_s() function which - * generates cryptographically secure random numbers. Internally it - * uses RtlGenRandom API which is present in Windows XP and later. - */ -static int -writeRandomBytes_rand_s(void *target, size_t count) { - size_t bytesWrittenTotal = 0; + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(rootParser->m_parentParser == NULL); + + // Extract original size (to the eyes of the caller) and the original + // pointer returned by malloc/realloc + void *mallocedPtr = (char *)ptr - EXPAT_MALLOC_PADDING - sizeof(size_t); + const size_t prevSize = *(size_t *)mallocedPtr; + + // Classify upcoming change + const bool isIncrease = (size > prevSize); + const size_t absDiff + = (size > prevSize) ? (size - prevSize) : (prevSize - size); + + // Ask for permission from accounting + if (isIncrease) { + if (! expat_heap_increase_tolerable(rootParser, absDiff, sourceLine)) { + return NULL; // i.e. signal violation as out-of-memory + } + } + + // Detect and prevent integer overflow + if (size > SIZE_MAX - sizeof(size_t) - EXPAT_MALLOC_PADDING) { + return NULL; + } + + // Actually allocate + mallocedPtr = parser->m_mem.realloc_fcn( + mallocedPtr, sizeof(size_t) + EXPAT_MALLOC_PADDING + size); - while (bytesWrittenTotal < count) { - unsigned int random32 = 0; - size_t i = 0; + if (mallocedPtr == NULL) { + return NULL; + } - if (rand_s(&random32)) - return 0; /* failure */ + // Update accounting + if (isIncrease) { + assert(SIZE_MAX - rootParser->m_alloc_tracker.bytesAllocated >= absDiff); + rootParser->m_alloc_tracker.bytesAllocated += absDiff; + } else { // i.e. decrease + assert(rootParser->m_alloc_tracker.bytesAllocated >= absDiff); + rootParser->m_alloc_tracker.bytesAllocated -= absDiff; + } - for (; (i < sizeof(random32)) && (bytesWrittenTotal < count); - i++, bytesWrittenTotal++) { - const uint8_t random8 = (uint8_t)(random32 >> (i * 8)); - ((uint8_t *)target)[bytesWrittenTotal] = random8; + // Report as needed + if (rootParser->m_alloc_tracker.debugLevel >= 2) { + if (rootParser->m_alloc_tracker.bytesAllocated + > rootParser->m_alloc_tracker.peakBytesAllocated) { + rootParser->m_alloc_tracker.peakBytesAllocated + = rootParser->m_alloc_tracker.bytesAllocated; } + expat_heap_stat(rootParser, isIncrease ? '+' : '-', absDiff, + rootParser->m_alloc_tracker.bytesAllocated, + rootParser->m_alloc_tracker.peakBytesAllocated, sourceLine); } - return 1; /* success */ + + // Update in-block recorded size + *(size_t *)mallocedPtr = size; + + return (char *)mallocedPtr + sizeof(size_t) + EXPAT_MALLOC_PADDING; +} +#endif // XML_GE == 1 + +XML_Parser XMLCALL +XML_ParserCreate(const XML_Char *encodingName) { + return XML_ParserCreate_MM(encodingName, NULL, NULL); } -#endif /* _WIN32 */ +XML_Parser XMLCALL +XML_ParserCreateNS(const XML_Char *encodingName, XML_Char nsSep) { + XML_Char tmp[2] = {nsSep, 0}; + return XML_ParserCreate_MM(encodingName, NULL, tmp); +} + +// "xml=http://www.w3.org/XML/1998/namespace" +static const XML_Char implicitContext[] + = {ASCII_x, ASCII_m, ASCII_l, ASCII_EQUALS, ASCII_h, + ASCII_t, ASCII_t, ASCII_p, ASCII_COLON, ASCII_SLASH, + ASCII_SLASH, ASCII_w, ASCII_w, ASCII_w, ASCII_PERIOD, + ASCII_w, ASCII_3, ASCII_PERIOD, ASCII_o, ASCII_r, + ASCII_g, ASCII_SLASH, ASCII_X, ASCII_M, ASCII_L, + ASCII_SLASH, ASCII_1, ASCII_9, ASCII_9, ASCII_8, + ASCII_SLASH, ASCII_n, ASCII_a, ASCII_m, ASCII_e, + ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e, + '\0'}; #if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) @@ -938,63 +1107,104 @@ gather_time_entropy(void) { #endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */ -static unsigned long -ENTROPY_DEBUG(const char *label, unsigned long entropy) { +static struct sipkey +ENTROPY_DEBUG(const char *label, struct sipkey entropy_128) { if (getDebugLevel("EXPAT_ENTROPY_DEBUG", 0) >= 1u) { - fprintf(stderr, "expat: Entropy: %s --> 0x%0*lx (%lu bytes)\n", label, - (int)sizeof(entropy) * 2, entropy, (unsigned long)sizeof(entropy)); + fprintf(stderr, + "expat: Entropy: %s --> [0x" EXPAT_FMT_LLX( + "016") ", 0x" EXPAT_FMT_LLX("016") "] (16 bytes)\n", + label, (unsigned long long)entropy_128.k[0], + (unsigned long long)entropy_128.k[1]); } - return entropy; + return entropy_128; } -static unsigned long -generate_hash_secret_salt(XML_Parser parser) { - unsigned long entropy; - (void)parser; +static struct sipkey +generate_hash_secret_salt(void) { + struct sipkey entropy; /* "Failproof" high quality providers: */ #if defined(HAVE_ARC4RANDOM_BUF) - arc4random_buf(&entropy, sizeof(entropy)); + writeRandomBytes_arc4random_buf(&entropy, sizeof(entropy)); return ENTROPY_DEBUG("arc4random_buf", entropy); #elif defined(HAVE_ARC4RANDOM) - writeRandomBytes_arc4random((void *)&entropy, sizeof(entropy)); + writeRandomBytes_arc4random(&entropy, sizeof(entropy)); return ENTROPY_DEBUG("arc4random", entropy); #else /* Try high quality providers first .. */ -# ifdef _WIN32 - if (writeRandomBytes_rand_s((void *)&entropy, sizeof(entropy))) { +# if defined(_WIN32) && ! defined(XML_POOR_ENTROPY) + if (writeRandomBytes_rand_s(&entropy, sizeof(entropy))) { return ENTROPY_DEBUG("rand_s", entropy); } +# elif defined(HAVE_GETENTROPY) + if (writeRandomBytes_getentropy(&entropy, sizeof(entropy))) { + return ENTROPY_DEBUG("getentropy", entropy); + } + errno = 0; # elif defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) - if (writeRandomBytes_getrandom_nonblock((void *)&entropy, sizeof(entropy))) { + if (writeRandomBytes_getrandom_nonblock(&entropy, sizeof(entropy))) { return ENTROPY_DEBUG("getrandom", entropy); } # endif # if ! defined(_WIN32) && defined(XML_DEV_URANDOM) - if (writeRandomBytes_dev_urandom((void *)&entropy, sizeof(entropy))) { + if (writeRandomBytes_dev_urandom(&entropy, sizeof(entropy))) { return ENTROPY_DEBUG("/dev/urandom", entropy); } # endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */ /* .. and self-made low quality for backup: */ + entropy.k[0] = 0; + entropy.k[1] = gather_time_entropy(); +# if ! defined(__wasi__) /* Process ID is 0 bits entropy if attacker has local access */ - entropy = gather_time_entropy() ^ getpid(); + entropy.k[1] ^= getpid(); +# endif /* Factors are 2^31-1 and 2^61-1 (Mersenne primes M31 and M61) */ if (sizeof(unsigned long) == 4) { - return ENTROPY_DEBUG("fallback(4)", entropy * 2147483647); + entropy.k[1] *= 2147483647; + return ENTROPY_DEBUG("fallback(4)", entropy); } else { - return ENTROPY_DEBUG("fallback(8)", - entropy * (unsigned long)2305843009213693951ULL); + entropy.k[1] *= 2305843009213693951ULL; + return ENTROPY_DEBUG("fallback(8)", entropy); } #endif } -static unsigned long -get_hash_secret_salt(XML_Parser parser) { - if (parser->m_parentParser != NULL) - return get_hash_secret_salt(parser->m_parentParser); - return parser->m_hash_secret_salt; +static void +beforeHandler(XML_Parser parser) { + assert(parser->m_handlerCallDepth < UINT_MAX); + parser->m_handlerCallDepth++; +} + +static void +afterHandler(XML_Parser parser) { + assert(parser->m_handlerCallDepth > 0); + parser->m_handlerCallDepth--; +} + +static bool +isCalledFromInsideHandler(XML_Parser parser) { + return parser->m_handlerCallDepth > 0; +} + +static void +callUnknownEncodingRelease(XML_Parser parser) { + beforeHandler(parser); + parser->m_unknownEncodingRelease(parser->m_unknownEncodingData); + afterHandler(parser); + parser->m_unknownEncodingRelease = NULL; + parser->m_unknownEncodingData = NULL; +} + +static int XMLCALL +callUnknownEncodingConvert(void *data, const char *p) { + XML_Parser parser = data; + beforeHandler(parser); + const int result + = parser->m_unknownEncodingConvert(parser->m_unknownEncodingData, p); + afterHandler(parser); + return result; } static enum XML_Error @@ -1028,7 +1238,29 @@ callProcessor(XML_Parser parser, const char *start, const char *end, #if defined(XML_TESTING) g_bytesScanned += (unsigned)have_now; #endif - const enum XML_Error ret = parser->m_processor(parser, start, end, endPtr); + // Run in a loop to eliminate dangerous recursion depths + enum XML_Error ret; + *endPtr = start; + while (1) { + // Use endPtr as the new start in each iteration, since it will + // be set to the next start point by m_processor. + ret = parser->m_processor(parser, *endPtr, end, endPtr); + + // Make parsing status (and in particular XML_SUSPENDED) take + // precedence over re-enter flag when they disagree + if (parser->m_parsingStatus.parsing != XML_PARSING) { + parser->m_reenter = XML_FALSE; + } + + if (! parser->m_reenter) { + break; + } + + parser->m_reenter = XML_FALSE; + if (ret != XML_ERROR_NONE) + return ret; + } + if (ret == XML_ERROR_NONE) { // if we consumed nothing, remember what we had on this parse attempt. if (*endPtr == start) { @@ -1043,8 +1275,10 @@ callProcessor(XML_Parser parser, const char *start, const char *end, static XML_Bool /* only valid for root parser */ startParsing(XML_Parser parser) { /* hash functions must be initialized before setContext() is called */ - if (parser->m_hash_secret_salt == 0) - parser->m_hash_secret_salt = generate_hash_secret_salt(parser); + if (parser->m_hash_secret_salt_set != XML_TRUE) { + parser->m_hash_secret_salt_128 = generate_hash_secret_salt(); + parser->m_hash_secret_salt_set = XML_TRUE; + } if (parser->m_ns) { /* implicit context only set for root parser, since child parsers (i.e. external entity parsers) will inherit it @@ -1058,19 +1292,43 @@ XML_Parser XMLCALL XML_ParserCreate_MM(const XML_Char *encodingName, const XML_Memory_Handling_Suite *memsuite, const XML_Char *nameSep) { - return parserCreate(encodingName, memsuite, nameSep, NULL); + return parserCreate(encodingName, memsuite, nameSep, NULL, NULL); } static XML_Parser parserCreate(const XML_Char *encodingName, const XML_Memory_Handling_Suite *memsuite, const XML_Char *nameSep, - DTD *dtd) { - XML_Parser parser; + DTD *dtd, XML_Parser parentParser) { + XML_Parser parser = NULL; + +#if XML_GE == 1 + const size_t increase + = sizeof(size_t) + EXPAT_MALLOC_PADDING + sizeof(struct XML_ParserStruct); + + if (parentParser != NULL) { + const XML_Parser rootParser = getRootParserOf(parentParser, NULL); + if (! expat_heap_increase_tolerable(rootParser, increase, __LINE__)) { + return NULL; + } + } +#else + UNUSED_P(parentParser); +#endif if (memsuite) { XML_Memory_Handling_Suite *mtemp; +#if XML_GE == 1 + void *const sizeAndParser + = memsuite->malloc_fcn(sizeof(size_t) + EXPAT_MALLOC_PADDING + + sizeof(struct XML_ParserStruct)); + if (sizeAndParser != NULL) { + *(size_t *)sizeAndParser = sizeof(struct XML_ParserStruct); + parser = (XML_Parser)((char *)sizeAndParser + sizeof(size_t) + + EXPAT_MALLOC_PADDING); +#else parser = memsuite->malloc_fcn(sizeof(struct XML_ParserStruct)); if (parser != NULL) { +#endif mtemp = (XML_Memory_Handling_Suite *)&(parser->m_mem); mtemp->malloc_fcn = memsuite->malloc_fcn; mtemp->realloc_fcn = memsuite->realloc_fcn; @@ -1078,44 +1336,80 @@ parserCreate(const XML_Char *encodingName, } } else { XML_Memory_Handling_Suite *mtemp; - parser = (XML_Parser)malloc(sizeof(struct XML_ParserStruct)); +#if XML_GE == 1 + void *const sizeAndParser = malloc(sizeof(size_t) + EXPAT_MALLOC_PADDING + + sizeof(struct XML_ParserStruct)); + if (sizeAndParser != NULL) { + *(size_t *)sizeAndParser = sizeof(struct XML_ParserStruct); + parser = (XML_Parser)((char *)sizeAndParser + sizeof(size_t) + + EXPAT_MALLOC_PADDING); +#else + parser = malloc(sizeof(struct XML_ParserStruct)); if (parser != NULL) { +#endif mtemp = (XML_Memory_Handling_Suite *)&(parser->m_mem); mtemp->malloc_fcn = malloc; mtemp->realloc_fcn = realloc; mtemp->free_fcn = free; } - } + } // cppcheck-suppress[memleak symbolName=sizeAndParser] // Cppcheck >=2.18.0 if (! parser) return parser; +#if XML_GE == 1 + // Initialize .m_alloc_tracker + memset(&parser->m_alloc_tracker, 0, sizeof(MALLOC_TRACKER)); + if (parentParser == NULL) { + parser->m_alloc_tracker.debugLevel + = getDebugLevel("EXPAT_MALLOC_DEBUG", 0u); + parser->m_alloc_tracker.maximumAmplificationFactor + = EXPAT_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT; + parser->m_alloc_tracker.activationThresholdBytes + = EXPAT_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT; + + // NOTE: This initialization needs to come this early because these fields + // are read by allocation tracking code + parser->m_parentParser = NULL; + parser->m_accounting.countBytesDirect = 0; + } else { + parser->m_parentParser = parentParser; + } + + // Record XML_ParserStruct allocation we did a few lines up before + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(rootParser->m_parentParser == NULL); + assert(SIZE_MAX - rootParser->m_alloc_tracker.bytesAllocated >= increase); + rootParser->m_alloc_tracker.bytesAllocated += increase; + + // Report on allocation + if (rootParser->m_alloc_tracker.debugLevel >= 2) { + if (rootParser->m_alloc_tracker.bytesAllocated + > rootParser->m_alloc_tracker.peakBytesAllocated) { + rootParser->m_alloc_tracker.peakBytesAllocated + = rootParser->m_alloc_tracker.bytesAllocated; + } + + expat_heap_stat(rootParser, '+', increase, + rootParser->m_alloc_tracker.bytesAllocated, + rootParser->m_alloc_tracker.peakBytesAllocated, __LINE__); + } +#else + parser->m_parentParser = NULL; +#endif // XML_GE == 1 + parser->m_buffer = NULL; parser->m_bufferLim = NULL; parser->m_attsSize = INIT_ATTS_SIZE; - parser->m_atts - = (ATTRIBUTE *)MALLOC(parser, parser->m_attsSize * sizeof(ATTRIBUTE)); + parser->m_atts = MALLOC(parser, parser->m_attsSize * sizeof(ATTRIBUTE)); if (parser->m_atts == NULL) { FREE(parser, parser); return NULL; } -#ifdef XML_ATTR_INFO - parser->m_attInfo = (XML_AttrInfo *)MALLOC( - parser, parser->m_attsSize * sizeof(XML_AttrInfo)); - if (parser->m_attInfo == NULL) { - FREE(parser, parser->m_atts); - FREE(parser, parser); - return NULL; - } -#endif - parser->m_dataBuf - = (XML_Char *)MALLOC(parser, INIT_DATA_BUF_SIZE * sizeof(XML_Char)); + parser->m_dataBuf = MALLOC(parser, INIT_DATA_BUF_SIZE * sizeof(XML_Char)); if (parser->m_dataBuf == NULL) { FREE(parser, parser->m_atts); -#ifdef XML_ATTR_INFO - FREE(parser, parser->m_attInfo); -#endif FREE(parser, parser); return NULL; } @@ -1124,13 +1418,10 @@ parserCreate(const XML_Char *encodingName, if (dtd) parser->m_dtd = dtd; else { - parser->m_dtd = dtdCreate(&parser->m_mem); + parser->m_dtd = dtdCreate(parser); if (parser->m_dtd == NULL) { FREE(parser, parser->m_dataBuf); FREE(parser, parser->m_atts); -#ifdef XML_ATTR_INFO - FREE(parser, parser->m_attInfo); -#endif FREE(parser, parser); return NULL; } @@ -1138,7 +1429,7 @@ parserCreate(const XML_Char *encodingName, parser->m_freeBindingList = NULL; parser->m_freeTagList = NULL; - parser->m_freeInternalEntities = NULL; + parser->m_freeEntities = NULL; parser->m_groupSize = 0; parser->m_groupConnector = NULL; @@ -1156,8 +1447,8 @@ parserCreate(const XML_Char *encodingName, parser->m_protocolEncodingName = NULL; - poolInit(&parser->m_tempPool, &(parser->m_mem)); - poolInit(&parser->m_temp2Pool, &(parser->m_mem)); + poolInit(&parser->m_tempPool, parser); + poolInit(&parser->m_temp2Pool, parser); parserInit(parser, encodingName); if (encodingName && ! parser->m_protocolEncodingName) { @@ -1189,7 +1480,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_processor = prologInitProcessor; XmlPrologStateInit(&parser->m_prologState); if (encodingName != NULL) { - parser->m_protocolEncodingName = copyString(encodingName, &(parser->m_mem)); + parser->m_protocolEncodingName = copyString(encodingName, parser); } parser->m_curBase = NULL; XmlInitEncoding(&parser->m_initEncoding, &parser->m_encoding, 0); @@ -1241,22 +1532,29 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_eventEndPtr = NULL; parser->m_positionPtr = NULL; parser->m_openInternalEntities = NULL; + parser->m_openAttributeEntities = NULL; + parser->m_openValueEntities = NULL; parser->m_defaultExpandInternalEntities = XML_TRUE; parser->m_tagLevel = 0; parser->m_tagStack = NULL; parser->m_inheritedBindings = NULL; parser->m_nSpecifiedAtts = 0; parser->m_unknownEncodingMem = NULL; + parser->m_unknownEncodingConvert = NULL; parser->m_unknownEncodingRelease = NULL; parser->m_unknownEncodingData = NULL; - parser->m_parentParser = NULL; parser->m_parsingStatus.parsing = XML_INITIALIZED; + // Reentry can only be triggered inside m_processor calls + parser->m_reenter = XML_FALSE; + parser->m_handlerCallDepth = 0; #ifdef XML_DTD parser->m_isParamEntity = XML_FALSE; parser->m_useForeignDTD = XML_FALSE; parser->m_paramEntityParsing = XML_PARAM_ENTITY_PARSING_NEVER; #endif - parser->m_hash_secret_salt = 0; + parser->m_hash_secret_salt_128.k[0] = 0; + parser->m_hash_secret_salt_128.k[1] = 0; + parser->m_hash_secret_salt_set = XML_FALSE; #if XML_GE == 1 memset(&parser->m_accounting, 0, sizeof(ACCOUNTING)); @@ -1272,7 +1570,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { } /* moves list of bindings to m_freeBindingList */ -static void FASTCALL +static void moveToFreeBindingList(XML_Parser parser, BINDING *bindings) { while (bindings) { BINDING *b = bindings; @@ -1282,16 +1580,30 @@ moveToFreeBindingList(XML_Parser parser, BINDING *bindings) { } } +/* Moves a list of entities onto the start of another list. */ +static void +moveEntityList(OPEN_INTERNAL_ENTITY **dst, OPEN_INTERNAL_ENTITY **src) { + for (OPEN_INTERNAL_ENTITY *head = *src; head != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = head; + head = head->next; + openEntity->next = *dst; + *dst = openEntity; + } +} + XML_Bool XMLCALL XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { TAG *tStk; - OPEN_INTERNAL_ENTITY *openEntityList; - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return XML_FALSE; if (parser->m_parentParser) return XML_FALSE; + // The application-defined release callback may access the parser, so it + // must run before any parser state is freed. + if (parser->m_unknownEncodingRelease) + callUnknownEncodingRelease(parser); /* move m_tagStack to m_freeTagList */ tStk = parser->m_tagStack; while (tStk) { @@ -1302,27 +1614,42 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { tag->bindings = NULL; parser->m_freeTagList = tag; } - /* move m_openInternalEntities to m_freeInternalEntities */ - openEntityList = parser->m_openInternalEntities; - while (openEntityList) { - OPEN_INTERNAL_ENTITY *openEntity = openEntityList; - openEntityList = openEntity->next; - openEntity->next = parser->m_freeInternalEntities; - parser->m_freeInternalEntities = openEntity; - } + /* move m_openInternalEntities to m_freeEntities */ + moveEntityList(&parser->m_freeEntities, &parser->m_openInternalEntities); + /* move m_openAttributeEntities to m_freeEntities (i.e. same task but for + * attributes) */ + moveEntityList(&parser->m_freeEntities, &parser->m_openAttributeEntities); + /* move m_openValueEntities to m_freeEntities (i.e. same task but for value + * entities) */ + moveEntityList(&parser->m_freeEntities, &parser->m_openValueEntities); moveToFreeBindingList(parser, parser->m_inheritedBindings); FREE(parser, parser->m_unknownEncodingMem); - if (parser->m_unknownEncodingRelease) - parser->m_unknownEncodingRelease(parser->m_unknownEncodingData); poolClear(&parser->m_tempPool); poolClear(&parser->m_temp2Pool); FREE(parser, (void *)parser->m_protocolEncodingName); parser->m_protocolEncodingName = NULL; parserInit(parser, encodingName); - dtdReset(parser->m_dtd, &parser->m_mem); + dtdReset(parser->m_dtd, parser); + if (encodingName && ! parser->m_protocolEncodingName) { + parser->m_errorCode = XML_ERROR_NO_MEMORY; + return XML_FALSE; + } return XML_TRUE; } +static XML_Bool +parserBusy(XML_Parser parser) { + switch (parser->m_parsingStatus.parsing) { + case XML_PARSING: + case XML_SUSPENDED: + return XML_TRUE; + case XML_INITIALIZED: + case XML_FINISHED: + default: + return XML_FALSE; + } +} + enum XML_Status XMLCALL XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName) { if (parser == NULL) @@ -1331,8 +1658,7 @@ XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName) { XXX There's no way for the caller to determine which of the XXX possible error cases caused the XML_STATUS_ERROR return. */ - if (parser->m_parsingStatus.parsing == XML_PARSING - || parser->m_parsingStatus.parsing == XML_SUSPENDED) + if (parserBusy(parser)) return XML_STATUS_ERROR; /* Get rid of any previous encoding name */ @@ -1343,7 +1669,7 @@ XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName) { parser->m_protocolEncodingName = NULL; else { /* Copy the new encoding name into allocated memory */ - parser->m_protocolEncodingName = copyString(encodingName, &(parser->m_mem)); + parser->m_protocolEncodingName = copyString(encodingName, parser); if (! parser->m_protocolEncodingName) return XML_STATUS_ERROR; } @@ -1372,6 +1698,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, XML_ExternalEntityRefHandler oldExternalEntityRefHandler; XML_SkippedEntityHandler oldSkippedEntityHandler; XML_UnknownEncodingHandler oldUnknownEncodingHandler; + void *oldUnknownEncodingHandlerData; XML_ElementDeclHandler oldElementDeclHandler; XML_AttlistDeclHandler oldAttlistDeclHandler; XML_EntityDeclHandler oldEntityDeclHandler; @@ -1392,7 +1719,8 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, from hash tables associated with either parser without us having to worry which hash secrets each table has. */ - unsigned long oldhash_secret_salt; + struct sipkey oldhash_secret_salt_128; + XML_Bool oldhash_secret_salt_set; XML_Bool oldReparseDeferralEnabled; /* Validate the oldParser parameter before we pull everything out of it */ @@ -1417,6 +1745,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, oldExternalEntityRefHandler = parser->m_externalEntityRefHandler; oldSkippedEntityHandler = parser->m_skippedEntityHandler; oldUnknownEncodingHandler = parser->m_unknownEncodingHandler; + oldUnknownEncodingHandlerData = parser->m_unknownEncodingHandlerData; oldElementDeclHandler = parser->m_elementDeclHandler; oldAttlistDeclHandler = parser->m_attlistDeclHandler; oldEntityDeclHandler = parser->m_entityDeclHandler; @@ -1437,7 +1766,8 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, from hash tables associated with either parser without us having to worry which hash secrets each table has. */ - oldhash_secret_salt = parser->m_hash_secret_salt; + oldhash_secret_salt_128 = parser->m_hash_secret_salt_128; + oldhash_secret_salt_set = parser->m_hash_secret_salt_set; oldReparseDeferralEnabled = parser->m_reparseDeferralEnabled; #ifdef XML_DTD @@ -1445,16 +1775,12 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, newDtd = oldDtd; #endif /* XML_DTD */ - /* Note that the magical uses of the pre-processor to make field - access look more like C++ require that `parser' be overwritten - here. This makes this function more painful to follow than it - would be otherwise. - */ if (parser->m_ns) { XML_Char tmp[2] = {parser->m_namespaceSeparator, 0}; - parser = parserCreate(encodingName, &parser->m_mem, tmp, newDtd); + parser = parserCreate(encodingName, &parser->m_mem, tmp, newDtd, oldParser); } else { - parser = parserCreate(encodingName, &parser->m_mem, NULL, newDtd); + parser + = parserCreate(encodingName, &parser->m_mem, NULL, newDtd, oldParser); } if (! parser) @@ -1476,6 +1802,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, parser->m_externalEntityRefHandler = oldExternalEntityRefHandler; parser->m_skippedEntityHandler = oldSkippedEntityHandler; parser->m_unknownEncodingHandler = oldUnknownEncodingHandler; + parser->m_unknownEncodingHandlerData = oldUnknownEncodingHandlerData; parser->m_elementDeclHandler = oldElementDeclHandler; parser->m_attlistDeclHandler = oldAttlistDeclHandler; parser->m_entityDeclHandler = oldEntityDeclHandler; @@ -1490,7 +1817,8 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, parser->m_externalEntityRefHandlerArg = oldExternalEntityRefHandlerArg; parser->m_defaultExpandInternalEntities = oldDefaultExpandInternalEntities; parser->m_ns_triplets = oldns_triplets; - parser->m_hash_secret_salt = oldhash_secret_salt; + parser->m_hash_secret_salt_128 = oldhash_secret_salt_128; + parser->m_hash_secret_salt_set = oldhash_secret_salt_set; parser->m_reparseDeferralEnabled = oldReparseDeferralEnabled; parser->m_parentParser = oldParser; #ifdef XML_DTD @@ -1498,7 +1826,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, parser->m_prologState.inEntityValue = oldInEntityValue; if (context) { #endif /* XML_DTD */ - if (! dtdCopy(oldParser, parser->m_dtd, oldDtd, &parser->m_mem) + if (! dtdCopy(oldParser, parser->m_dtd, oldDtd, parser) || ! setContext(parser, context)) { XML_ParserFree(parser); return NULL; @@ -1521,7 +1849,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, return parser; } -static void FASTCALL +static void destroyBindings(BINDING *bindings, XML_Parser parser) { for (;;) { BINDING *b = bindings; @@ -1536,9 +1864,12 @@ destroyBindings(BINDING *bindings, XML_Parser parser) { void XMLCALL XML_ParserFree(XML_Parser parser) { TAG *tagList; - OPEN_INTERNAL_ENTITY *entityList; - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return; + // The application-defined release callback may access the parser, so it + // must run before any parser state is freed. + if (parser->m_unknownEncodingRelease) + callUnknownEncodingRelease(parser); /* free m_tagStack and m_freeTagList */ tagList = parser->m_tagStack; for (;;) { @@ -1551,25 +1882,39 @@ XML_ParserFree(XML_Parser parser) { } p = tagList; tagList = tagList->parent; - FREE(parser, p->buf); + FREE(parser, p->buf.raw); destroyBindings(p->bindings, parser); FREE(parser, p); } - /* free m_openInternalEntities and m_freeInternalEntities */ - entityList = parser->m_openInternalEntities; - for (;;) { - OPEN_INTERNAL_ENTITY *openEntity; - if (entityList == NULL) { - if (parser->m_freeInternalEntities == NULL) - break; - entityList = parser->m_freeInternalEntities; - parser->m_freeInternalEntities = NULL; - } - openEntity = entityList; + /* free m_openInternalEntities */ + for (OPEN_INTERNAL_ENTITY *entityList = parser->m_openInternalEntities; + entityList != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = entityList; entityList = entityList->next; FREE(parser, openEntity); } - + /* free m_openAttributeEntities */ + for (OPEN_INTERNAL_ENTITY *entityList = parser->m_openAttributeEntities; + entityList != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = entityList; + entityList = entityList->next; + FREE(parser, openEntity); + } + /* free m_openValueEntities */ + for (OPEN_INTERNAL_ENTITY *entityList = parser->m_openValueEntities; + entityList != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = entityList; + entityList = entityList->next; + FREE(parser, openEntity); + } + /* free m_freeEntities */ + for (OPEN_INTERNAL_ENTITY *entityList = parser->m_freeEntities; + entityList != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = entityList; + entityList = entityList->next; + FREE(parser, openEntity); + } + parser->m_freeEntities = NULL; destroyBindings(parser->m_freeBindingList, parser); destroyBindings(parser->m_inheritedBindings, parser); poolDestroy(&parser->m_tempPool); @@ -1583,19 +1928,16 @@ XML_ParserFree(XML_Parser parser) { #else if (parser->m_dtd) #endif /* XML_DTD */ - dtdDestroy(parser->m_dtd, (XML_Bool)! parser->m_parentParser, - &parser->m_mem); - FREE(parser, (void *)parser->m_atts); -#ifdef XML_ATTR_INFO - FREE(parser, (void *)parser->m_attInfo); -#endif + dtdDestroy(parser->m_dtd, (XML_Bool)! parser->m_parentParser, parser); + FREE(parser, parser->m_atts); FREE(parser, parser->m_groupConnector); - FREE(parser, parser->m_buffer); + // NOTE: We are avoiding FREE(..) here because parser->m_buffer + // is not being allocated with MALLOC(..) but with plain + // .malloc_fcn(..). + parser->m_mem.free_fcn(parser->m_buffer); FREE(parser, parser->m_dataBuf); FREE(parser, parser->m_nsAtts); FREE(parser, parser->m_unknownEncodingMem); - if (parser->m_unknownEncodingRelease) - parser->m_unknownEncodingRelease(parser->m_unknownEncodingData); FREE(parser, parser); } @@ -1611,8 +1953,7 @@ XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD) { return XML_ERROR_INVALID_ARGUMENT; #ifdef XML_DTD /* block after XML_Parse()/XML_ParseBuffer() has been called */ - if (parser->m_parsingStatus.parsing == XML_PARSING - || parser->m_parsingStatus.parsing == XML_SUSPENDED) + if (parserBusy(parser)) return XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING; parser->m_useForeignDTD = useDTD; return XML_ERROR_NONE; @@ -1627,8 +1968,7 @@ XML_SetReturnNSTriplet(XML_Parser parser, int do_nst) { if (parser == NULL) return; /* block after XML_Parse()/XML_ParseBuffer() has been called */ - if (parser->m_parsingStatus.parsing == XML_PARSING - || parser->m_parsingStatus.parsing == XML_SUSPENDED) + if (parserBusy(parser)) return; parser->m_ns_triplets = do_nst ? XML_TRUE : XML_FALSE; } @@ -1678,15 +2018,6 @@ XML_GetIdAttributeIndex(XML_Parser parser) { return parser->m_idAttIndex; } -#ifdef XML_ATTR_INFO -const XML_AttrInfo *XMLCALL -XML_GetAttributeInfo(XML_Parser parser) { - if (parser == NULL) - return NULL; - return parser->m_attInfo; -} -#endif - void XMLCALL XML_SetElementHandler(XML_Parser parser, XML_StartElementHandler start, XML_EndElementHandler end) { @@ -1897,8 +2228,7 @@ XML_SetParamEntityParsing(XML_Parser parser, if (parser == NULL) return 0; /* block after XML_Parse()/XML_ParseBuffer() has been called */ - if (parser->m_parsingStatus.parsing == XML_PARSING - || parser->m_parsingStatus.parsing == XML_SUSPENDED) + if (parserBusy(parser)) return 0; #ifdef XML_DTD parser->m_paramEntityParsing = peParsing; @@ -1908,20 +2238,65 @@ XML_SetParamEntityParsing(XML_Parser parser, #endif } -int XMLCALL -XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) { +/* The body of XML_SetHashSalt, so that Expat's own tests can reach it + without tripping the deprecation of the public function. */ +XML_NONTESTING_STATIC int +xmlSetHashSalt(XML_Parser parser, unsigned long hash_salt) { if (parser == NULL) return 0; - if (parser->m_parentParser) - return XML_SetHashSalt(parser->m_parentParser, hash_salt); + + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(! rootParser->m_parentParser); + /* block after XML_Parse()/XML_ParseBuffer() has been called */ - if (parser->m_parsingStatus.parsing == XML_PARSING - || parser->m_parsingStatus.parsing == XML_SUSPENDED) + if (parserBusy(rootParser)) return 0; - parser->m_hash_secret_salt = hash_salt; + + rootParser->m_hash_secret_salt_128.k[0] = 0; + rootParser->m_hash_secret_salt_128.k[1] = hash_salt; + + if (hash_salt != 0) { // to remain backwards compatible + rootParser->m_hash_secret_salt_set = XML_TRUE; + + if (sizeof(unsigned long) == 4) + ENTROPY_DEBUG("explicit(4)", rootParser->m_hash_secret_salt_128); + else + ENTROPY_DEBUG("explicit(8)", rootParser->m_hash_secret_salt_128); + } + return 1; } +// DEPRECATED since Expat 2.8.0. +int XMLCALL +XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) { + return xmlSetHashSalt(parser, hash_salt); +} + +XML_Bool XMLCALL +XML_SetHashSalt16Bytes(XML_Parser parser, const uint8_t entropy[16]) { + if (parser == NULL) + return XML_FALSE; + + if (entropy == NULL) + return XML_FALSE; + + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(! rootParser->m_parentParser); + + /* block after XML_Parse()/XML_ParseBuffer() has been called */ + if (parserBusy(rootParser)) + return XML_FALSE; + + sip_tokey(&(rootParser->m_hash_secret_salt_128), entropy); + + rootParser->m_hash_secret_salt_set = XML_TRUE; + + ENTROPY_DEBUG("explicit(16)", rootParser->m_hash_secret_salt_128); + + return XML_TRUE; +} + enum XML_Status XMLCALL XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { if ((parser == NULL) || (len < 0) || ((s == NULL) && (len != 0))) { @@ -1929,6 +2304,8 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT; return XML_STATUS_ERROR; } + if (isCalledFromInsideHandler(parser)) + return XML_STATUS_ERROR; switch (parser->m_parsingStatus.parsing) { case XML_SUSPENDED: parser->m_errorCode = XML_ERROR_SUSPENDED; @@ -1941,7 +2318,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return XML_STATUS_ERROR; } - /* fall through */ + EXPAT_FALLTHROUGH; default: parser->m_parsingStatus.parsing = XML_PARSING; } @@ -1952,7 +2329,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { int nLeftOver; enum XML_Status result; /* Detect overflow (a+b > MAX <==> b > MAX-a) */ - if ((XML_Size)len > ((XML_Size)-1) / 2 - parser->m_parseEndByteIndex) { + if ((uint64_t)len > UINT64_MAX - parser->m_parseEndByteIndex) { parser->m_errorCode = XML_ERROR_NO_MEMORY; parser->m_eventPtr = parser->m_eventEndPtr = NULL; parser->m_processor = errorProcessor; @@ -1983,7 +2360,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_parsingStatus.parsing = XML_FINISHED; return XML_STATUS_OK; } - /* fall through */ + EXPAT_FALLTHROUGH; default: result = XML_STATUS_OK; } @@ -2038,7 +2415,7 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { const char *start; enum XML_Status result = XML_STATUS_OK; - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return XML_STATUS_ERROR; if (len < 0) { @@ -2064,11 +2441,24 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return XML_STATUS_ERROR; } - /* fall through */ + EXPAT_FALLTHROUGH; default: parser->m_parsingStatus.parsing = XML_PARSING; } + if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd)) { + parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT; + return XML_STATUS_ERROR; + } + + // Detect and avoid integer overflow + if ((uint64_t)len > UINT64_MAX - parser->m_parseEndByteIndex) { + parser->m_errorCode = XML_ERROR_NO_MEMORY; + parser->m_eventPtr = parser->m_eventEndPtr = NULL; + parser->m_processor = errorProcessor; + return XML_STATUS_ERROR; + } + start = parser->m_bufferPtr; parser->m_positionPtr = start; parser->m_bufferEnd += len; @@ -2094,6 +2484,7 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { parser->m_parsingStatus.parsing = XML_FINISHED; return result; } + break; default:; /* should not happen */ } } @@ -2104,9 +2495,32 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { return result; } +/* Modifies `parser`’s buffer to be backed by `newBuf`. */ +static void +setParserBuffer(XML_Parser parser, char *newBuf, int newBufSize, int keep) { + parser->m_bufferLim = newBuf + newBufSize; + if (parser->m_bufferPtr) { + const int parsing + = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr); + memcpy(newBuf, parser->m_bufferPtr - keep, parsing + keep); + // NOTE: We are avoiding FREE(..) here because parser->m_buffer + // is not being allocated with MALLOC(..) but with plain + // .malloc_fcn(..). + parser->m_mem.free_fcn(parser->m_buffer); + parser->m_buffer = newBuf; + parser->m_bufferEnd = newBuf + parsing + keep; + parser->m_bufferPtr = newBuf + keep; + } else { + /* This must be a brand new buffer with no data in it yet */ + parser->m_buffer = newBuf; + parser->m_bufferEnd = newBuf; + parser->m_bufferPtr = newBuf; + } +} + void *XMLCALL XML_GetBuffer(XML_Parser parser, int len) { - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return NULL; if (len < 0) { parser->m_errorCode = XML_ERROR_NO_MEMORY; @@ -2127,9 +2541,6 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_lastBufferRequestSize = len; if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd) || parser->m_buffer == NULL) { -#if XML_CONTEXT_BYTES > 0 - int keep; -#endif /* XML_CONTEXT_BYTES > 0 */ /* Do not invoke signed arithmetic overflow: */ int neededSize = (int)((unsigned)len + (unsigned)EXPAT_SAFE_PTR_DIFF( @@ -2139,7 +2550,9 @@ XML_GetBuffer(XML_Parser parser, int len) { return NULL; } #if XML_CONTEXT_BYTES > 0 - keep = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer); + const int parsed + = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer); + int keep = parsed; if (keep > XML_CONTEXT_BYTES) keep = XML_CONTEXT_BYTES; /* Detect and prevent integer overflow */ @@ -2147,16 +2560,16 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return NULL; } - neededSize += keep; +#else + int keep = 0; #endif /* XML_CONTEXT_BYTES > 0 */ + neededSize += keep; if (parser->m_buffer && parser->m_bufferPtr && neededSize <= EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer)) { #if XML_CONTEXT_BYTES > 0 - if (keep < EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer)) { - int offset - = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer) - - keep; + if (keep < parsed) { + int offset = parsed - keep; /* The buffer pointers cannot be NULL here; we have at least some bytes * in the buffer */ memmove(parser->m_buffer, &parser->m_buffer[offset], @@ -2173,7 +2586,6 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_bufferPtr = parser->m_buffer; #endif /* XML_CONTEXT_BYTES > 0 */ } else { - char *newBuf; int bufferSize = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer); if (bufferSize == 0) @@ -2186,43 +2598,14 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return NULL; } - newBuf = (char *)MALLOC(parser, bufferSize); - if (newBuf == 0) { + // NOTE: We are avoiding MALLOC(..) here to leave limiting + // the input size to the application using Expat. + char *const newBuf = parser->m_mem.malloc_fcn(bufferSize); + if (newBuf == NULL) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return NULL; } - parser->m_bufferLim = newBuf + bufferSize; -#if XML_CONTEXT_BYTES > 0 - if (parser->m_bufferPtr) { - memcpy(newBuf, &parser->m_bufferPtr[-keep], - EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) - + keep); - FREE(parser, parser->m_buffer); - parser->m_buffer = newBuf; - parser->m_bufferEnd - = parser->m_buffer - + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) - + keep; - parser->m_bufferPtr = parser->m_buffer + keep; - } else { - /* This must be a brand new buffer with no data in it yet */ - parser->m_bufferEnd = newBuf; - parser->m_bufferPtr = parser->m_buffer = newBuf; - } -#else - if (parser->m_bufferPtr) { - memcpy(newBuf, parser->m_bufferPtr, - EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)); - FREE(parser, parser->m_buffer); - parser->m_bufferEnd - = newBuf - + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr); - } else { - /* This must be a brand new buffer with no data in it yet */ - parser->m_bufferEnd = newBuf; - } - parser->m_bufferPtr = parser->m_buffer = newBuf; -#endif /* XML_CONTEXT_BYTES > 0 */ + setParserBuffer(parser, newBuf, bufferSize, keep); } parser->m_eventPtr = parser->m_eventEndPtr = NULL; parser->m_positionPtr = NULL; @@ -2230,6 +2613,11 @@ XML_GetBuffer(XML_Parser parser, int len) { return parser->m_bufferEnd; } +static void +triggerReenter(XML_Parser parser) { + parser->m_reenter = XML_TRUE; +} + enum XML_Status XMLCALL XML_StopParser(XML_Parser parser, XML_Bool resumable) { if (parser == NULL) @@ -2270,7 +2658,7 @@ enum XML_Status XMLCALL XML_ResumeParser(XML_Parser parser) { enum XML_Status result = XML_STATUS_OK; - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return XML_STATUS_ERROR; if (parser->m_parsingStatus.parsing != XML_SUSPENDED) { parser->m_errorCode = XML_ERROR_NOT_SUSPENDED; @@ -2296,6 +2684,7 @@ XML_ResumeParser(XML_Parser parser) { parser->m_parsingStatus.parsing = XML_FINISHED; return result; } + break; default:; } } @@ -2321,35 +2710,57 @@ XML_GetErrorCode(XML_Parser parser) { return parser->m_errorCode; } -XML_Index XMLCALL -XML_GetCurrentByteIndex(XML_Parser parser) { +int64_t XMLCALL +XML_GetCurrentByteIndex64(XML_Parser parser) { if (parser == NULL) return -1; - if (parser->m_eventPtr) - return (XML_Index)(parser->m_parseEndByteIndex - - (parser->m_parseEndPtr - parser->m_eventPtr)); + if (parser->m_eventPtr) { + return (int64_t)(parser->m_parseEndByteIndex + - (parser->m_parseEndPtr - parser->m_eventPtr)); + } return -1; } -int XMLCALL -XML_GetCurrentByteCount(XML_Parser parser) { +// DEPRECATED since Expat 2.9.0. +XML_Index XMLCALL +XML_GetCurrentByteIndex(XML_Parser parser) { + // NOTE: XML_Index is known to wrap around for >2 GiB content + // on 32bit machines and 64bit Windows, unless (non-default and + // uncommon) XML_LARGE_SIZE is defined. + // That's a bug and it only lives on because we cannot break + // ABI compatibility of public API. + return (XML_Index)XML_GetCurrentByteIndex64(parser); +} + +uint64_t XMLCALL +XML_GetCurrentByteCount64(XML_Parser parser) { if (parser == NULL) return 0; - if (parser->m_eventEndPtr && parser->m_eventPtr) - return (int)(parser->m_eventEndPtr - parser->m_eventPtr); + if (parser->m_eventEndPtr && parser->m_eventPtr) { + return parser->m_eventEndPtr - parser->m_eventPtr; + } return 0; } +// DEPRECATED since Expat 2.9.0. +int XMLCALL +XML_GetCurrentByteCount(XML_Parser parser) { + // NOTE: int is known to wrap around for >2 GiB content. + // That's a bug and it only lives on because we cannot break + // ABI compatibility of public API. + return (int)XML_GetCurrentByteCount64(parser); +} + const char *XMLCALL -XML_GetInputContext(XML_Parser parser, int *offset, int *size) { +XML_GetInputContext64(XML_Parser parser, int64_t *offset, uint64_t *size) { #if XML_CONTEXT_BYTES > 0 if (parser == NULL) return NULL; if (parser->m_eventPtr && parser->m_buffer) { if (offset != NULL) - *offset = (int)(parser->m_eventPtr - parser->m_buffer); + *offset = parser->m_eventPtr - parser->m_buffer; if (size != NULL) - *size = (int)(parser->m_bufferEnd - parser->m_buffer); + *size = parser->m_bufferEnd - parser->m_buffer; return parser->m_buffer; } #else @@ -2357,11 +2768,44 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size) { (void)offset; (void)size; #endif /* XML_CONTEXT_BYTES > 0 */ - return (const char *)0; + return NULL; } -XML_Size XMLCALL -XML_GetCurrentLineNumber(XML_Parser parser) { +// DEPRECATED since Expat 2.9.0. +const char *XMLCALL +XML_GetInputContext(XML_Parser parser, int *offset, int *size) { +#if XML_CONTEXT_BYTES > 0 + if (parser == NULL) + return NULL; + + int64_t offset64; + uint64_t size64; + + const char *const buffer = XML_GetInputContext64(parser, &offset64, &size64); + + if (buffer == NULL) + return NULL; + + // NOTE: int is known to wrap around for >2 GiB content. + // That's a bug and it only lives on because we cannot break + // ABI compatibility of public API. + if (offset != NULL) + *offset = (int)offset64; + + if (size != NULL) + *size = (int)size64; + + return buffer; +#else + (void)parser; + (void)offset; + (void)size; +#endif /* XML_CONTEXT_BYTES > 0 */ + return NULL; +} + +uint64_t XMLCALL +XML_GetCurrentLineNumber64(XML_Parser parser) { if (parser == NULL) return 0; if (parser->m_eventPtr && parser->m_eventPtr >= parser->m_positionPtr) { @@ -2372,8 +2816,19 @@ XML_GetCurrentLineNumber(XML_Parser parser) { return parser->m_position.lineNumber + 1; } +// DEPRECATED since Expat 2.9.0. XML_Size XMLCALL -XML_GetCurrentColumnNumber(XML_Parser parser) { +XML_GetCurrentLineNumber(XML_Parser parser) { + // NOTE: XML_Size is known to wrap around for >4 GiB content + // on 32bit machines and 64bit Windows, unless (non-default and + // uncommon) XML_LARGE_SIZE is defined. + // That's a bug and it only lives on because we cannot break + // ABI compatibility of public API. + return (XML_Size)XML_GetCurrentLineNumber64(parser); +} + +uint64_t XMLCALL +XML_GetCurrentColumnNumber64(XML_Parser parser) { if (parser == NULL) return 0; if (parser->m_eventPtr && parser->m_eventPtr >= parser->m_positionPtr) { @@ -2384,35 +2839,61 @@ XML_GetCurrentColumnNumber(XML_Parser parser) { return parser->m_position.columnNumber; } +// DEPRECATED since Expat 2.9.0. +XML_Size XMLCALL +XML_GetCurrentColumnNumber(XML_Parser parser) { + // NOTE: XML_Size is known to wrap around for >4 GiB content + // on 32bit machines and 64bit Windows, unless (non-default and + // uncommon) XML_LARGE_SIZE is defined. + // That's a bug and it only lives on because we cannot break + // ABI compatibility of public API. + return (XML_Size)XML_GetCurrentColumnNumber64(parser); +} + void XMLCALL XML_FreeContentModel(XML_Parser parser, XML_Content *model) { - if (parser != NULL) - FREE(parser, model); + if (parser == NULL) + return; + + // NOTE: We are avoiding FREE(..) here because the content model + // has been created using plain .malloc_fcn(..) rather than MALLOC(..). + parser->m_mem.free_fcn(model); } void *XMLCALL XML_MemMalloc(XML_Parser parser, size_t size) { if (parser == NULL) return NULL; - return MALLOC(parser, size); + + // NOTE: We are avoiding MALLOC(..) here to not include + // user allocations with allocation tracking and limiting. + return parser->m_mem.malloc_fcn(size); } void *XMLCALL XML_MemRealloc(XML_Parser parser, void *ptr, size_t size) { if (parser == NULL) return NULL; - return REALLOC(parser, ptr, size); + + // NOTE: We are avoiding REALLOC(..) here to not include + // user allocations with allocation tracking and limiting. + return parser->m_mem.realloc_fcn(ptr, size); } void XMLCALL XML_MemFree(XML_Parser parser, void *ptr) { - if (parser != NULL) - FREE(parser, ptr); + if (parser == NULL) + return; + + // NOTE: We are avoiding FREE(..) here because XML_MemMalloc and + // XML_MemRealloc are not using MALLOC(..) and REALLOC(..) + // but plain .malloc_fcn(..) and .realloc_fcn(..), internally. + parser->m_mem.free_fcn(ptr); } void XMLCALL XML_DefaultCurrent(XML_Parser parser) { - if (parser == NULL) + if (parser == NULL || ! isCalledFromInsideHandler(parser)) return; if (parser->m_defaultHandler) { if (parser->m_openInternalEntities) @@ -2583,67 +3064,326 @@ XML_GetFeatureList(void) { {XML_FEATURE_CONTEXT_BYTES, XML_L("XML_CONTEXT_BYTES"), XML_CONTEXT_BYTES}, #endif -#ifdef XML_MIN_SIZE - {XML_FEATURE_MIN_SIZE, XML_L("XML_MIN_SIZE"), 0}, -#endif #ifdef XML_NS {XML_FEATURE_NS, XML_L("XML_NS"), 0}, #endif #ifdef XML_LARGE_SIZE {XML_FEATURE_LARGE_SIZE, XML_L("XML_LARGE_SIZE"), 0}, #endif -#ifdef XML_ATTR_INFO - {XML_FEATURE_ATTR_INFO, XML_L("XML_ATTR_INFO"), 0}, +#if XML_GE == 1 + /* Added in Expat 2.4.0 for XML_DTD defined and + * added in Expat 2.6.0 for XML_GE == 1. */ + {XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT, + XML_L("XML_BLAP_MAX_AMP"), + (long int) + EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT}, + {XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT, + XML_L("XML_BLAP_ACT_THRES"), + EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT}, + /* Added in Expat 2.6.0. */ + {XML_FEATURE_GE, XML_L("XML_GE"), 0}, + /* Added in Expat 2.7.2. */ + {XML_FEATURE_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT, + XML_L("XML_AT_MAX_AMP"), + (long int)EXPAT_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT}, + {XML_FEATURE_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT, + XML_L("XML_AT_ACT_THRES"), + (long int)EXPAT_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT}, +#endif + {XML_FEATURE_END, NULL, 0}}; + + return features; +} + +#if XML_GE == 1 +static enum XML_Prop_Error +setBillionLaughsAttackProtectionMaximumAmplification( + XML_Parser parser, float maximumAmplificationFactor) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (parser->m_parentParser != NULL) + return XML_PROP_ERROR_PARSER_NOT_ROOT; + + if (isnan(maximumAmplificationFactor) || (maximumAmplificationFactor < 1.0f)) + return XML_PROP_ERROR_INVALID_VALUE; + + parser->m_accounting.maximumAmplificationFactor = maximumAmplificationFactor; + + return XML_PROP_ERROR_NONE; +} + +XML_Bool XMLCALL +XML_SetBillionLaughsAttackProtectionMaximumAmplification( + XML_Parser parser, float maximumAmplificationFactor) { + return (setBillionLaughsAttackProtectionMaximumAmplification( + parser, maximumAmplificationFactor) + == XML_PROP_ERROR_NONE) + ? XML_TRUE + : XML_FALSE; +} + +static enum XML_Prop_Error +setBillionLaughsAttackProtectionActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (parser->m_parentParser != NULL) + return XML_PROP_ERROR_PARSER_NOT_ROOT; + + parser->m_accounting.activationThresholdBytes = activationThresholdBytes; + + return XML_PROP_ERROR_NONE; +} + +XML_Bool XMLCALL +XML_SetBillionLaughsAttackProtectionActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes) { + return (setBillionLaughsAttackProtectionActivationThreshold( + parser, activationThresholdBytes) + == XML_PROP_ERROR_NONE) + ? XML_TRUE + : XML_FALSE; +} + +static enum XML_Prop_Error +setAllocTrackerMaximumAmplification(XML_Parser parser, + float maximumAmplificationFactor) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (parser->m_parentParser != NULL) + return XML_PROP_ERROR_PARSER_NOT_ROOT; + + if (isnan(maximumAmplificationFactor) || (maximumAmplificationFactor < 1.0f)) + return XML_PROP_ERROR_INVALID_VALUE; + + parser->m_alloc_tracker.maximumAmplificationFactor + = maximumAmplificationFactor; + + return XML_PROP_ERROR_NONE; +} + +XML_Bool XMLCALL +XML_SetAllocTrackerMaximumAmplification(XML_Parser parser, + float maximumAmplificationFactor) { + return (setAllocTrackerMaximumAmplification(parser, + maximumAmplificationFactor) + == XML_PROP_ERROR_NONE) + ? XML_TRUE + : XML_FALSE; +} + +static enum XML_Prop_Error +setAllocTrackerActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (parser->m_parentParser != NULL) + return XML_PROP_ERROR_PARSER_NOT_ROOT; + + parser->m_alloc_tracker.activationThresholdBytes = activationThresholdBytes; + + return XML_PROP_ERROR_NONE; +} + +XML_Bool XMLCALL +XML_SetAllocTrackerActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes) { + return (setAllocTrackerActivationThreshold(parser, activationThresholdBytes) + == XML_PROP_ERROR_NONE) + ? XML_TRUE + : XML_FALSE; +} +#endif /* XML_GE == 1 */ + +static enum XML_Prop_Error +setReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (enabled != XML_TRUE && enabled != XML_FALSE) + return XML_PROP_ERROR_INVALID_VALUE; + + parser->m_reparseDeferralEnabled = enabled; + + return XML_PROP_ERROR_NONE; +} + +XML_Bool XMLCALL +XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled) { + return (setReparseDeferralEnabled(parser, enabled) == XML_PROP_ERROR_NONE) + ? XML_TRUE + : XML_FALSE; +} + +enum XML_Prop_Error XMLCALL +XML_SetPropertyBool(XML_Parser parser, enum XML_Parser_Property property, + XML_Bool value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + switch (property) { +#if XML_GE == 1 + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: + return XML_PROP_ERROR_INVALID_TYPE; +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + return setReparseDeferralEnabled(parser, value); + default: + return XML_PROP_ERROR_INVALID_KEY; + } + + assert(0 && "considered unreachable"); +} + +enum XML_Prop_Error XMLCALL +XML_SetPropertyDouble(XML_Parser parser, enum XML_Parser_Property property, + double value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + +#if XML_GE == 0 + UNUSED_P(value); #endif + + switch (property) { #if XML_GE == 1 - /* Added in Expat 2.4.0 for XML_DTD defined and - * added in Expat 2.6.0 for XML_GE == 1. */ - {XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT, - XML_L("XML_BLAP_MAX_AMP"), - (long int) - EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT}, - {XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT, - XML_L("XML_BLAP_ACT_THRES"), - EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT}, - /* Added in Expat 2.6.0. */ - {XML_FEATURE_GE, XML_L("XML_GE"), 0}, -#endif - {XML_FEATURE_END, NULL, 0}}; + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + return setAllocTrackerMaximumAmplification(parser, (float)value); + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: + return setBillionLaughsAttackProtectionMaximumAmplification(parser, + (float)value); + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + return XML_PROP_ERROR_INVALID_TYPE; + default: + return XML_PROP_ERROR_INVALID_KEY; + } - return features; + assert(0 && "considered unreachable"); } +enum XML_Prop_Error XMLCALL +XML_SetPropertyUInt64(XML_Parser parser, enum XML_Parser_Property property, + uint64_t value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + +#if XML_GE == 0 + UNUSED_P(value); +#endif + + switch (property) { #if XML_GE == 1 -XML_Bool XMLCALL -XML_SetBillionLaughsAttackProtectionMaximumAmplification( - XML_Parser parser, float maximumAmplificationFactor) { - if ((parser == NULL) || (parser->m_parentParser != NULL) - || isnan(maximumAmplificationFactor) - || (maximumAmplificationFactor < 1.0f)) { - return XML_FALSE; + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + return setAllocTrackerActivationThreshold(parser, + (unsigned long long)value); + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: + return setBillionLaughsAttackProtectionActivationThreshold( + parser, (unsigned long long)value); + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + return XML_PROP_ERROR_INVALID_TYPE; + default: + return XML_PROP_ERROR_INVALID_KEY; } - parser->m_accounting.maximumAmplificationFactor = maximumAmplificationFactor; - return XML_TRUE; + + assert(0 && "considered unreachable"); } -XML_Bool XMLCALL -XML_SetBillionLaughsAttackProtectionActivationThreshold( - XML_Parser parser, unsigned long long activationThresholdBytes) { - if ((parser == NULL) || (parser->m_parentParser != NULL)) { - return XML_FALSE; +enum XML_Prop_Error XMLCALL +XML_GetPropertyBool(XML_Parser parser, enum XML_Parser_Property property, + XML_Bool *value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (value == NULL) + return XML_PROP_ERROR_INVALID_VALUE; + + switch (property) { +#if XML_GE == 1 + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: + return XML_PROP_ERROR_INVALID_TYPE; +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + *value = parser->m_reparseDeferralEnabled; + break; + default: + return XML_PROP_ERROR_INVALID_KEY; } - parser->m_accounting.activationThresholdBytes = activationThresholdBytes; - return XML_TRUE; + + return XML_PROP_ERROR_NONE; } + +enum XML_Prop_Error XMLCALL +XML_GetPropertyDouble(XML_Parser parser, enum XML_Parser_Property property, + double *value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (value == NULL) + return XML_PROP_ERROR_INVALID_VALUE; + + switch (property) { +#if XML_GE == 1 + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: + *value = (double)parser->m_accounting.maximumAmplificationFactor; + break; + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + *value = (double)parser->m_alloc_tracker.maximumAmplificationFactor; + break; + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: #endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + return XML_PROP_ERROR_INVALID_TYPE; + default: + return XML_PROP_ERROR_INVALID_KEY; + } -XML_Bool XMLCALL -XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled) { - if (parser != NULL && (enabled == XML_TRUE || enabled == XML_FALSE)) { - parser->m_reparseDeferralEnabled = enabled; - return XML_TRUE; + return XML_PROP_ERROR_NONE; +} + +enum XML_Prop_Error XMLCALL +XML_GetPropertyUInt64(XML_Parser parser, enum XML_Parser_Property property, + uint64_t *value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (value == NULL) + return XML_PROP_ERROR_INVALID_VALUE; + + switch (property) { +#if XML_GE == 1 + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: + *value = parser->m_accounting.activationThresholdBytes; + break; + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + *value = parser->m_alloc_tracker.activationThresholdBytes; + break; + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + return XML_PROP_ERROR_INVALID_TYPE; + default: + return XML_PROP_ERROR_INVALID_KEY; } - return XML_FALSE; + + return XML_PROP_ERROR_NONE; } /* Initially tag->rawName always points into the parse buffer; @@ -2655,10 +3395,10 @@ static XML_Bool storeRawNames(XML_Parser parser) { TAG *tag = parser->m_tagStack; while (tag) { - int bufSize; - int nameLen = sizeof(XML_Char) * (tag->name.strLen + 1); + size_t bufSize; + size_t nameLen = sizeof(XML_Char) * (tag->name.strLen + 1); size_t rawNameLen; - char *rawNameBuf = tag->buf + nameLen; + char *rawNameBuf = tag->buf.raw + nameLen; /* Stop if already stored. Since m_tagStack is a stack, we can stop at the first entry that has already been copied; everything below it in the stack is already been accounted for in a @@ -2671,25 +3411,25 @@ storeRawNames(XML_Parser parser) { */ rawNameLen = ROUND_UP(tag->rawNameLength, sizeof(XML_Char)); /* Detect and prevent integer overflow. */ - if (rawNameLen > (size_t)INT_MAX - nameLen) + if (rawNameLen > SIZE_MAX - nameLen) return XML_FALSE; - bufSize = nameLen + (int)rawNameLen; - if (bufSize > tag->bufEnd - tag->buf) { - char *temp = (char *)REALLOC(parser, tag->buf, bufSize); + bufSize = nameLen + rawNameLen; + if (bufSize > (size_t)(tag->bufEnd - tag->buf.raw)) { + char *temp = REALLOC(parser, tag->buf.raw, bufSize); if (temp == NULL) return XML_FALSE; - /* if tag->name.str points to tag->buf (only when namespace + /* if tag->name.str points to tag->buf.str (only when namespace processing is off) then we have to update it */ - if (tag->name.str == (XML_Char *)tag->buf) + if (tag->name.str == tag->buf.str) tag->name.str = (XML_Char *)temp; /* if tag->name.localPart is set (when namespace processing is on) then update it as well, since it will always point into tag->buf */ if (tag->name.localPart) tag->name.localPart - = (XML_Char *)temp + (tag->name.localPart - (XML_Char *)tag->buf); - tag->buf = temp; + = (XML_Char *)temp + (tag->name.localPart - tag->buf.str); + tag->buf.raw = temp; tag->bufEnd = temp + bufSize; rawNameBuf = temp + nameLen; } @@ -2700,12 +3440,13 @@ storeRawNames(XML_Parser parser) { return XML_TRUE; } -static enum XML_Error PTRCALL +static enum XML_Error contentProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { enum XML_Error result = doContent( - parser, 0, parser->m_encoding, start, end, endPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_ACCOUNT_DIRECT); + parser, parser->m_parentParser ? 1 : 0, parser->m_encoding, start, end, + endPtr, (XML_Bool)! parser->m_parsingStatus.finalBuffer, + XML_ACCOUNT_DIRECT); if (result == XML_ERROR_NONE) { if (! storeRawNames(parser)) return XML_ERROR_NO_MEMORY; @@ -2713,7 +3454,7 @@ contentProcessor(XML_Parser parser, const char *start, const char *end, return result; } -static enum XML_Error PTRCALL +static enum XML_Error externalEntityInitProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { enum XML_Error result = initializeEncoding(parser); @@ -2723,7 +3464,7 @@ externalEntityInitProcessor(XML_Parser parser, const char *start, return externalEntityInitProcessor2(parser, start, end, endPtr); } -static enum XML_Error PTRCALL +static enum XML_Error externalEntityInitProcessor2(XML_Parser parser, const char *start, const char *end, const char **endPtr) { const char *next = start; /* XmlContentTok doesn't always set the last arg */ @@ -2768,7 +3509,7 @@ externalEntityInitProcessor2(XML_Parser parser, const char *start, return externalEntityInitProcessor3(parser, start, end, endPtr); } -static enum XML_Error PTRCALL +static enum XML_Error externalEntityInitProcessor3(XML_Parser parser, const char *start, const char *end, const char **endPtr) { int tok; @@ -2793,6 +3534,11 @@ externalEntityInitProcessor3(XML_Parser parser, const char *start, return XML_ERROR_NONE; case XML_FINISHED: return XML_ERROR_ABORTED; + case XML_PARSING: + if (parser->m_reenter) { + return XML_ERROR_UNEXPECTED_STATE; // LCOV_EXCL_LINE + } + EXPAT_FALLTHROUGH; default: start = next; } @@ -2815,7 +3561,7 @@ externalEntityInitProcessor3(XML_Parser parser, const char *start, return externalEntityContentProcessor(parser, start, end, endPtr); } -static enum XML_Error PTRCALL +static enum XML_Error externalEntityContentProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { enum XML_Error result @@ -2871,7 +3617,9 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, *eventEndPP = end; if (parser->m_characterDataHandler) { XML_Char c = 0xA; + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, &c, 1); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, end); /* We are at the end of the final buffer, should we check for @@ -2924,9 +3672,11 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, ((char *)&ch) + sizeof(XML_Char), __LINE__, XML_ACCOUNT_ENTITY_EXPANSION); #endif /* XML_GE == 1 */ - if (parser->m_characterDataHandler) + if (parser->m_characterDataHandler) { + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, &ch, 1); - else if (parser->m_defaultHandler) + afterHandler(parser); + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; } @@ -2946,9 +3696,11 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, else if (! entity->is_internal) return XML_ERROR_ENTITY_DECLARED_IN_PE; } else if (! entity) { - if (parser->m_skippedEntityHandler) + if (parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0); - else if (parser->m_defaultHandler) + afterHandler(parser); + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; } @@ -2959,26 +3711,31 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, if (entity->textPtr) { enum XML_Error result; if (! parser->m_defaultExpandInternalEntities) { - if (parser->m_skippedEntityHandler) + if (parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, entity->name, 0); - else if (parser->m_defaultHandler) + afterHandler(parser); + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; } - result = processInternalEntity(parser, entity, XML_FALSE); + result = processEntity(parser, entity, XML_FALSE, ENTITY_INTERNAL); if (result != XML_ERROR_NONE) return result; } else if (parser->m_externalEntityRefHandler) { const XML_Char *context; - entity->open = XML_TRUE; + entity->open = true; context = getContext(parser); - entity->open = XML_FALSE; + entity->open = false; if (! context) return XML_ERROR_NO_MEMORY; - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, context, entity->base, - entity->systemId, entity->publicId)) + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, context, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) return XML_ERROR_EXTERNAL_ENTITY_HANDLING; poolDiscard(&parser->m_tempPool); } else if (parser->m_defaultHandler) @@ -2986,7 +3743,6 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, break; } case XML_TOK_START_TAG_NO_ATTS: - /* fall through */ case XML_TOK_START_TAG_WITH_ATTS: { TAG *tag; enum XML_Error result; @@ -2995,15 +3751,15 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, tag = parser->m_freeTagList; parser->m_freeTagList = parser->m_freeTagList->parent; } else { - tag = (TAG *)MALLOC(parser, sizeof(TAG)); + tag = MALLOC(parser, sizeof(TAG)); if (! tag) return XML_ERROR_NO_MEMORY; - tag->buf = (char *)MALLOC(parser, INIT_TAG_BUF_SIZE); - if (! tag->buf) { + tag->buf.raw = MALLOC(parser, INIT_TAG_BUF_SIZE); + if (! tag->buf.raw) { FREE(parser, tag); return XML_ERROR_NO_MEMORY; } - tag->bufEnd = tag->buf + INIT_TAG_BUF_SIZE; + tag->bufEnd = tag->buf.raw + INIT_TAG_BUF_SIZE; } tag->bindings = NULL; tag->parent = parser->m_tagStack; @@ -3016,46 +3772,47 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, { const char *rawNameEnd = tag->rawName + tag->rawNameLength; const char *fromPtr = tag->rawName; - toPtr = (XML_Char *)tag->buf; + toPtr = tag->buf.str; for (;;) { - int bufSize; - int convLen; const enum XML_Convert_Result convert_res = XmlConvert(enc, &fromPtr, rawNameEnd, (ICHAR **)&toPtr, (ICHAR *)tag->bufEnd - 1); - convLen = (int)(toPtr - (XML_Char *)tag->buf); + const size_t convLen = (size_t)(toPtr - tag->buf.str); if ((fromPtr >= rawNameEnd) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) { tag->name.strLen = convLen; break; } - bufSize = (int)(tag->bufEnd - tag->buf) << 1; + if (SIZE_MAX / 2 < (size_t)(tag->bufEnd - tag->buf.raw)) + return XML_ERROR_NO_MEMORY; + const size_t bufSize = (size_t)(tag->bufEnd - tag->buf.raw) * 2; { - char *temp = (char *)REALLOC(parser, tag->buf, bufSize); + char *temp = REALLOC(parser, tag->buf.raw, bufSize); if (temp == NULL) return XML_ERROR_NO_MEMORY; - tag->buf = temp; + tag->buf.raw = temp; tag->bufEnd = temp + bufSize; toPtr = (XML_Char *)temp + convLen; } } } - tag->name.str = (XML_Char *)tag->buf; + tag->name.str = tag->buf.str; *toPtr = XML_T('\0'); result = storeAtts(parser, enc, s, &(tag->name), &(tag->bindings), account); if (result) return result; - if (parser->m_startElementHandler) + if (parser->m_startElementHandler) { + beforeHandler(parser); parser->m_startElementHandler(parser->m_handlerArg, tag->name.str, (const XML_Char **)parser->m_atts); - else if (parser->m_defaultHandler) + afterHandler(parser); + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); poolClear(&parser->m_tempPool); break; } case XML_TOK_EMPTY_ELEMENT_NO_ATTS: - /* fall through */ case XML_TOK_EMPTY_ELEMENT_WITH_ATTS: { const char *rawName = s + enc->minBytesPerChar; enum XML_Error result; @@ -3075,14 +3832,18 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, } poolFinish(&parser->m_tempPool); if (parser->m_startElementHandler) { + beforeHandler(parser); parser->m_startElementHandler(parser->m_handlerArg, name.str, (const XML_Char **)parser->m_atts); + afterHandler(parser); noElmHandlers = XML_FALSE; } if (parser->m_endElementHandler) { if (parser->m_startElementHandler) *eventPP = *eventEndPP; + beforeHandler(parser); parser->m_endElementHandler(parser->m_handlerArg, name.str); + afterHandler(parser); noElmHandlers = XML_FALSE; } if (noElmHandlers && parser->m_defaultHandler) @@ -3092,7 +3853,9 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, } if ((parser->m_tagLevel == 0) && (parser->m_parsingStatus.parsing != XML_FINISHED)) { - if (parser->m_parsingStatus.parsing == XML_SUSPENDED) + if (parser->m_parsingStatus.parsing == XML_SUSPENDED + || (parser->m_parsingStatus.parsing == XML_PARSING + && parser->m_reenter)) parser->m_processor = epilogProcessor; else return epilogProcessor(parser, next, end, nextPtr); @@ -3138,14 +3901,19 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, } *uri = XML_T('\0'); } + beforeHandler(parser); parser->m_endElementHandler(parser->m_handlerArg, tag->name.str); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); while (tag->bindings) { BINDING *b = tag->bindings; - if (parser->m_endNamespaceDeclHandler) + if (parser->m_endNamespaceDeclHandler) { + beforeHandler(parser); parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name); + afterHandler(parser); + } tag->bindings = tag->bindings->nextTagBinding; b->nextTagBinding = parser->m_freeBindingList; parser->m_freeBindingList = b; @@ -3153,7 +3921,9 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, } if ((parser->m_tagLevel == 0) && (parser->m_parsingStatus.parsing != XML_FINISHED)) { - if (parser->m_parsingStatus.parsing == XML_SUSPENDED) + if (parser->m_parsingStatus.parsing == XML_SUSPENDED + || (parser->m_parsingStatus.parsing == XML_PARSING + && parser->m_reenter)) parser->m_processor = epilogProcessor; else return epilogProcessor(parser, next, end, nextPtr); @@ -3166,8 +3936,10 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, return XML_ERROR_BAD_CHAR_REF; if (parser->m_characterDataHandler) { XML_Char buf[XML_ENCODE_MAX]; + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, buf, XmlEncode(n, (ICHAR *)buf)); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); } break; @@ -3176,32 +3948,38 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, case XML_TOK_DATA_NEWLINE: if (parser->m_characterDataHandler) { XML_Char c = 0xA; + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, &c, 1); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; case XML_TOK_CDATA_SECT_OPEN: { enum XML_Error result; - if (parser->m_startCdataSectionHandler) + if (parser->m_startCdataSectionHandler) { + beforeHandler(parser); parser->m_startCdataSectionHandler(parser->m_handlerArg); - /* BEGIN disabled code */ - /* Suppose you doing a transformation on a document that involves - changing only the character data. You set up a defaultHandler - and a characterDataHandler. The defaultHandler simply copies - characters through. The characterDataHandler does the - transformation and writes the characters out escaping them as - necessary. This case will fail to work if we leave out the - following two lines (because & and < inside CDATA sections will - be incorrectly escaped). - - However, now we have a start/endCdataSectionHandler, so it seems - easier to let the user deal with this. - */ - else if ((0) && parser->m_characterDataHandler) + afterHandler(parser); + /* BEGIN disabled code */ + /* Suppose you doing a transformation on a document that involves + changing only the character data. You set up a defaultHandler + and a characterDataHandler. The defaultHandler simply copies + characters through. The characterDataHandler does the + transformation and writes the characters out escaping them as + necessary. This case will fail to work if we leave out the + following two lines (because & and < inside CDATA sections will + be incorrectly escaped). + + However, now we have a start/endCdataSectionHandler, so it seems + easier to let the user deal with this. + */ + } else if ((0) && parser->m_characterDataHandler) { + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0); - /* END disabled code */ - else if (parser->m_defaultHandler) + afterHandler(parser); + /* END disabled code */ + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); result = doCdataSection(parser, enc, &next, end, nextPtr, haveMore, account); @@ -3221,13 +3999,18 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, if (MUST_CONVERT(enc, s)) { ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf; XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd); + beforeHandler(parser); parser->m_characterDataHandler( parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); - } else + afterHandler(parser); + } else { + beforeHandler(parser); parser->m_characterDataHandler( parser->m_handlerArg, (const XML_Char *)s, (int)((const XML_Char *)end - (const XML_Char *)s)); + afterHandler(parser); + } } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, end); /* We are at the end of the final buffer, should we check for @@ -3252,16 +4035,21 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, const enum XML_Convert_Result convert_res = XmlConvert( enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd); *eventEndPP = s; + beforeHandler(parser); charDataHandler(parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); + afterHandler(parser); if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; *eventPP = s; } - } else + } else { + beforeHandler(parser); charDataHandler(parser->m_handlerArg, (const XML_Char *)s, (int)((const XML_Char *)next - (const XML_Char *)s)); + afterHandler(parser); + } } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); } break; @@ -3286,14 +4074,22 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, break; /* LCOV_EXCL_STOP */ } - *eventPP = s = next; switch (parser->m_parsingStatus.parsing) { case XML_SUSPENDED: + *eventPP = next; *nextPtr = next; return XML_ERROR_NONE; case XML_FINISHED: + *eventPP = next; return XML_ERROR_ABORTED; + case XML_PARSING: + if (parser->m_reenter) { + *nextPtr = next; + return XML_ERROR_NONE; + } + EXPAT_FALLTHROUGH; default:; + *eventPP = s = next; } } /* not reached */ @@ -3311,8 +4107,11 @@ freeBindings(XML_Parser parser, BINDING *bindings) { /* m_startNamespaceDeclHandler will have been called for this * binding in addBindings(), so call the end handler now. */ - if (parser->m_endNamespaceDeclHandler) + if (parser->m_endNamespaceDeclHandler) { + beforeHandler(parser); parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name); + afterHandler(parser); + } bindings = bindings->nextTagBinding; b->nextTagBinding = parser->m_freeBindingList; @@ -3336,20 +4135,14 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, TAG_NAME *tagNamePtr, BINDING **bindingsPtr, enum XML_Account account) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ - ELEMENT_TYPE *elementType; - int nDefaultAtts; - const XML_Char **appAtts; /* the attribute list for the application */ int attIndex = 0; - int prefixLen; - int i; - int n; XML_Char *uri; int nPrefixes = 0; BINDING *binding; const XML_Char *localPart; /* lookup the element type name */ - elementType + ELEMENT_TYPE *elementType = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, tagNamePtr->str, 0); if (! elementType) { const XML_Char *name = poolCopyString(&dtd->pool, tagNamePtr->str); @@ -3359,98 +4152,68 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sizeof(ELEMENT_TYPE)); if (! elementType) return XML_ERROR_NO_MEMORY; + if (! elementType->defaultAttForName.parser) + hashTableInit(&(elementType->defaultAttForName), parser); if (parser->m_ns && ! setElementTypePrefix(parser, elementType)) return XML_ERROR_NO_MEMORY; } - nDefaultAtts = elementType->nDefaultAtts; + const size_t nDefaultAtts = elementType->nDefaultAtts; + + /* Detect and prevent integer overflow. */ + if (parser->m_attsSize > (size_t)INT_MAX) + return XML_ERROR_NO_MEMORY; /* get the attributes from the tokenizer */ - n = XmlGetAttributes(enc, attStr, parser->m_attsSize, parser->m_atts); + size_t n = (size_t)XmlGetAttributes(enc, attStr, (int)parser->m_attsSize, + parser->m_atts); /* Detect and prevent integer overflow */ - if (n > INT_MAX - nDefaultAtts) { + if (n > SIZE_MAX - nDefaultAtts) { return XML_ERROR_NO_MEMORY; } if (n + nDefaultAtts > parser->m_attsSize) { - int oldAttsSize = parser->m_attsSize; - ATTRIBUTE *temp; -#ifdef XML_ATTR_INFO - XML_AttrInfo *temp2; -#endif + size_t oldAttsSize = parser->m_attsSize; /* Detect and prevent integer overflow */ - if ((nDefaultAtts > INT_MAX - INIT_ATTS_SIZE) - || (n > INT_MAX - (nDefaultAtts + INIT_ATTS_SIZE))) { + if ((nDefaultAtts > SIZE_MAX - INIT_ATTS_SIZE) + || (n > SIZE_MAX - (nDefaultAtts + INIT_ATTS_SIZE))) { return XML_ERROR_NO_MEMORY; } parser->m_attsSize = n + nDefaultAtts + INIT_ATTS_SIZE; - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)parser->m_attsSize > (size_t)(-1) / sizeof(ATTRIBUTE)) { + /* Detect and prevent integer overflow. */ + if (parser->m_attsSize > SIZE_MAX / sizeof(ATTRIBUTE)) { parser->m_attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; } -#endif - temp = (ATTRIBUTE *)REALLOC(parser, (void *)parser->m_atts, - parser->m_attsSize * sizeof(ATTRIBUTE)); + ATTRIBUTE *const temp = REALLOC(parser, parser->m_atts, + parser->m_attsSize * sizeof(ATTRIBUTE)); if (temp == NULL) { parser->m_attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; } parser->m_atts = temp; -#ifdef XML_ATTR_INFO - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -# if UINT_MAX >= SIZE_MAX - if ((unsigned)parser->m_attsSize > (size_t)(-1) / sizeof(XML_AttrInfo)) { - parser->m_attsSize = oldAttsSize; - return XML_ERROR_NO_MEMORY; - } -# endif - - temp2 = (XML_AttrInfo *)REALLOC(parser, (void *)parser->m_attInfo, - parser->m_attsSize * sizeof(XML_AttrInfo)); - if (temp2 == NULL) { - parser->m_attsSize = oldAttsSize; - return XML_ERROR_NO_MEMORY; + if (n > oldAttsSize) { + /* Detect and prevent integer overflow. */ + if (n > (size_t)INT_MAX) + return XML_ERROR_NO_MEMORY; + XmlGetAttributes(enc, attStr, (int)n, parser->m_atts); } - parser->m_attInfo = temp2; -#endif - if (n > oldAttsSize) - XmlGetAttributes(enc, attStr, n, parser->m_atts); } - appAtts = (const XML_Char **)parser->m_atts; - for (i = 0; i < n; i++) { + /* the attribute list for the application */ + const XML_Char **const appAtts = (const XML_Char **)parser->m_atts; + for (size_t i = 0; i < n; i++) { ATTRIBUTE *currAtt = &parser->m_atts[i]; -#ifdef XML_ATTR_INFO - XML_AttrInfo *currAttInfo = &parser->m_attInfo[i]; -#endif /* add the name and value to the attribute list */ ATTRIBUTE_ID *attId = getAttributeId(parser, enc, currAtt->name, currAtt->name + XmlNameLength(enc, currAtt->name)); if (! attId) return XML_ERROR_NO_MEMORY; -#ifdef XML_ATTR_INFO - currAttInfo->nameStart - = parser->m_parseEndByteIndex - (parser->m_parseEndPtr - currAtt->name); - currAttInfo->nameEnd - = currAttInfo->nameStart + XmlNameLength(enc, currAtt->name); - currAttInfo->valueStart = parser->m_parseEndByteIndex - - (parser->m_parseEndPtr - currAtt->valuePtr); - currAttInfo->valueEnd = parser->m_parseEndByteIndex - - (parser->m_parseEndPtr - currAtt->valueEnd); -#endif /* Detect duplicate attributes by their QNames. This does not work when namespace processing is turned on and different prefixes for the same namespace are used. For this case we have a check further down. @@ -3463,22 +4226,23 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, (attId->name)[-1] = 1; appAtts[attIndex++] = attId->name; if (! parser->m_atts[i].normalized) { - enum XML_Error result; XML_Bool isCdata = XML_TRUE; /* figure out whether declared as other than CDATA */ if (attId->maybeTokenized) { - int j; - for (j = 0; j < nDefaultAtts; j++) { - if (attId == elementType->defaultAtts[j].id) { - isCdata = elementType->defaultAtts[j].isCdata; - break; - } + NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute + = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup( + parser, &(elementType->defaultAttForName), attId->name, 0); + if (nameAndDefaultAttribute != NULL) { + assert(nameAndDefaultAttribute->attIndex < elementType->nDefaultAtts); + const DEFAULT_ATTRIBUTE *const att + = elementType->defaultAtts + nameAndDefaultAttribute->attIndex; + isCdata = att->isCdata; } } /* normalize the attribute value */ - result = storeAttributeValue( + const enum XML_Error result = storeAttributeValue( parser, enc, isCdata, parser->m_atts[i].valuePtr, parser->m_atts[i].valueEnd, &parser->m_tempPool, account); if (result) @@ -3516,7 +4280,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* set-up for XML_GetSpecifiedAttributeCount and XML_GetIdAttributeIndex */ parser->m_nSpecifiedAtts = attIndex; if (elementType->idAtt && (elementType->idAtt->name)[-1]) { - for (i = 0; i < attIndex; i += 2) + for (int i = 0; i < attIndex; i += 2) if (appAtts[i] == elementType->idAtt->name) { parser->m_idAttIndex = i; break; @@ -3525,7 +4289,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, parser->m_idAttIndex = -1; /* do attribute defaulting */ - for (i = 0; i < nDefaultAtts; i++) { + for (size_t i = 0; i < nDefaultAtts; i++) { const DEFAULT_ATTRIBUTE *da = elementType->defaultAtts + i; if (! (da->id->name)[-1] && da->value) { if (da->id->prefix) { @@ -3551,9 +4315,9 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* expand prefixed attribute names, check for duplicates, and clear flags that say whether attributes were specified */ - i = 0; + int i = 0; if (nPrefixes) { - int j; /* hash table index */ + unsigned int j; /* hash table index */ unsigned long version = parser->m_nsAttsVersion; /* Detect and prevent invalid shift */ @@ -3564,9 +4328,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, unsigned int nsAttsSize = 1u << parser->m_nsAttsPower; unsigned char oldNsAttsPower = parser->m_nsAttsPower; /* size of hash table must be at least 2 * (# of prefixed attributes) */ - if ((nPrefixes << 1) - >> parser->m_nsAttsPower) { /* true for m_nsAttsPower = 0 */ - NS_ATT *temp; + if (parser->m_nsAttsPower == 0 + || (nPrefixes >> (parser->m_nsAttsPower - 1))) { /* hash table size must also be a power of 2 and >= 8 */ while (nPrefixes >> parser->m_nsAttsPower++) ; @@ -3587,15 +4350,15 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, * from -Wtype-limits on platforms where * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ #if UINT_MAX >= SIZE_MAX - if (nsAttsSize > (size_t)(-1) / sizeof(NS_ATT)) { + if (nsAttsSize > SIZE_MAX / sizeof(NS_ATT)) { /* Restore actual size of memory in m_nsAtts */ parser->m_nsAttsPower = oldNsAttsPower; return XML_ERROR_NO_MEMORY; } #endif - temp = (NS_ATT *)REALLOC(parser, parser->m_nsAtts, - nsAttsSize * sizeof(NS_ATT)); + NS_ATT *const temp + = REALLOC(parser, parser->m_nsAtts, nsAttsSize * sizeof(NS_ATT)); if (! temp) { /* Restore actual size of memory in m_nsAtts */ parser->m_nsAttsPower = oldNsAttsPower; @@ -3616,9 +4379,6 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, for (; i < attIndex; i += 2) { const XML_Char *s = appAtts[i]; if (s[-1] == 2) { /* prefixed */ - ATTRIBUTE_ID *id; - const BINDING *b; - unsigned long uriHash; struct siphash sip_state; struct sipkey sip_key; @@ -3626,7 +4386,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sip24_init(&sip_state, &sip_key); ((XML_Char *)s)[-1] = 0; /* clear flag */ - id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0); + ATTRIBUTE_ID *const id + = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0); if (! id || ! id->prefix) { /* This code is walking through the appAtts array, dealing * with (in this case) a prefixed attribute name. To be in @@ -3634,7 +4395,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, * has to have passed through the hash table lookup once * already. That implies that an entry for it already * exists, so the lookup above will return a pointer to - * already allocated memory. There is no opportunaity for + * already allocated memory. There is no opportunity for * the allocator to fail, so the condition above cannot be * fulfilled. * @@ -3644,15 +4405,12 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, */ return XML_ERROR_NO_MEMORY; /* LCOV_EXCL_LINE */ } - b = id->prefix->binding; + const BINDING *const b = id->prefix->binding; if (! b) return XML_ERROR_UNBOUND_PREFIX; - for (j = 0; j < b->uriLen; j++) { - const XML_Char c = b->uri[j]; - if (! poolAppendChar(&parser->m_tempPool, c)) - return XML_ERROR_NO_MEMORY; - } + if (! poolAppendChars(&parser->m_tempPool, b->uri, b->uriLen)) + return XML_ERROR_NO_MEMORY; sip24_update(&sip_state, b->uri, b->uriLen * sizeof(XML_Char)); @@ -3661,12 +4419,13 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sip24_update(&sip_state, s, keylen(s) * sizeof(XML_Char)); - do { /* copies null terminator */ - if (! poolAppendChar(&parser->m_tempPool, *s)) + { + const size_t len = xcslen(s) + /*null terminator*/ 1; + if (! poolAppendChars(&parser->m_tempPool, s, len)) return XML_ERROR_NO_MEMORY; - } while (*s++); + } - uriHash = (unsigned long)sip24_final(&sip_state); + const unsigned long uriHash = (unsigned long)sip24_final(&sip_state); { /* Check hash table for duplicate of expanded name (uriName). Derived from code in lookup(parser, HASH_TABLE *table, ...). @@ -3694,10 +4453,9 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, if (parser->m_ns_triplets) { /* append namespace separator and prefix */ parser->m_tempPool.ptr[-1] = parser->m_namespaceSeparator; s = b->prefix->name; - do { - if (! poolAppendChar(&parser->m_tempPool, *s)) - return XML_ERROR_NO_MEMORY; - } while (*s++); + const size_t len = xcslen(s) + /*null terminator*/ 1; + if (! poolAppendChars(&parser->m_tempPool, s, len)) + return XML_ERROR_NO_MEMORY; } /* store expanded name in attribute list */ @@ -3740,48 +4498,36 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, localPart = tagNamePtr->str; } else return XML_ERROR_NONE; - prefixLen = 0; - if (parser->m_ns_triplets && binding->prefix->name) { - for (; binding->prefix->name[prefixLen++];) - ; /* prefixLen includes null terminator */ - } + size_t prefixLen = 0; + if (parser->m_ns_triplets && binding->prefix->name) + prefixLen = xcslen(binding->prefix->name) + /*null terminator*/ 1; tagNamePtr->localPart = localPart; tagNamePtr->uriLen = binding->uriLen; tagNamePtr->prefix = binding->prefix->name; tagNamePtr->prefixLen = prefixLen; - for (i = 0; localPart[i++];) - ; /* i includes null terminator */ + + const size_t localPartLen = xcslen(localPart) + /*null terminator*/ 1; /* Detect and prevent integer overflow */ - if (binding->uriLen > INT_MAX - prefixLen - || i > INT_MAX - (binding->uriLen + prefixLen)) { + if (binding->uriLen > SIZE_MAX - prefixLen + || localPartLen > SIZE_MAX - (binding->uriLen + prefixLen)) { return XML_ERROR_NO_MEMORY; } - n = i + binding->uriLen + prefixLen; - if (n > binding->uriAlloc) { - TAG *p; - + const size_t totalLen = localPartLen + binding->uriLen + prefixLen; + if (totalLen > binding->uriAlloc) { /* Detect and prevent integer overflow */ - if (n > INT_MAX - EXPAND_SPARE) { - return XML_ERROR_NO_MEMORY; - } - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)(n + EXPAND_SPARE) > (size_t)(-1) / sizeof(XML_Char)) { + if (totalLen > SIZE_MAX - EXPAND_SPARE + || totalLen + EXPAND_SPARE > SIZE_MAX / sizeof(XML_Char)) { return XML_ERROR_NO_MEMORY; } -#endif - uri = (XML_Char *)MALLOC(parser, (n + EXPAND_SPARE) * sizeof(XML_Char)); + uri = MALLOC(parser, (totalLen + EXPAND_SPARE) * sizeof(XML_Char)); if (! uri) return XML_ERROR_NO_MEMORY; - binding->uriAlloc = n + EXPAND_SPARE; + binding->uriAlloc = totalLen + EXPAND_SPARE; memcpy(uri, binding->uri, binding->uriLen * sizeof(XML_Char)); - for (p = parser->m_tagStack; p; p = p->parent) + for (TAG *p = parser->m_tagStack; p; p = p->parent) if (p->name.str == binding->uri) p->name.str = uri; FREE(parser, binding->uri); @@ -3789,10 +4535,14 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, } /* if m_namespaceSeparator != '\0' then uri includes it already */ uri = binding->uri + binding->uriLen; - memcpy(uri, localPart, i * sizeof(XML_Char)); + /* Detect and prevent integer overflow */ + if (localPartLen > SIZE_MAX / sizeof(XML_Char)) { + return XML_ERROR_NO_MEMORY; + } + memcpy(uri, localPart, localPartLen * sizeof(XML_Char)); /* we always have a namespace separator between localPart and prefix */ if (prefixLen) { - uri += i - 1; + uri += localPartLen - 1; *uri = parser->m_namespaceSeparator; /* replace null terminator */ memcpy(uri + 1, binding->prefix->name, prefixLen * sizeof(XML_Char)); } @@ -3927,7 +4677,7 @@ addBinding(XML_Parser parser, PPREFIX *prefix, const ATTRIBUTE_ID *attId, ASCII_8, ASCII_SLASH, ASCII_n, ASCII_a, ASCII_m, ASCII_e, ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e, '\0'}; - static const int xmlLen = (int)sizeof(xmlNamespace) / sizeof(XML_Char) - 1; + static const size_t xmlLen = sizeof(xmlNamespace) / sizeof(XML_Char) - 1; // "http://www.w3.org/2000/xmlns/" static const XML_Char xmlnsNamespace[] = {ASCII_h, ASCII_t, ASCII_t, ASCII_p, ASCII_COLON, ASCII_SLASH, @@ -3935,15 +4685,14 @@ addBinding(XML_Parser parser, PPREFIX *prefix, const ATTRIBUTE_ID *attId, ASCII_3, ASCII_PERIOD, ASCII_o, ASCII_r, ASCII_g, ASCII_SLASH, ASCII_2, ASCII_0, ASCII_0, ASCII_0, ASCII_SLASH, ASCII_x, ASCII_m, ASCII_l, ASCII_n, ASCII_s, ASCII_SLASH, '\0'}; - static const int xmlnsLen - = (int)sizeof(xmlnsNamespace) / sizeof(XML_Char) - 1; + static const size_t xmlnsLen = sizeof(xmlnsNamespace) / sizeof(XML_Char) - 1; XML_Bool mustBeXML = XML_FALSE; XML_Bool isXML = XML_TRUE; XML_Bool isXMLNS = XML_TRUE; BINDING *b; - int len; + size_t len; /* empty URI is only valid for default namespace per XML NS 1.0 (not 1.1) */ if (*uri == XML_T('\0') && prefix->name) @@ -3962,6 +4711,10 @@ addBinding(XML_Parser parser, PPREFIX *prefix, const ATTRIBUTE_ID *attId, } for (len = 0; uri[len]; len++) { + /* Detect and prevent integer overflow */ + if (len == SIZE_MAX) { + return XML_ERROR_NO_MEMORY; + } if (isXML && (len > xmlLen || uri[len] != xmlNamespace[len])) isXML = XML_FALSE; @@ -4002,28 +4755,24 @@ addBinding(XML_Parser parser, PPREFIX *prefix, const ATTRIBUTE_ID *attId, if (isXMLNS) return XML_ERROR_RESERVED_NAMESPACE_URI; - if (parser->m_namespaceSeparator) + if (parser->m_namespaceSeparator) { + /* Detect and prevent integer overflow */ + if (len == SIZE_MAX) { + return XML_ERROR_NO_MEMORY; + } len++; + } if (parser->m_freeBindingList) { b = parser->m_freeBindingList; if (len > b->uriAlloc) { /* Detect and prevent integer overflow */ - if (len > INT_MAX - EXPAND_SPARE) { - return XML_ERROR_NO_MEMORY; - } - - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)(len + EXPAND_SPARE) > (size_t)(-1) / sizeof(XML_Char)) { + if (len > SIZE_MAX - EXPAND_SPARE + || len + EXPAND_SPARE > SIZE_MAX / sizeof(XML_Char)) { return XML_ERROR_NO_MEMORY; } -#endif - XML_Char *temp = (XML_Char *)REALLOC( - parser, b->uri, sizeof(XML_Char) * (len + EXPAND_SPARE)); + XML_Char *temp + = REALLOC(parser, b->uri, sizeof(XML_Char) * (len + EXPAND_SPARE)); if (temp == NULL) return XML_ERROR_NO_MEMORY; b->uri = temp; @@ -4031,26 +4780,18 @@ addBinding(XML_Parser parser, PPREFIX *prefix, const ATTRIBUTE_ID *attId, } parser->m_freeBindingList = b->nextTagBinding; } else { - b = (BINDING *)MALLOC(parser, sizeof(BINDING)); + b = MALLOC(parser, sizeof(BINDING)); if (! b) return XML_ERROR_NO_MEMORY; /* Detect and prevent integer overflow */ - if (len > INT_MAX - EXPAND_SPARE) { - return XML_ERROR_NO_MEMORY; - } - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)(len + EXPAND_SPARE) > (size_t)(-1) / sizeof(XML_Char)) { + if (len > SIZE_MAX - EXPAND_SPARE + || len + EXPAND_SPARE > SIZE_MAX / sizeof(XML_Char)) { + FREE(parser, b); return XML_ERROR_NO_MEMORY; } -#endif - b->uri - = (XML_Char *)MALLOC(parser, sizeof(XML_Char) * (len + EXPAND_SPARE)); + b->uri = MALLOC(parser, sizeof(XML_Char) * (len + EXPAND_SPARE)); if (! b->uri) { FREE(parser, b); return XML_ERROR_NO_MEMORY; @@ -4072,16 +4813,19 @@ addBinding(XML_Parser parser, PPREFIX *prefix, const ATTRIBUTE_ID *attId, b->nextTagBinding = *bindingsPtr; *bindingsPtr = b; /* if attId == NULL then we are not starting a namespace scope */ - if (attId && parser->m_startNamespaceDeclHandler) + if (attId && parser->m_startNamespaceDeclHandler) { + beforeHandler(parser); parser->m_startNamespaceDeclHandler(parser->m_handlerArg, prefix->name, prefix->binding ? uri : 0); + afterHandler(parser); + } return XML_ERROR_NONE; } /* The idea here is to avoid using stack for each CDATA section when the whole file is parsed with one call. */ -static enum XML_Error PTRCALL +static enum XML_Error cdataSectionProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { enum XML_Error result = doCdataSection( @@ -4136,15 +4880,20 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, *eventEndPP = next; switch (tok) { case XML_TOK_CDATA_SECT_CLOSE: - if (parser->m_endCdataSectionHandler) + if (parser->m_endCdataSectionHandler) { + beforeHandler(parser); parser->m_endCdataSectionHandler(parser->m_handlerArg); + afterHandler(parser); + } /* BEGIN disabled code */ /* see comment under XML_TOK_CDATA_SECT_OPEN */ - else if ((0) && parser->m_characterDataHandler) + else if ((0) && parser->m_characterDataHandler) { + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0); - /* END disabled code */ - else if (parser->m_defaultHandler) + afterHandler(parser); + /* END disabled code */ + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); *startPtr = next; *nextPtr = next; @@ -4155,7 +4904,9 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, case XML_TOK_DATA_NEWLINE: if (parser->m_characterDataHandler) { XML_Char c = 0xA; + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, &c, 1); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; @@ -4168,16 +4919,21 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, const enum XML_Convert_Result convert_res = XmlConvert( enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd); *eventEndPP = next; + beforeHandler(parser); charDataHandler(parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); + afterHandler(parser); if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; *eventPP = s; } - } else + } else { + beforeHandler(parser); charDataHandler(parser->m_handlerArg, (const XML_Char *)s, (int)((const XML_Char *)next - (const XML_Char *)s)); + afterHandler(parser); + } } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); } break; @@ -4210,14 +4966,21 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, /* LCOV_EXCL_STOP */ } - *eventPP = s = next; switch (parser->m_parsingStatus.parsing) { case XML_SUSPENDED: + *eventPP = next; *nextPtr = next; return XML_ERROR_NONE; case XML_FINISHED: + *eventPP = next; return XML_ERROR_ABORTED; + case XML_PARSING: + if (parser->m_reenter) { + return XML_ERROR_UNEXPECTED_STATE; // LCOV_EXCL_LINE + } + EXPAT_FALLTHROUGH; default:; + *eventPP = s = next; } } /* not reached */ @@ -4228,7 +4991,7 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, /* The idea here is to avoid using stack for each IGNORE section when the whole file is parsed with one call. */ -static enum XML_Error PTRCALL +static enum XML_Error ignoreSectionProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { enum XML_Error result @@ -4409,8 +5172,10 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity, const char *s, if (! storedversion) return XML_ERROR_NO_MEMORY; } + beforeHandler(parser); parser->m_xmlDeclHandler(parser->m_handlerArg, storedversion, storedEncName, standalone); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, parser->m_encoding, s, next); if (parser->m_protocolEncodingName == NULL) { @@ -4460,31 +5225,43 @@ handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName) { info.convert = NULL; info.data = NULL; info.release = NULL; - if (parser->m_unknownEncodingHandler(parser->m_unknownEncodingHandlerData, - encodingName, &info)) { + beforeHandler(parser); + const int status = parser->m_unknownEncodingHandler( + parser->m_unknownEncodingHandlerData, encodingName, &info); + afterHandler(parser); + + parser->m_unknownEncodingRelease = info.release; + parser->m_unknownEncodingData = info.data; + + if (status) { ENCODING *enc; parser->m_unknownEncodingMem = MALLOC(parser, XmlSizeOfUnknownEncoding()); if (! parser->m_unknownEncodingMem) { - if (info.release) - info.release(info.data); + if (parser->m_unknownEncodingRelease) + callUnknownEncodingRelease(parser); + else + parser->m_unknownEncodingData = NULL; return XML_ERROR_NO_MEMORY; } + parser->m_unknownEncodingConvert = info.convert; enc = (parser->m_ns ? XmlInitUnknownEncodingNS : XmlInitUnknownEncoding)( - parser->m_unknownEncodingMem, info.map, info.convert, info.data); + parser->m_unknownEncodingMem, info.map, + info.convert ? callUnknownEncodingConvert : NULL, parser); if (enc) { - parser->m_unknownEncodingData = info.data; - parser->m_unknownEncodingRelease = info.release; parser->m_encoding = enc; return XML_ERROR_NONE; } + parser->m_unknownEncodingConvert = NULL; } - if (info.release != NULL) - info.release(info.data); + if (parser->m_unknownEncodingRelease != NULL) + callUnknownEncodingRelease(parser); + else + parser->m_unknownEncodingData = NULL; } return XML_ERROR_UNKNOWN_ENCODING; } -static enum XML_Error PTRCALL +static enum XML_Error prologInitProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { enum XML_Error result = initializeEncoding(parser); @@ -4496,7 +5273,7 @@ prologInitProcessor(XML_Parser parser, const char *s, const char *end, #ifdef XML_DTD -static enum XML_Error PTRCALL +static enum XML_Error externalParEntInitProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { enum XML_Error result = initializeEncoding(parser); @@ -4505,7 +5282,7 @@ externalParEntInitProcessor(XML_Parser parser, const char *s, const char *end, /* we know now that XML_Parse(Buffer) has been called, so we consider the external parameter entity read */ - parser->m_dtd->paramEntityRead = XML_TRUE; + parser->m_dtd->paramEntityRead = true; if (parser->m_prologState.inEntityValue) { parser->m_processor = entityValueInitProcessor; @@ -4516,7 +5293,7 @@ externalParEntInitProcessor(XML_Parser parser, const char *s, const char *end, } } -static enum XML_Error PTRCALL +static enum XML_Error entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { int tok; @@ -4549,7 +5326,7 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, } /* found end of entity value - can store it now */ return storeEntityValue(parser, parser->m_encoding, s, end, - XML_ACCOUNT_DIRECT); + XML_ACCOUNT_DIRECT, NULL); } else if (tok == XML_TOK_XML_DECL) { enum XML_Error result; result = processXmlDecl(parser, 0, start, next); @@ -4589,7 +5366,7 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, } /* If we get this token, we have the start of what might be a normal tag, but not a declaration (i.e. it doesn't begin with - "m_startDoctypeDeclHandler) { + beforeHandler(parser); parser->m_startDoctypeDeclHandler( parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid, parser->m_doctypePubid, 1); + afterHandler(parser); parser->m_doctypeName = NULL; poolClear(&parser->m_tempPool); handleDefault = XML_FALSE; @@ -4870,7 +5658,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, handleDefault = XML_FALSE; goto alreadyChecked; } - /* fall through */ + EXPAT_FALLTHROUGH; case XML_ROLE_ENTITY_PUBLIC_ID: if (! XmlIsPublicId(enc, s, next, eventPP)) return XML_ERROR_PUBLICID; @@ -4898,9 +5686,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } if (parser->m_doctypeName) { + beforeHandler(parser); parser->m_startDoctypeDeclHandler( parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid, parser->m_doctypePubid, 0); + afterHandler(parser); poolClear(&parser->m_tempPool); handleDefault = XML_FALSE; } @@ -4926,15 +5716,23 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } if (parser->m_useForeignDTD) entity->base = parser->m_curBase; - dtd->paramEntityRead = XML_FALSE; - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, 0, entity->base, - entity->systemId, entity->publicId)) + dtd->paramEntityRead = false; + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, 0, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) return XML_ERROR_EXTERNAL_ENTITY_HANDLING; if (dtd->paramEntityRead) { - if (! dtd->standalone && parser->m_notStandaloneHandler - && ! parser->m_notStandaloneHandler(parser->m_handlerArg)) - return XML_ERROR_NOT_STANDALONE; + if (! dtd->standalone && parser->m_notStandaloneHandler) { + beforeHandler(parser); + const int handlerStatus + = parser->m_notStandaloneHandler(parser->m_handlerArg); + afterHandler(parser); + if (! handlerStatus) + return XML_ERROR_NOT_STANDALONE; + } } /* if we didn't read the foreign DTD then this means that there is no external subset and we must reset dtd->hasParamEntityRefs @@ -4947,7 +5745,9 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } #endif /* XML_DTD */ if (parser->m_endDoctypeDeclHandler) { + beforeHandler(parser); parser->m_endDoctypeDeclHandler(parser->m_handlerArg); + afterHandler(parser); handleDefault = XML_FALSE; } break; @@ -4966,15 +5766,23 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (! entity) return XML_ERROR_NO_MEMORY; entity->base = parser->m_curBase; - dtd->paramEntityRead = XML_FALSE; - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, 0, entity->base, - entity->systemId, entity->publicId)) + dtd->paramEntityRead = false; + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, 0, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) return XML_ERROR_EXTERNAL_ENTITY_HANDLING; if (dtd->paramEntityRead) { - if (! dtd->standalone && parser->m_notStandaloneHandler - && ! parser->m_notStandaloneHandler(parser->m_handlerArg)) - return XML_ERROR_NOT_STANDALONE; + if (! dtd->standalone && parser->m_notStandaloneHandler) { + beforeHandler(parser); + const int handlerStatus + = parser->m_notStandaloneHandler(parser->m_handlerArg); + afterHandler(parser); + if (! handlerStatus) + return XML_ERROR_NOT_STANDALONE; + } } /* if we didn't read the foreign DTD then this means that there is no external subset and we must reset dtd->hasParamEntityRefs @@ -5067,10 +5875,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, poolFinish(&parser->m_tempPool); } *eventEndPP = s; + beforeHandler(parser); parser->m_attlistDeclHandler( parser->m_handlerArg, parser->m_declElementType->name, parser->m_declAttributeId->name, parser->m_declAttributeType, 0, role == XML_ROLE_REQUIRED_ATTRIBUTE_VALUE); + afterHandler(parser); handleDefault = XML_FALSE; } } @@ -5105,10 +5915,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, poolFinish(&parser->m_tempPool); } *eventEndPP = s; + beforeHandler(parser); parser->m_attlistDeclHandler( parser->m_handlerArg, parser->m_declElementType->name, parser->m_declAttributeId->name, parser->m_declAttributeType, attVal, role == XML_ROLE_FIXED_ATTRIBUTE_VALUE); + afterHandler(parser); poolClear(&parser->m_tempPool); handleDefault = XML_FALSE; } @@ -5119,20 +5931,26 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, #if XML_GE == 1 // This will store the given replacement text in // parser->m_declEntity->textPtr. - enum XML_Error result - = storeEntityValue(parser, enc, s + enc->minBytesPerChar, - next - enc->minBytesPerChar, XML_ACCOUNT_NONE); + enum XML_Error result = callStoreEntityValue( + parser, enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar, + XML_ACCOUNT_NONE); if (parser->m_declEntity) { + /* Detect and prevent signed integer overflow */ + if ((size_t)poolLength(&dtd->entityValuePool) > (size_t)INT_MAX) { + return XML_ERROR_NO_MEMORY; + } parser->m_declEntity->textPtr = poolStart(&dtd->entityValuePool); parser->m_declEntity->textLen = (int)(poolLength(&dtd->entityValuePool)); poolFinish(&dtd->entityValuePool); if (parser->m_entityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->is_param, parser->m_declEntity->textPtr, parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0); + afterHandler(parser); handleDefault = XML_FALSE; } } else @@ -5150,10 +5968,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (parser->m_entityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->is_param, parser->m_declEntity->textPtr, parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0); + afterHandler(parser); handleDefault = XML_FALSE; } } @@ -5184,9 +6004,13 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, #ifdef XML_DTD && ! parser->m_paramEntityParsing #endif /* XML_DTD */ - && parser->m_notStandaloneHandler - && ! parser->m_notStandaloneHandler(parser->m_handlerArg)) - return XML_ERROR_NOT_STANDALONE; + && parser->m_notStandaloneHandler) { + beforeHandler(parser); + const int status = parser->m_notStandaloneHandler(parser->m_handlerArg); + afterHandler(parser); + if (! status) + return XML_ERROR_NOT_STANDALONE; + } #ifndef XML_DTD break; #else /* XML_DTD */ @@ -5198,7 +6022,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, parser->m_declEntity->publicId = NULL; } #endif /* XML_DTD */ - /* fall through */ + EXPAT_FALLTHROUGH; case XML_ROLE_ENTITY_SYSTEM_ID: if (dtd->keepProcessing && parser->m_declEntity) { parser->m_declEntity->systemId @@ -5229,10 +6053,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (dtd->keepProcessing && parser->m_declEntity && parser->m_entityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->is_param, 0, 0, parser->m_declEntity->base, parser->m_declEntity->systemId, parser->m_declEntity->publicId, 0); + afterHandler(parser); handleDefault = XML_FALSE; } break; @@ -5245,17 +6071,21 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, poolFinish(&dtd->pool); if (parser->m_unparsedEntityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_unparsedEntityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->base, parser->m_declEntity->systemId, parser->m_declEntity->publicId, parser->m_declEntity->notation); + afterHandler(parser); handleDefault = XML_FALSE; } else if (parser->m_entityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, 0, 0, 0, parser->m_declEntity->base, parser->m_declEntity->systemId, parser->m_declEntity->publicId, parser->m_declEntity->notation); + afterHandler(parser); handleDefault = XML_FALSE; } } @@ -5279,7 +6109,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } else { poolFinish(&dtd->pool); parser->m_declEntity->publicId = NULL; - parser->m_declEntity->is_param = XML_FALSE; + parser->m_declEntity->is_param = false; /* if we have a parent parser or are reading an internal parameter entity, then the entity declaration is not considered "internal" */ @@ -5309,7 +6139,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } else { poolFinish(&dtd->pool); parser->m_declEntity->publicId = NULL; - parser->m_declEntity->is_param = XML_TRUE; + parser->m_declEntity->is_param = true; /* if we have a parent parser or are reading an internal parameter entity, then the entity declaration is not considered "internal" */ @@ -5362,9 +6192,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (! systemId) return XML_ERROR_NO_MEMORY; *eventEndPP = s; + beforeHandler(parser); parser->m_notationDeclHandler( parser->m_handlerArg, parser->m_declNotationName, parser->m_curBase, systemId, parser->m_declNotationPublicId); + afterHandler(parser); handleDefault = XML_FALSE; } poolClear(&parser->m_tempPool); @@ -5372,9 +6204,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, case XML_ROLE_NOTATION_NO_SYSTEM_ID: if (parser->m_declNotationPublicId && parser->m_notationDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_notationDeclHandler( parser->m_handlerArg, parser->m_declNotationName, parser->m_curBase, 0, parser->m_declNotationPublicId); + afterHandler(parser); handleDefault = XML_FALSE; } poolClear(&parser->m_tempPool); @@ -5408,41 +6242,20 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, case XML_ROLE_GROUP_OPEN: if (parser->m_prologState.level >= parser->m_groupSize) { if (parser->m_groupSize) { - { - /* Detect and prevent integer overflow */ - if (parser->m_groupSize > (unsigned int)(-1) / 2u) { - return XML_ERROR_NO_MEMORY; - } - - char *const new_connector = (char *)REALLOC( - parser, parser->m_groupConnector, parser->m_groupSize *= 2); - if (new_connector == NULL) { - parser->m_groupSize /= 2; - return XML_ERROR_NO_MEMORY; - } - parser->m_groupConnector = new_connector; + /* Detect and prevent integer overflow */ + if (parser->m_groupSize > SIZE_MAX / 2) { + return XML_ERROR_NO_MEMORY; } - if (dtd->scaffIndex) { - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if (parser->m_groupSize > (size_t)(-1) / sizeof(int)) { - return XML_ERROR_NO_MEMORY; - } -#endif - - int *const new_scaff_index = (int *)REALLOC( - parser, dtd->scaffIndex, parser->m_groupSize * sizeof(int)); - if (new_scaff_index == NULL) - return XML_ERROR_NO_MEMORY; - dtd->scaffIndex = new_scaff_index; + char *const new_connector = REALLOC(parser, parser->m_groupConnector, + parser->m_groupSize *= 2); + if (new_connector == NULL) { + parser->m_groupSize /= 2; + return XML_ERROR_NO_MEMORY; } + parser->m_groupConnector = new_connector; } else { - parser->m_groupConnector - = (char *)MALLOC(parser, parser->m_groupSize = 32); + parser->m_groupConnector = MALLOC(parser, parser->m_groupSize = 32); if (! parser->m_groupConnector) { parser->m_groupSize = 0; return XML_ERROR_NO_MEMORY; @@ -5455,6 +6268,21 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (myindex < 0) return XML_ERROR_NO_MEMORY; assert(dtd->scaffIndex != NULL); + if ((size_t)dtd->scaffLevel >= dtd->scaffIndexSize) { + /* Detect and prevent integer overflow */ + if (dtd->scaffIndexSize > SIZE_MAX / 2 / sizeof(int)) { + return XML_ERROR_NO_MEMORY; + } + assert(dtd->scaffIndexSize > 0); + const size_t new_size = dtd->scaffIndexSize * 2; + int *const new_scaff_index + = REALLOC(parser, dtd->scaffIndex, new_size * sizeof(int)); + if (new_scaff_index == NULL) { + return XML_ERROR_NO_MEMORY; + } + dtd->scaffIndex = new_scaff_index; + dtd->scaffIndexSize = new_size; + } dtd->scaffIndex[dtd->scaffLevel] = myindex; dtd->scaffLevel++; dtd->scaffold[myindex].type = XML_CTYPE_SEQ; @@ -5535,7 +6363,9 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, /* cannot report skipped entities in declarations */ if ((role == XML_ROLE_PARAM_ENTITY_REF) && parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, name, 1); + afterHandler(parser); handleDefault = XML_FALSE; } break; @@ -5544,27 +6374,29 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, return XML_ERROR_RECURSIVE_ENTITY_REF; if (entity->textPtr) { enum XML_Error result; - XML_Bool betweenDecl - = (role == XML_ROLE_PARAM_ENTITY_REF ? XML_TRUE : XML_FALSE); - result = processInternalEntity(parser, entity, betweenDecl); + bool betweenDecl = (role == XML_ROLE_PARAM_ENTITY_REF); + result = processEntity(parser, entity, betweenDecl, ENTITY_INTERNAL); if (result != XML_ERROR_NONE) return result; handleDefault = XML_FALSE; break; } if (parser->m_externalEntityRefHandler) { - dtd->paramEntityRead = XML_FALSE; - entity->open = XML_TRUE; + dtd->paramEntityRead = false; + entity->open = true; entityTrackingOnOpen(parser, entity, __LINE__); - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, 0, entity->base, - entity->systemId, entity->publicId)) { + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, 0, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) { entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; return XML_ERROR_EXTERNAL_ENTITY_HANDLING; } entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; handleDefault = XML_FALSE; if (! dtd->paramEntityRead) { dtd->keepProcessing = dtd->standalone; @@ -5576,9 +6408,13 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } } #endif /* XML_DTD */ - if (! dtd->standalone && parser->m_notStandaloneHandler - && ! parser->m_notStandaloneHandler(parser->m_handlerArg)) - return XML_ERROR_NOT_STANDALONE; + if (! dtd->standalone && parser->m_notStandaloneHandler) { + beforeHandler(parser); + const int status = parser->m_notStandaloneHandler(parser->m_handlerArg); + afterHandler(parser); + if (! status) + return XML_ERROR_NOT_STANDALONE; + } break; /* Element declaration stuff */ @@ -5590,7 +6426,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, return XML_ERROR_NO_MEMORY; dtd->scaffLevel = 0; dtd->scaffCount = 0; - dtd->in_eldecl = XML_TRUE; + dtd->in_eldecl = true; handleDefault = XML_FALSE; } break; @@ -5599,8 +6435,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, case XML_ROLE_CONTENT_EMPTY: if (dtd->in_eldecl) { if (parser->m_elementDeclHandler) { - XML_Content *content - = (XML_Content *)MALLOC(parser, sizeof(XML_Content)); + // NOTE: We are avoiding MALLOC(..) here to so that + // applications that are not using XML_FreeContentModel but + // plain free(..) or .free_fcn() to free the content model's + // memory are safe. + XML_Content *content = parser->m_mem.malloc_fcn(sizeof(XML_Content)); if (! content) return XML_ERROR_NO_MEMORY; content->quant = XML_CQUANT_NONE; @@ -5610,11 +6449,13 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, content->type = ((role == XML_ROLE_CONTENT_ANY) ? XML_CTYPE_ANY : XML_CTYPE_EMPTY); *eventEndPP = s; + beforeHandler(parser); parser->m_elementDeclHandler( parser->m_handlerArg, parser->m_declElementType->name, content); + afterHandler(parser); handleDefault = XML_FALSE; } - dtd->in_eldecl = XML_FALSE; + dtd->in_eldecl = false; } break; @@ -5655,9 +6496,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, return XML_ERROR_NO_MEMORY; name = el->name; dtd->scaffold[myindex].name = name; - nameLen = 0; - for (; name[nameLen++];) - ; + nameLen = xcslen(name) + /*null terminator*/ 1; /* Detect and prevent integer overflow */ if (nameLen > UINT_MAX - dtd->contentStringLen) { @@ -5693,10 +6532,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (! model) return XML_ERROR_NO_MEMORY; *eventEndPP = s; + beforeHandler(parser); parser->m_elementDeclHandler( parser->m_handlerArg, parser->m_declElementType->name, model); + afterHandler(parser); } - dtd->in_eldecl = XML_FALSE; + dtd->in_eldecl = false; dtd->contentStringLen = 0; } } @@ -5751,6 +6592,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, return XML_ERROR_NONE; case XML_FINISHED: return XML_ERROR_ABORTED; + case XML_PARSING: + if (parser->m_reenter) { + *nextPtr = next; + return XML_ERROR_NONE; + } + EXPAT_FALLTHROUGH; default: s = next; tok = XmlPrologTok(enc, s, end, &next); @@ -5759,7 +6606,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, /* not reached */ } -static enum XML_Error PTRCALL +static enum XML_Error epilogProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { parser->m_processor = epilogProcessor; @@ -5818,84 +6665,90 @@ epilogProcessor(XML_Parser parser, const char *s, const char *end, default: return XML_ERROR_JUNK_AFTER_DOC_ELEMENT; } - parser->m_eventPtr = s = next; switch (parser->m_parsingStatus.parsing) { case XML_SUSPENDED: + parser->m_eventPtr = next; *nextPtr = next; return XML_ERROR_NONE; case XML_FINISHED: + parser->m_eventPtr = next; return XML_ERROR_ABORTED; + case XML_PARSING: + if (parser->m_reenter) { + return XML_ERROR_UNEXPECTED_STATE; // LCOV_EXCL_LINE + } + EXPAT_FALLTHROUGH; default:; + parser->m_eventPtr = s = next; } } } static enum XML_Error -processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl) { - const char *textStart, *textEnd; - const char *next; - enum XML_Error result; - OPEN_INTERNAL_ENTITY *openEntity; +processEntity(XML_Parser parser, ENTITY *entity, bool betweenDecl, + enum EntityType type) { + OPEN_INTERNAL_ENTITY *openEntity, **openEntityList; + OPEN_INTERNAL_ENTITY **const freeEntityList = &parser->m_freeEntities; + switch (type) { + case ENTITY_INTERNAL: + parser->m_processor = internalEntityProcessor; + openEntityList = &parser->m_openInternalEntities; + break; + case ENTITY_ATTRIBUTE: + openEntityList = &parser->m_openAttributeEntities; + break; + case ENTITY_VALUE: + openEntityList = &parser->m_openValueEntities; + break; + /* default case serves merely as a safety net in case of a + * wrong entityType. Therefore we exclude the following lines + * from the test coverage. + * + * LCOV_EXCL_START + */ + default: + // Should not reach here + assert(0); + /* LCOV_EXCL_STOP */ + } - if (parser->m_freeInternalEntities) { - openEntity = parser->m_freeInternalEntities; - parser->m_freeInternalEntities = openEntity->next; + if (*freeEntityList) { + openEntity = *freeEntityList; + *freeEntityList = openEntity->next; } else { - openEntity - = (OPEN_INTERNAL_ENTITY *)MALLOC(parser, sizeof(OPEN_INTERNAL_ENTITY)); + openEntity = MALLOC(parser, sizeof(OPEN_INTERNAL_ENTITY)); if (! openEntity) return XML_ERROR_NO_MEMORY; } - entity->open = XML_TRUE; + entity->open = true; + entity->hasMore = true; #if XML_GE == 1 entityTrackingOnOpen(parser, entity, __LINE__); #endif entity->processed = 0; - openEntity->next = parser->m_openInternalEntities; - parser->m_openInternalEntities = openEntity; + openEntity->next = *openEntityList; + *openEntityList = openEntity; openEntity->entity = entity; + openEntity->type = type; openEntity->startTagLevel = parser->m_tagLevel; openEntity->betweenDecl = betweenDecl; openEntity->internalEventPtr = NULL; openEntity->internalEventEndPtr = NULL; - textStart = (const char *)entity->textPtr; - textEnd = (const char *)(entity->textPtr + entity->textLen); - /* Set a safe default value in case 'next' does not get set */ - next = textStart; - - if (entity->is_param) { - int tok - = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next); - result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, - tok, next, &next, XML_FALSE, XML_FALSE, - XML_ACCOUNT_ENTITY_EXPANSION); - } else { - result = doContent(parser, parser->m_tagLevel, parser->m_internalEncoding, - textStart, textEnd, &next, XML_FALSE, - XML_ACCOUNT_ENTITY_EXPANSION); - } - if (result == XML_ERROR_NONE) { - if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) { - entity->processed = (int)(next - textStart); - parser->m_processor = internalEntityProcessor; - } else if (parser->m_openInternalEntities->entity == entity) { -#if XML_GE == 1 - entityTrackingOnClose(parser, entity, __LINE__); -#endif /* XML_GE == 1 */ - entity->open = XML_FALSE; - parser->m_openInternalEntities = openEntity->next; - /* put openEntity back in list of free instances */ - openEntity->next = parser->m_freeInternalEntities; - parser->m_freeInternalEntities = openEntity; - } + // Only internal entities make use of the reenter flag + // therefore no need to set it for other entity types + if (type == ENTITY_INTERNAL) { + triggerReenter(parser); } - return result; + return XML_ERROR_NONE; } -static enum XML_Error PTRCALL +static enum XML_Error internalEntityProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { + UNUSED_P(s); + UNUSED_P(end); + UNUSED_P(nextPtr); ENTITY *entity; const char *textStart, *textEnd; const char *next; @@ -5905,71 +6758,74 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, return XML_ERROR_UNEXPECTED_STATE; entity = openEntity->entity; - textStart = ((const char *)entity->textPtr) + entity->processed; - textEnd = (const char *)(entity->textPtr + entity->textLen); - /* Set a safe default value in case 'next' does not get set */ - next = textStart; - - if (entity->is_param) { - int tok - = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next); - result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, - tok, next, &next, XML_FALSE, XML_TRUE, - XML_ACCOUNT_ENTITY_EXPANSION); - } else { - result = doContent(parser, openEntity->startTagLevel, - parser->m_internalEncoding, textStart, textEnd, &next, - XML_FALSE, XML_ACCOUNT_ENTITY_EXPANSION); - } - if (result != XML_ERROR_NONE) - return result; + // This will return early + if (entity->hasMore) { + textStart = ((const char *)entity->textPtr) + entity->processed; + textEnd = (const char *)(entity->textPtr + entity->textLen); + /* Set a safe default value in case 'next' does not get set */ + next = textStart; + + if (entity->is_param) { + int tok + = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next); + result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, + tok, next, &next, XML_FALSE, XML_FALSE, + XML_ACCOUNT_ENTITY_EXPANSION); + } else { + result = doContent(parser, openEntity->startTagLevel, + parser->m_internalEncoding, textStart, textEnd, &next, + XML_FALSE, XML_ACCOUNT_ENTITY_EXPANSION); + } + + if (result != XML_ERROR_NONE) + return result; + // Check if entity is complete, if not, mark down how much of it is + // processed + if (textEnd != next + && (parser->m_parsingStatus.parsing == XML_SUSPENDED + || (parser->m_parsingStatus.parsing == XML_PARSING + && parser->m_reenter))) { + entity->processed = (int)(next - (const char *)entity->textPtr); + return result; + } - if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) { - entity->processed = (int)(next - (const char *)entity->textPtr); + // Entity is complete. We cannot close it here since we need to first + // process its possible inner entities (which are added to the + // m_openInternalEntities during doProlog or doContent calls above) + entity->hasMore = false; + if (! entity->is_param + && (openEntity->startTagLevel != parser->m_tagLevel)) { + return XML_ERROR_ASYNC_ENTITY; + } + triggerReenter(parser); return result; - } + } // End of entity processing, "if" block will return here + // Remove fully processed openEntity from open entity list. #if XML_GE == 1 entityTrackingOnClose(parser, entity, __LINE__); #endif - entity->open = XML_FALSE; - parser->m_openInternalEntities = openEntity->next; + // openEntity is m_openInternalEntities' head, as we set it at the start of + // this function and we skipped doProlog and doContent calls with hasMore set + // to false. This means we can directly remove the head of + // m_openInternalEntities + assert(parser->m_openInternalEntities == openEntity); + entity->open = false; + parser->m_openInternalEntities = parser->m_openInternalEntities->next; + /* put openEntity back in list of free instances */ - openEntity->next = parser->m_freeInternalEntities; - parser->m_freeInternalEntities = openEntity; - - // If there are more open entities we want to stop right here and have the - // upcoming call to XML_ResumeParser continue with entity content, or it would - // be ignored altogether. - if (parser->m_openInternalEntities != NULL - && parser->m_parsingStatus.parsing == XML_SUSPENDED) { - return XML_ERROR_NONE; - } + openEntity->next = parser->m_freeEntities; + parser->m_freeEntities = openEntity; - if (entity->is_param) { - int tok; - parser->m_processor = prologProcessor; - tok = XmlPrologTok(parser->m_encoding, s, end, &next); - return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE, - XML_ACCOUNT_DIRECT); - } else { - parser->m_processor = contentProcessor; - /* see externalEntityContentProcessor vs contentProcessor */ - result = doContent(parser, parser->m_parentParser ? 1 : 0, - parser->m_encoding, s, end, nextPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer, - XML_ACCOUNT_DIRECT); - if (result == XML_ERROR_NONE) { - if (! storeRawNames(parser)) - return XML_ERROR_NO_MEMORY; - } - return result; + if (parser->m_openInternalEntities == NULL) { + parser->m_processor = entity->is_param ? prologProcessor : contentProcessor; } + triggerReenter(parser); + return XML_ERROR_NONE; } -static enum XML_Error PTRCALL +static enum XML_Error errorProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { UNUSED_P(s); @@ -5982,8 +6838,71 @@ static enum XML_Error storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, const char *ptr, const char *end, STRING_POOL *pool, enum XML_Account account) { - enum XML_Error result - = appendAttributeValue(parser, enc, isCdata, ptr, end, pool, account); + const char *next = ptr; + enum XML_Error result = XML_ERROR_NONE; + + while (1) { + if (! parser->m_openAttributeEntities) { + result = appendAttributeValue(parser, enc, isCdata, next, end, pool, + account, &next); + } else { + OPEN_INTERNAL_ENTITY *const openEntity = parser->m_openAttributeEntities; + if (! openEntity) + return XML_ERROR_UNEXPECTED_STATE; + + ENTITY *const entity = openEntity->entity; + const char *const textStart + = ((const char *)entity->textPtr) + entity->processed; + const char *const textEnd + = (const char *)(entity->textPtr + entity->textLen); + /* Set a safe default value in case 'next' does not get set */ + const char *nextInEntity = textStart; + if (entity->hasMore) { + result = appendAttributeValue( + parser, parser->m_internalEncoding, isCdata, textStart, textEnd, + pool, XML_ACCOUNT_ENTITY_EXPANSION, &nextInEntity); + if (result != XML_ERROR_NONE) + break; + // Check if entity is complete, if not, mark down how much of it is + // processed. A XML_SUSPENDED check here is not required as + // appendAttributeValue will never suspend the parser. + if (nextInEntity < textEnd) { + entity->processed + = (int)(nextInEntity - (const char *)entity->textPtr); + continue; + } + assert(nextInEntity == textEnd); + + // Entity is complete. We cannot close it here since we need to first + // process its possible inner entities (which are added to the + // m_openAttributeEntities during appendAttributeValue) + entity->hasMore = false; + continue; + } // End of entity processing, "if" block skips the rest + + // Remove fully processed openEntity from open entity list. +#if XML_GE == 1 + entityTrackingOnClose(parser, entity, __LINE__); +#endif + // openEntity is m_openAttributeEntities' head, since we set it at the + // start of this function and because we skipped appendAttributeValue call + // with hasMore set to false. This means we can directly remove the head + // of m_openAttributeEntities + assert(parser->m_openAttributeEntities == openEntity); + entity->open = false; + parser->m_openAttributeEntities = parser->m_openAttributeEntities->next; + + /* put openEntity back in list of free instances */ + openEntity->next = parser->m_freeEntities; + parser->m_freeEntities = openEntity; + } + + // Break if an error occurred or there is nothing left to process + if (result || (parser->m_openAttributeEntities == NULL && end == next)) { + break; + } + } + if (result) return result; if (! isCdata && poolLength(pool) && poolLastChar(pool) == 0x20) @@ -5996,7 +6915,7 @@ storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, static enum XML_Error appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, const char *ptr, const char *end, STRING_POOL *pool, - enum XML_Account account) { + enum XML_Account account, const char **nextPtr) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ #ifndef XML_DTD UNUSED_P(account); @@ -6014,6 +6933,9 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, #endif switch (tok) { case XML_TOK_NONE: + if (nextPtr) { + *nextPtr = next; + } return XML_ERROR_NONE; case XML_TOK_INVALID: if (enc == parser->m_encoding) @@ -6025,7 +6947,6 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, return XML_ERROR_INVALID_TOKEN; case XML_TOK_CHAR_REF: { XML_Char buf[XML_ENCODE_MAX]; - int i; int n = XmlCharRefNumber(enc, ptr); if (n < 0) { if (enc == parser->m_encoding) @@ -6045,10 +6966,9 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, * XmlEncode() is never passed a value it might return an * error for. */ - for (i = 0; i < n; i++) { - if (! poolAppendChar(pool, buf[i])) - return XML_ERROR_NO_MEMORY; - } + + if (! poolAppendChars(pool, buf, n)) + return XML_ERROR_NO_MEMORY; } break; case XML_TOK_DATA_CHARS: if (! poolAppend(pool, enc, ptr, next)) @@ -6056,7 +6976,7 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, break; case XML_TOK_TRAILING_CR: next = ptr + enc->minBytesPerChar; - /* fall through */ + EXPAT_FALLTHROUGH; case XML_TOK_ATTRIBUTE_VALUE_S: case XML_TOK_DATA_NEWLINE: if (! isCdata && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20)) @@ -6067,7 +6987,7 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, case XML_TOK_ENTITY_REF: { const XML_Char *name; ENTITY *entity; - char checkEntityDecl; + bool checkEntityDecl; XML_Char ch = (XML_Char)XmlPredefinedEntityName( enc, ptr + enc->minBytesPerChar, next - enc->minBytesPerChar); if (ch) { @@ -6110,8 +7030,11 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, } else if (! entity) { /* Cannot report skipped entity here - see comments on parser->m_skippedEntityHandler. - if (parser->m_skippedEntityHandler) + if (parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0); + afterHandler(parser); + } */ /* Cannot call the default handler because this would be out of sync with the call to the startElementHandler. @@ -6154,21 +7077,11 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, return XML_ERROR_ATTRIBUTE_EXTERNAL_ENTITY_REF; } else { enum XML_Error result; - const XML_Char *textEnd = entity->textPtr + entity->textLen; - entity->open = XML_TRUE; -#if XML_GE == 1 - entityTrackingOnOpen(parser, entity, __LINE__); -#endif - result = appendAttributeValue(parser, parser->m_internalEncoding, - isCdata, (const char *)entity->textPtr, - (const char *)textEnd, pool, - XML_ACCOUNT_ENTITY_EXPANSION); -#if XML_GE == 1 - entityTrackingOnClose(parser, entity, __LINE__); -#endif - entity->open = XML_FALSE; - if (result) - return result; + result = processEntity(parser, entity, XML_FALSE, ENTITY_ATTRIBUTE); + if ((result == XML_ERROR_NONE) && (nextPtr != NULL)) { + *nextPtr = next; + } + return result; } } break; default: @@ -6197,7 +7110,7 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, static enum XML_Error storeEntityValue(XML_Parser parser, const ENCODING *enc, const char *entityTextPtr, const char *entityTextEnd, - enum XML_Account account) { + enum XML_Account account, const char **nextPtr) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ STRING_POOL *pool = &(dtd->entityValuePool); enum XML_Error result = XML_ERROR_NONE; @@ -6215,8 +7128,16 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, return XML_ERROR_NO_MEMORY; } + const char *next = entityTextPtr; + + /* Nothing to tokenize. */ + if (entityTextPtr >= entityTextEnd) { + result = XML_ERROR_NONE; + goto endEntityValue; + } + for (;;) { - const char *next + next = entityTextPtr; /* XmlEntityValueTok doesn't always set the last arg */ int tok = XmlEntityValueTok(enc, entityTextPtr, entityTextEnd, &next); @@ -6246,8 +7167,11 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, /* not a well-formedness error - see XML 1.0: WFC Entity Declared */ /* cannot report skipped entity here - see comments on parser->m_skippedEntityHandler - if (parser->m_skippedEntityHandler) + if (parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0); + afterHandler(parser); + } */ dtd->keepProcessing = dtd->standalone; goto endEntityValue; @@ -6260,34 +7184,29 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, } if (entity->systemId) { if (parser->m_externalEntityRefHandler) { - dtd->paramEntityRead = XML_FALSE; - entity->open = XML_TRUE; + dtd->paramEntityRead = false; + entity->open = true; entityTrackingOnOpen(parser, entity, __LINE__); - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, 0, entity->base, - entity->systemId, entity->publicId)) { + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, 0, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) { entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; result = XML_ERROR_EXTERNAL_ENTITY_HANDLING; goto endEntityValue; } entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; if (! dtd->paramEntityRead) dtd->keepProcessing = dtd->standalone; } else dtd->keepProcessing = dtd->standalone; } else { - entity->open = XML_TRUE; - entityTrackingOnOpen(parser, entity, __LINE__); - result = storeEntityValue( - parser, parser->m_internalEncoding, (const char *)entity->textPtr, - (const char *)(entity->textPtr + entity->textLen), - XML_ACCOUNT_ENTITY_EXPANSION); - entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; - if (result) - goto endEntityValue; + result = processEntity(parser, entity, XML_FALSE, ENTITY_VALUE); + goto endEntityValue; } break; } @@ -6309,17 +7228,15 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, break; case XML_TOK_TRAILING_CR: next = entityTextPtr + enc->minBytesPerChar; - /* fall through */ + EXPAT_FALLTHROUGH; case XML_TOK_DATA_NEWLINE: - if (pool->end == pool->ptr && ! poolGrow(pool)) { + if (! poolAppendChar(pool, 0xA)) { result = XML_ERROR_NO_MEMORY; goto endEntityValue; } - *(pool->ptr)++ = 0xA; break; case XML_TOK_CHAR_REF: { XML_Char buf[XML_ENCODE_MAX]; - int i; int n = XmlCharRefNumber(enc, entityTextPtr); if (n < 0) { if (enc == parser->m_encoding) @@ -6337,12 +7254,9 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, * XmlEncode() is never passed a value it might return an * error for. */ - for (i = 0; i < n; i++) { - if (pool->end == pool->ptr && ! poolGrow(pool)) { - result = XML_ERROR_NO_MEMORY; - goto endEntityValue; - } - *(pool->ptr)++ = buf[i]; + if (! poolAppendChars(pool, buf, n)) { + result = XML_ERROR_NO_MEMORY; + goto endEntityValue; } } break; case XML_TOK_PARTIAL: @@ -6375,6 +7289,81 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, # ifdef XML_DTD parser->m_prologState.inEntityValue = oldInEntityValue; # endif /* XML_DTD */ + // If 'nextPtr' is given, it should be updated during the processing + if (nextPtr != NULL) { + *nextPtr = next; + } + return result; +} + +static enum XML_Error +callStoreEntityValue(XML_Parser parser, const ENCODING *enc, + const char *entityTextPtr, const char *entityTextEnd, + enum XML_Account account) { + const char *next = entityTextPtr; + enum XML_Error result = XML_ERROR_NONE; + while (1) { + if (! parser->m_openValueEntities) { + result + = storeEntityValue(parser, enc, next, entityTextEnd, account, &next); + } else { + OPEN_INTERNAL_ENTITY *const openEntity = parser->m_openValueEntities; + if (! openEntity) + return XML_ERROR_UNEXPECTED_STATE; + + ENTITY *const entity = openEntity->entity; + const char *const textStart + = ((const char *)entity->textPtr) + entity->processed; + const char *const textEnd + = (const char *)(entity->textPtr + entity->textLen); + /* Set a safe default value in case 'next' does not get set */ + const char *nextInEntity = textStart; + if (entity->hasMore) { + result = storeEntityValue(parser, parser->m_internalEncoding, textStart, + textEnd, XML_ACCOUNT_ENTITY_EXPANSION, + &nextInEntity); + if (result != XML_ERROR_NONE) + break; + // Check if entity is complete, if not, mark down how much of it is + // processed. A XML_SUSPENDED check here is not required as + // appendAttributeValue will never suspend the parser. + if (textEnd != nextInEntity) { + entity->processed + = (int)(nextInEntity - (const char *)entity->textPtr); + continue; + } + + // Entity is complete. We cannot close it here since we need to first + // process its possible inner entities (which are added to the + // m_openValueEntities during storeEntityValue) + entity->hasMore = false; + continue; + } // End of entity processing, "if" block skips the rest + + // Remove fully processed openEntity from open entity list. +# if XML_GE == 1 + entityTrackingOnClose(parser, entity, __LINE__); +# endif + // openEntity is m_openValueEntities' head, since we set it at the + // start of this function and because we skipped storeEntityValue call + // with hasMore set to false. This means we can directly remove the head + // of m_openValueEntities + assert(parser->m_openValueEntities == openEntity); + entity->open = false; + parser->m_openValueEntities = parser->m_openValueEntities->next; + + /* put openEntity back in list of free instances */ + openEntity->next = parser->m_freeEntities; + parser->m_freeEntities = openEntity; + } + + // Break if an error occurred or there is nothing left to process + if (result + || (parser->m_openValueEntities == NULL && entityTextEnd == next)) { + break; + } + } + return result; } @@ -6395,6 +7384,11 @@ storeSelfEntityValue(XML_Parser parser, ENTITY *entity) { return XML_ERROR_NO_MEMORY; } + /* Detect and prevent signed integer overflow */ + if ((size_t)poolLength(pool) > (size_t)INT_MAX) { + poolDiscard(pool); + return XML_ERROR_NO_MEMORY; + } entity->textPtr = poolStart(pool); entity->textLen = (int)(poolLength(pool)); poolFinish(pool); @@ -6404,7 +7398,7 @@ storeSelfEntityValue(XML_Parser parser, ENTITY *entity) { #endif /* XML_GE == 0 */ -static void FASTCALL +static void normalizeLines(XML_Char *s) { XML_Char *p; for (;; s++) { @@ -6447,7 +7441,9 @@ reportProcessingInstruction(XML_Parser parser, const ENCODING *enc, if (! data) return 0; normalizeLines(data); + beforeHandler(parser); parser->m_processingInstructionHandler(parser->m_handlerArg, target, data); + afterHandler(parser); poolClear(&parser->m_tempPool); return 1; } @@ -6467,7 +7463,9 @@ reportComment(XML_Parser parser, const ENCODING *enc, const char *start, if (! data) return 0; normalizeLines(data); + beforeHandler(parser); parser->m_commentHandler(parser->m_handlerArg, data); + afterHandler(parser); poolClear(&parser->m_tempPool); return 1; } @@ -6508,15 +7506,20 @@ reportDefault(XML_Parser parser, const ENCODING *enc, const char *s, convert_res = XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd); *eventEndPP = s; + beforeHandler(parser); parser->m_defaultHandler(parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); + afterHandler(parser); *eventPP = s; } while ((convert_res != XML_CONVERT_COMPLETED) && (convert_res != XML_CONVERT_INPUT_INCOMPLETE)); - } else + } else { + beforeHandler(parser); parser->m_defaultHandler( parser->m_handlerArg, (const XML_Char *)s, (int)((const XML_Char *)end - (const XML_Char *)s)); + afterHandler(parser); + } } static int @@ -6526,56 +7529,61 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, if (value || isId) { /* The handling of default attributes gets messed up if we have a default which duplicates a non-default. */ - int i; - for (i = 0; i < type->nDefaultAtts; i++) - if (attId == type->defaultAtts[i].id) - return 1; + NAMED *const nameFound + = lookup(parser, &(type->defaultAttForName), attId->name, 0); + if (nameFound) + return 1; if (isId && ! type->idAtt && ! attId->xmlns) type->idAtt = attId; } if (type->nDefaultAtts == type->allocDefaultAtts) { - if (type->allocDefaultAtts == 0) { - type->allocDefaultAtts = 8; - type->defaultAtts = (DEFAULT_ATTRIBUTE *)MALLOC( - parser, type->allocDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); - if (! type->defaultAtts) { - type->allocDefaultAtts = 0; - return 0; - } - } else { - DEFAULT_ATTRIBUTE *temp; - - /* Detect and prevent integer overflow */ - if (type->allocDefaultAtts > INT_MAX / 2) { - return 0; - } - - int count = type->allocDefaultAtts * 2; + /* Detect and prevent integer overflow */ + if (type->allocDefaultAtts > SIZE_MAX / 2) { + return 0; + } - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)count > (size_t)(-1) / sizeof(DEFAULT_ATTRIBUTE)) { - return 0; - } -#endif + size_t count = type->allocDefaultAtts * 2; + if (count == 0) { + count = 8; + } - temp = (DEFAULT_ATTRIBUTE *)REALLOC(parser, type->defaultAtts, - (count * sizeof(DEFAULT_ATTRIBUTE))); - if (temp == NULL) - return 0; - type->allocDefaultAtts = count; - type->defaultAtts = temp; + /* Detect and prevent integer overflow. */ + if (count > SIZE_MAX / sizeof(DEFAULT_ATTRIBUTE)) { + return 0; } + + DEFAULT_ATTRIBUTE *const temp = REALLOC( + parser, type->defaultAtts, (count * sizeof(DEFAULT_ATTRIBUTE))); + if (temp == NULL) + return 0; + type->allocDefaultAtts = count; + type->defaultAtts = temp; } att = type->defaultAtts + type->nDefaultAtts; att->id = attId; att->value = value; att->isCdata = isCdata; if (! isCdata) - attId->maybeTokenized = XML_TRUE; + attId->maybeTokenized = true; + + NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute + = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup( + parser, &(type->defaultAttForName), attId->name, + sizeof(NAME_AND_DEFAULT_ATTRIBUTE)); + if (! nameAndDefaultAttribute) + return 0; + + assert(nameAndDefaultAttribute->name == attId->name); + + // NOTE: The XML 1.0r4 spec says: + // "When more than one definition is provided for the same attribute of a + // given element type, the first declaration is binding and later + // declarations are ignored." + if (! nameAndDefaultAttribute->initialized) { + nameAndDefaultAttribute->attIndex = type->nDefaultAtts; + nameAndDefaultAttribute->initialized = true; + } + type->nDefaultAtts += 1; return 1; } @@ -6641,17 +7649,18 @@ getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start, else id->prefix = (PPREFIX *)lookup(parser, &dtd->prefixes, name + 6, sizeof(PPREFIX)); - id->xmlns = XML_TRUE; + id->xmlns = true; } else { int i; for (i = 0; name[i]; i++) { + /* Detect and prevent signed integer overflow */ + if (i == INT_MAX) { + return NULL; + } /* attributes without prefix are *not* in the default namespace */ if (name[i] == XML_T(ASCII_COLON)) { - int j; - for (j = 0; j < i; j++) { - if (! poolAppendChar(&dtd->pool, name[j])) - return NULL; - } + if (! poolAppendChars(&dtd->pool, name, i)) + return NULL; if (! poolAppendChar(&dtd->pool, XML_T('\0'))) return NULL; id->prefix = (PPREFIX *)lookup(parser, &dtd->prefixes, @@ -6679,46 +7688,39 @@ getContext(XML_Parser parser) { XML_Bool needSep = XML_FALSE; if (dtd->defaultPrefix.binding) { - int i; - int len; if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS))) return NULL; - len = dtd->defaultPrefix.binding->uriLen; + size_t len = dtd->defaultPrefix.binding->uriLen; if (parser->m_namespaceSeparator) len--; - for (i = 0; i < len; i++) { - if (! poolAppendChar(&parser->m_tempPool, - dtd->defaultPrefix.binding->uri[i])) { - /* Because of memory caching, I don't believe this line can be - * executed. - * - * This is part of a loop copying the default prefix binding - * URI into the parser's temporary string pool. Previously, - * that URI was copied into the same string pool, with a - * terminating NUL character, as part of setContext(). When - * the pool was cleared, that leaves a block definitely big - * enough to hold the URI on the free block list of the pool. - * The URI copy in getContext() therefore cannot run out of - * memory. - * - * If the pool is used between the setContext() and - * getContext() calls, the worst it can do is leave a bigger - * block on the front of the free list. Given that this is - * all somewhat inobvious and program logic can be changed, we - * don't delete the line but we do exclude it from the test - * coverage statistics. - */ - return NULL; /* LCOV_EXCL_LINE */ - } + if (! poolAppendChars(&parser->m_tempPool, dtd->defaultPrefix.binding->uri, + len)) { + /* Because of memory caching, I don't believe this line can be + * executed. + * + * This is part of a loop copying the default prefix binding + * URI into the parser's temporary string pool. Previously, + * that URI was copied into the same string pool, with a + * terminating NUL character, as part of setContext(). When + * the pool was cleared, that leaves a block definitely big + * enough to hold the URI on the free block list of the pool. + * The URI copy in getContext() therefore cannot run out of + * memory. + * + * If the pool is used between the setContext() and + * getContext() calls, the worst it can do is leave a bigger + * block on the front of the free list. Given that this is + * all somewhat inobvious and program logic can be changed, we + * don't delete the line but we do exclude it from the test + * coverage statistics. + */ + return NULL; /* LCOV_EXCL_LINE */ } needSep = XML_TRUE; } hashTableIterInit(&iter, &(dtd->prefixes)); for (;;) { - int i; - int len; - const XML_Char *s; PPREFIX *prefix = (PPREFIX *)hashTableIterNext(&iter); if (! prefix) break; @@ -6733,23 +7735,21 @@ getContext(XML_Parser parser) { } if (needSep && ! poolAppendChar(&parser->m_tempPool, CONTEXT_SEP)) return NULL; - for (s = prefix->name; *s; s++) - if (! poolAppendChar(&parser->m_tempPool, *s)) - return NULL; + if (! poolAppendChars(&parser->m_tempPool, prefix->name, + xcslen(prefix->name))) + return NULL; if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS))) return NULL; - len = prefix->binding->uriLen; + size_t len = prefix->binding->uriLen; if (parser->m_namespaceSeparator) len--; - for (i = 0; i < len; i++) - if (! poolAppendChar(&parser->m_tempPool, prefix->binding->uri[i])) - return NULL; + if (! poolAppendChars(&parser->m_tempPool, prefix->binding->uri, len)) + return NULL; needSep = XML_TRUE; } hashTableIterInit(&iter, &(dtd->generalEntities)); for (;;) { - const XML_Char *s; ENTITY *e = (ENTITY *)hashTableIterNext(&iter); if (! e) break; @@ -6757,9 +7757,8 @@ getContext(XML_Parser parser) { continue; if (needSep && ! poolAppendChar(&parser->m_tempPool, CONTEXT_SEP)) return NULL; - for (s = e->name; *s; s++) - if (! poolAppendChar(&parser->m_tempPool, *s)) - return 0; + if (! poolAppendChars(&parser->m_tempPool, e->name, xcslen(e->name))) + return NULL; needSep = XML_TRUE; } @@ -6785,7 +7784,7 @@ setContext(XML_Parser parser, const XML_Char *context) { e = (ENTITY *)lookup(parser, &dtd->generalEntities, poolStart(&parser->m_tempPool), 0); if (e) - e->open = XML_TRUE; + e->open = true; if (*s != XML_T('\0')) s++; context = s; @@ -6797,16 +7796,24 @@ setContext(XML_Parser parser, const XML_Char *context) { else { if (! poolAppendChar(&parser->m_tempPool, XML_T('\0'))) return XML_FALSE; - prefix - = (PPREFIX *)lookup(parser, &dtd->prefixes, - poolStart(&parser->m_tempPool), sizeof(PPREFIX)); - if (! prefix) + const XML_Char *const prefixName = poolCopyStringNoFinish( + &dtd->pool, poolStart(&parser->m_tempPool)); + if (! prefixName) { return XML_FALSE; - if (prefix->name == poolStart(&parser->m_tempPool)) { - prefix->name = poolCopyString(&dtd->pool, prefix->name); - if (! prefix->name) - return XML_FALSE; } + + prefix = (PPREFIX *)lookup(parser, &dtd->prefixes, prefixName, + sizeof(PPREFIX)); + + const bool prefixNameUsed = prefix && prefix->name == prefixName; + if (prefixNameUsed) + poolFinish(&dtd->pool); + else + poolDiscard(&dtd->pool); + + if (! prefix) + return XML_FALSE; + poolDiscard(&parser->m_tempPool); } for (context = s + 1; *context != CONTEXT_SEP && *context != XML_T('\0'); @@ -6832,7 +7839,7 @@ setContext(XML_Parser parser, const XML_Char *context) { return XML_TRUE; } -static void FASTCALL +static void normalizePublicId(XML_Char *publicId) { XML_Char *p = publicId; XML_Char *s; @@ -6854,25 +7861,26 @@ normalizePublicId(XML_Char *publicId) { } static DTD * -dtdCreate(const XML_Memory_Handling_Suite *ms) { - DTD *p = ms->malloc_fcn(sizeof(DTD)); +dtdCreate(XML_Parser parser) { + DTD *p = MALLOC(parser, sizeof(DTD)); if (p == NULL) return p; - poolInit(&(p->pool), ms); - poolInit(&(p->entityValuePool), ms); - hashTableInit(&(p->generalEntities), ms); - hashTableInit(&(p->elementTypes), ms); - hashTableInit(&(p->attributeIds), ms); - hashTableInit(&(p->prefixes), ms); + poolInit(&(p->pool), parser); + poolInit(&(p->entityValuePool), parser); + hashTableInit(&(p->generalEntities), parser); + hashTableInit(&(p->elementTypes), parser); + hashTableInit(&(p->attributeIds), parser); + hashTableInit(&(p->prefixes), parser); #ifdef XML_DTD - p->paramEntityRead = XML_FALSE; - hashTableInit(&(p->paramEntities), ms); + p->paramEntityRead = false; + hashTableInit(&(p->paramEntities), parser); #endif /* XML_DTD */ p->defaultPrefix.name = NULL; p->defaultPrefix.binding = NULL; - p->in_eldecl = XML_FALSE; + p->in_eldecl = false; p->scaffIndex = NULL; + p->scaffIndexSize = 0; p->scaffold = NULL; p->scaffLevel = 0; p->scaffSize = 0; @@ -6886,19 +7894,19 @@ dtdCreate(const XML_Memory_Handling_Suite *ms) { } static void -dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms) { +dtdReset(DTD *p, XML_Parser parser) { HASH_TABLE_ITER iter; hashTableIterInit(&iter, &(p->elementTypes)); for (;;) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; - if (e->allocDefaultAtts != 0) - ms->free_fcn(e->defaultAtts); + hashTableDestroy(&(e->defaultAttForName)); + FREE(parser, e->defaultAtts); } hashTableClear(&(p->generalEntities)); #ifdef XML_DTD - p->paramEntityRead = XML_FALSE; + p->paramEntityRead = false; hashTableClear(&(p->paramEntities)); #endif /* XML_DTD */ hashTableClear(&(p->elementTypes)); @@ -6909,11 +7917,12 @@ dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms) { p->defaultPrefix.name = NULL; p->defaultPrefix.binding = NULL; - p->in_eldecl = XML_FALSE; + p->in_eldecl = false; - ms->free_fcn(p->scaffIndex); + FREE(parser, p->scaffIndex); p->scaffIndex = NULL; - ms->free_fcn(p->scaffold); + p->scaffIndexSize = 0; + FREE(parser, p->scaffold); p->scaffold = NULL; p->scaffLevel = 0; @@ -6927,15 +7936,15 @@ dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms) { } static void -dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms) { +dtdDestroy(DTD *p, XML_Bool isDocEntity, XML_Parser parser) { HASH_TABLE_ITER iter; hashTableIterInit(&iter, &(p->elementTypes)); for (;;) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; - if (e->allocDefaultAtts != 0) - ms->free_fcn(e->defaultAtts); + hashTableDestroy(&(e->defaultAttForName)); + FREE(parser, e->defaultAtts); } hashTableDestroy(&(p->generalEntities)); #ifdef XML_DTD @@ -6947,10 +7956,10 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms) { poolDestroy(&(p->pool)); poolDestroy(&(p->entityValuePool)); if (isDocEntity) { - ms->free_fcn(p->scaffIndex); - ms->free_fcn(p->scaffold); + FREE(parser, p->scaffIndex); + FREE(parser, p->scaffold); } - ms->free_fcn(p); + FREE(parser, p); } /* Do a deep copy of the DTD. Return 0 for out of memory, non-zero otherwise. @@ -6958,7 +7967,7 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms) { */ static int dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, - const XML_Memory_Handling_Suite *ms) { + XML_Parser parser) { HASH_TABLE_ITER iter; /* Copy the prefix table. */ @@ -7014,7 +8023,6 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, hashTableIterInit(&iter, &(oldDtd->elementTypes)); for (;;) { - int i; ELEMENT_TYPE *newE; const XML_Char *name; const ELEMENT_TYPE *oldE = (ELEMENT_TYPE *)hashTableIterNext(&iter); @@ -7027,19 +8035,17 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, sizeof(ELEMENT_TYPE)); if (! newE) return 0; + + if (! newE->defaultAttForName.parser) + hashTableInit(&(newE->defaultAttForName), parser); + if (oldE->nDefaultAtts) { - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((size_t)oldE->nDefaultAtts - > ((size_t)(-1) / sizeof(DEFAULT_ATTRIBUTE))) { + /* Detect and prevent integer overflow. */ + if (oldE->nDefaultAtts > SIZE_MAX / sizeof(DEFAULT_ATTRIBUTE)) { return 0; } -#endif newE->defaultAtts - = ms->malloc_fcn(oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); + = MALLOC(parser, oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); if (! newE->defaultAtts) { return 0; } @@ -7051,9 +8057,10 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, if (oldE->prefix) newE->prefix = (PPREFIX *)lookup(oldParser, &(newDtd->prefixes), oldE->prefix->name, 0); - for (i = 0; i < newE->nDefaultAtts; i++) { + for (size_t i = 0; i < newE->nDefaultAtts; i++) { + const XML_Char *const attributeName = oldE->defaultAtts[i].id->name; newE->defaultAtts[i].id = (ATTRIBUTE_ID *)lookup( - oldParser, &(newDtd->attributeIds), oldE->defaultAtts[i].id->name, 0); + oldParser, &(newDtd->attributeIds), attributeName, 0); newE->defaultAtts[i].isCdata = oldE->defaultAtts[i].isCdata; if (oldE->defaultAtts[i].value) { newE->defaultAtts[i].value @@ -7062,6 +8069,23 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, return 0; } else newE->defaultAtts[i].value = NULL; + + NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute + = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup( + parser, &(newE->defaultAttForName), attributeName, + sizeof(NAME_AND_DEFAULT_ATTRIBUTE)); + if (! nameAndDefaultAttribute) { + return 0; + } + + // NOTE: The XML 1.0r4 spec says: + // "When more than one definition is provided for the same attribute of a + // given element type, the first declaration is binding and later + // declarations are ignored." + if (! nameAndDefaultAttribute->initialized) { + nameAndDefaultAttribute->attIndex = i; + nameAndDefaultAttribute->initialized = true; + } } } @@ -7088,6 +8112,7 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, newDtd->scaffSize = oldDtd->scaffSize; newDtd->scaffLevel = oldDtd->scaffLevel; newDtd->scaffIndex = oldDtd->scaffIndex; + newDtd->scaffIndexSize = oldDtd->scaffIndexSize; return 1; } /* End dtdCopy */ @@ -7157,41 +8182,72 @@ copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, #define INIT_POWER 6 -static XML_Bool FASTCALL -keyeq(KEY s1, KEY s2) { - for (; *s1 == *s2; s1++, s2++) - if (*s1 == 0) - return XML_TRUE; - return XML_FALSE; +// Compares two strings `s1` and `s2` whereas: +// - `s2` is zero-terminated but +// - `s1` is made up of exactly (not just up to) `s1len` non-zero characters. +XML_NONTESTING_STATIC XML_Bool +keyeq(KEY s1, size_t s1len, KEY s2) { +#ifdef XML_UNICODE +# ifdef XML_UNICODE_WCHAR_T + return (wcsncmp(s1, s2, s1len) == 0 && s2[s1len] == L'\0') ? XML_TRUE + : XML_FALSE; +# else + for (; s1len > 0 && *s1 == *s2; s1len--, s1++, s2++) + ; /* no loop body! */ + return ((s1len == 0) && (*s2 == 0)) ? XML_TRUE : XML_FALSE; +# endif +#else + return (strncmp(s1, s2, s1len) == 0 && s2[s1len] == '\0') ? XML_TRUE + : XML_FALSE; +#endif } -static size_t +XML_NONTESTING_STATIC size_t keylen(KEY s) { - size_t len = 0; - for (; *s; s++, len++) - ; - return len; + return xcslen(s); } static void copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key) { - key->k[0] = 0; - key->k[1] = get_hash_secret_salt(parser); + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(! rootParser->m_parentParser); + + *key = rootParser->m_hash_secret_salt_128; } -static unsigned long FASTCALL -hash(XML_Parser parser, KEY s) { +static unsigned long +hash(XML_Parser parser, KEY s, size_t keyLen) { struct siphash state; struct sipkey key; (void)sip24_valid; copy_salt_to_sipkey(parser, &key); sip24_init(&state, &key); - sip24_update(&state, s, keylen(s) * sizeof(XML_Char)); + sip24_update(&state, s, keyLen * sizeof(XML_Char)); return (unsigned long)sip24_final(&state); } -static NAMED * -lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { +// Function `lookupWithLength` can be used to either… +// +// a) check whether an element with key `name` exists in the given hash table +// (read-only mode where `createSize == 0`) or +// +// b) check whether an element with key `name` exists in the given hash table +// *and* insert it if missing (i.e. read-write mode where `createSize != 0`. +// +// When inserting, a block of `createSize` number of bytes will be allocated +// and set to zero, and the resulting block of memory will be considered +// to start with a `NAMED` structure, and `->name = name;` is performed. +// The fact that all other bytes in the structure are initially zero can +// be used to tell cases "existed and found" and "newly inserted" apart +// with the structure returned. +// +// NOTE: Read-only lookup does not need zero-terminated keys but +// read-write mode does, because keys can be re-hashed later and the +// hash table does not store key length information. +// +XML_NONTESTING_STATIC NAMED * +lookupWithLength(XML_Parser parser, HASH_TABLE *table, KEY name, size_t nameLen, + size_t createSize) { size_t i; if (table->size == 0) { size_t tsize; @@ -7201,20 +8257,20 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { /* table->size is a power of 2 */ table->size = (size_t)1 << INIT_POWER; tsize = table->size * sizeof(NAMED *); - table->v = table->mem->malloc_fcn(tsize); + table->v = MALLOC(table->parser, tsize); if (! table->v) { table->size = 0; return NULL; } memset(table->v, 0, tsize); - i = hash(parser, name) & ((unsigned long)table->size - 1); + i = hash(parser, name, nameLen) & ((unsigned long)table->size - 1); } else { - unsigned long h = hash(parser, name); + unsigned long h = hash(parser, name, nameLen); unsigned long mask = (unsigned long)table->size - 1; unsigned char step = 0; i = h & mask; while (table->v[i]) { - if (keyeq(name, table->v[i]->name)) + if (keyeq(name, nameLen, table->v[i]->name)) return table->v[i]; if (! step) step = PROBE_STEP(h, mask, table->power); @@ -7236,18 +8292,19 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { unsigned long newMask = (unsigned long)newSize - 1; /* Detect and prevent integer overflow */ - if (newSize > (size_t)(-1) / sizeof(NAMED *)) { + if (newSize > SIZE_MAX / sizeof(NAMED *)) { return NULL; } size_t tsize = newSize * sizeof(NAMED *); - NAMED **newV = table->mem->malloc_fcn(tsize); + NAMED **newV = MALLOC(table->parser, tsize); if (! newV) return NULL; memset(newV, 0, tsize); for (i = 0; i < table->size; i++) if (table->v[i]) { - unsigned long newHash = hash(parser, table->v[i]->name); + KEY const key = table->v[i]->name; + unsigned long newHash = hash(parser, key, keylen(key)); size_t j = newHash & newMask; step = 0; while (newV[j]) { @@ -7257,7 +8314,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { } newV[j] = table->v[i]; } - table->mem->free_fcn(table->v); + FREE(table->parser, table->v); table->v = newV; table->power = newPower; table->size = newSize; @@ -7270,49 +8327,70 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { } } } - table->v[i] = table->mem->malloc_fcn(createSize); + assert(createSize >= sizeof(NAMED)); + table->v[i] = MALLOC(table->parser, createSize); if (! table->v[i]) return NULL; memset(table->v[i], 0, createSize); - table->v[i]->name = name; + table->v[i]->name = name; // NOTE: This requires and assumes zero termination! (table->used)++; return table->v[i]; } -static void FASTCALL +// Function `lookup` can be used to either… +// +// a) check whether an element with key `name` exists in the given hash table +// (read-only mode where `createSize == 0`) or +// +// b) check whether an element with key `name` exists in the given hash table +// *and* insert it if missing (i.e. read-write mode where `createSize != 0`. +// +// When inserting, a block of `createSize` number of bytes will be allocated +// and set to zero, and the resulting block of memory will be considered +// to start with a `NAMED` structure, and `->name = name;` is performed. +// The fact that all other bytes in the structure are initially zero can +// be used to tell cases "existed and found" and "newly inserted" apart +// with the structure returned. +// +XML_NONTESTING_STATIC NAMED * +lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { + return lookupWithLength(parser, table, name, keylen(name), createSize); +} + +XML_NONTESTING_STATIC void hashTableClear(HASH_TABLE *table) { size_t i; for (i = 0; i < table->size; i++) { - table->mem->free_fcn(table->v[i]); + FREE(table->parser, table->v[i]); table->v[i] = NULL; } table->used = 0; } -static void FASTCALL +XML_NONTESTING_STATIC void hashTableDestroy(HASH_TABLE *table) { size_t i; for (i = 0; i < table->size; i++) - table->mem->free_fcn(table->v[i]); - table->mem->free_fcn(table->v); + FREE(table->parser, table->v[i]); + FREE(table->parser, table->v); } -static void FASTCALL -hashTableInit(HASH_TABLE *p, const XML_Memory_Handling_Suite *ms) { +XML_NONTESTING_STATIC void +hashTableInit(HASH_TABLE *p, XML_Parser parser) { p->power = 0; p->size = 0; p->used = 0; p->v = NULL; - p->mem = ms; + p->parser = parser; } -static void FASTCALL +XML_NONTESTING_STATIC void hashTableIterInit(HASH_TABLE_ITER *iter, const HASH_TABLE *table) { iter->p = table->v; iter->end = iter->p ? iter->p + table->size : NULL; } -static NAMED *FASTCALL +XML_NONTESTING_STATIC NAMED * hashTableIterNext(HASH_TABLE_ITER *iter) { while (iter->p != iter->end) { NAMED *tem = *(iter->p)++; @@ -7322,17 +8400,17 @@ hashTableIterNext(HASH_TABLE_ITER *iter) { return NULL; } -static void FASTCALL -poolInit(STRING_POOL *pool, const XML_Memory_Handling_Suite *ms) { +static void +poolInit(STRING_POOL *pool, XML_Parser parser) { pool->blocks = NULL; pool->freeBlocks = NULL; pool->start = NULL; pool->ptr = NULL; pool->end = NULL; - pool->mem = ms; + pool->parser = parser; } -static void FASTCALL +static void poolClear(STRING_POOL *pool) { if (! pool->freeBlocks) pool->freeBlocks = pool->blocks; @@ -7351,18 +8429,18 @@ poolClear(STRING_POOL *pool) { pool->end = NULL; } -static void FASTCALL +static void poolDestroy(STRING_POOL *pool) { BLOCK *p = pool->blocks; while (p) { BLOCK *tem = p->next; - pool->mem->free_fcn(p); + FREE(pool->parser, p); p = tem; } p = pool->freeBlocks; while (p) { BLOCK *tem = p->next; - pool->mem->free_fcn(p); + FREE(pool->parser, p); p = tem; } } @@ -7384,17 +8462,32 @@ poolAppend(STRING_POOL *pool, const ENCODING *enc, const char *ptr, return pool->start; } -static const XML_Char *FASTCALL +static const XML_Char * poolCopyString(STRING_POOL *pool, const XML_Char *s) { - do { - if (! poolAppendChar(pool, *s)) - return NULL; - } while (*s++); + if (! poolAppendChars(pool, s, xcslen(s) + /*null terminator*/ 1)) + return NULL; s = pool->start; poolFinish(pool); return s; } +// A version of `poolCopyString` that does not call `poolFinish` +// and reverts any partial advancement upon failure. +static const XML_Char * +poolCopyStringNoFinish(STRING_POOL *pool, const XML_Char *s) { + const XML_Char *const original = s; + do { + if (! poolAppendChar(pool, *s)) { + // Revert any previously successful advancement + const ptrdiff_t advancedBy = s - original; + if (advancedBy > 0) + pool->ptr -= advancedBy; + return NULL; + } + } while (*s++); + return pool->start; +} + static const XML_Char * poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) { if (! pool->ptr && ! poolGrow(pool)) { @@ -7411,22 +8504,17 @@ poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) { */ return NULL; /* LCOV_EXCL_LINE */ } - for (; n > 0; --n, s++) { - if (! poolAppendChar(pool, *s)) - return NULL; - } + if (n > 0 && ! poolAppendChars(pool, s, n)) + return NULL; s = pool->start; poolFinish(pool); return s; } -static const XML_Char *FASTCALL +static const XML_Char * poolAppendString(STRING_POOL *pool, const XML_Char *s) { - while (*s) { - if (! poolAppendChar(pool, *s)) - return NULL; - s++; - } + if (! poolAppendChars(pool, s, xcslen(s))) + return NULL; return pool->start; } @@ -7435,9 +8523,8 @@ poolStoreString(STRING_POOL *pool, const ENCODING *enc, const char *ptr, const char *end) { if (! poolAppend(pool, enc, ptr, end)) return NULL; - if (pool->ptr == pool->end && ! poolGrow(pool)) + if (! poolAppendChar(pool, 0)) return NULL; - *(pool->ptr)++ = 0; return pool->start; } @@ -7469,10 +8556,10 @@ poolBytesToAllocateFor(int blockSize) { } } -static XML_Bool FASTCALL +static XML_Bool poolGrow(STRING_POOL *pool) { if (pool->freeBlocks) { - if (pool->start == 0) { + if (pool->start == NULL) { pool->blocks = pool->freeBlocks; pool->freeBlocks = pool->freeBlocks->next; pool->blocks->next = NULL; @@ -7488,7 +8575,7 @@ poolGrow(STRING_POOL *pool) { pool->freeBlocks = tem; memcpy(pool->blocks->s, pool->start, (pool->end - pool->start) * sizeof(XML_Char)); - pool->ptr = pool->blocks->s + (pool->ptr - pool->start); + pool->ptr = pool->blocks->s + EXPAT_SAFE_PTR_DIFF(pool->ptr, pool->start); pool->start = pool->blocks->s; pool->end = pool->start + pool->blocks->size; return XML_TRUE; @@ -7501,7 +8588,8 @@ poolGrow(STRING_POOL *pool) { /* NOTE: Needs to be calculated prior to calling `realloc` to avoid dangling pointers: */ - const ptrdiff_t offsetInsideBlock = pool->ptr - pool->start; + const ptrdiff_t offsetInsideBlock + = EXPAT_SAFE_PTR_DIFF(pool->ptr, pool->start); if (blockSize < 0) { /* This condition traps a situation where either more than @@ -7517,8 +8605,7 @@ poolGrow(STRING_POOL *pool) { if (bytesToAllocate == 0) return XML_FALSE; - temp = (BLOCK *)pool->mem->realloc_fcn(pool->blocks, - (unsigned)bytesToAllocate); + temp = REALLOC(pool->parser, pool->blocks, bytesToAllocate); if (temp == NULL) return XML_FALSE; pool->blocks = temp; @@ -7558,43 +8645,58 @@ poolGrow(STRING_POOL *pool) { if (bytesToAllocate == 0) return XML_FALSE; - tem = pool->mem->malloc_fcn(bytesToAllocate); + tem = MALLOC(pool->parser, bytesToAllocate); if (! tem) return XML_FALSE; tem->size = blockSize; tem->next = pool->blocks; pool->blocks = tem; if (pool->ptr != pool->start) - memcpy(tem->s, pool->start, (pool->ptr - pool->start) * sizeof(XML_Char)); - pool->ptr = tem->s + (pool->ptr - pool->start); + memcpy(tem->s, pool->start, + EXPAT_SAFE_PTR_DIFF(pool->ptr, pool->start) * sizeof(XML_Char)); + pool->ptr = tem->s + EXPAT_SAFE_PTR_DIFF(pool->ptr, pool->start); pool->start = tem->s; pool->end = tem->s + blockSize; } return XML_TRUE; } -static int FASTCALL +static bool +poolGrowUntil(STRING_POOL *pool, size_t needed) { + for (;;) { + const size_t available = pool->end - pool->ptr; + if (available >= needed) { + return true; + } + if (! poolGrow(pool)) { + return false; + } + } +} + +static int nextScaffoldPart(XML_Parser parser) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ CONTENT_SCAFFOLD *me; int next; if (! dtd->scaffIndex) { - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if (parser->m_groupSize > ((size_t)(-1) / sizeof(int))) { + /* Detect and prevent integer overflow. */ + if (parser->m_groupSize > SIZE_MAX / sizeof(int)) { return -1; } -#endif - dtd->scaffIndex = (int *)MALLOC(parser, parser->m_groupSize * sizeof(int)); + dtd->scaffIndex = MALLOC(parser, parser->m_groupSize * sizeof(int)); if (! dtd->scaffIndex) return -1; + dtd->scaffIndexSize = parser->m_groupSize; dtd->scaffIndex[0] = 0; } + // Will casting to int be safe further down? + if (dtd->scaffCount > INT_MAX) { + return -1; + } + if (dtd->scaffCount >= dtd->scaffSize) { CONTENT_SCAFFOLD *temp; if (dtd->scaffold) { @@ -7607,26 +8709,25 @@ nextScaffoldPart(XML_Parser parser) { * from -Wtype-limits on platforms where * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ #if UINT_MAX >= SIZE_MAX - if (dtd->scaffSize > (size_t)(-1) / 2u / sizeof(CONTENT_SCAFFOLD)) { + if (dtd->scaffSize > SIZE_MAX / 2u / sizeof(CONTENT_SCAFFOLD)) { return -1; } #endif - temp = (CONTENT_SCAFFOLD *)REALLOC( - parser, dtd->scaffold, dtd->scaffSize * 2 * sizeof(CONTENT_SCAFFOLD)); + temp = REALLOC(parser, dtd->scaffold, + dtd->scaffSize * 2 * sizeof(CONTENT_SCAFFOLD)); if (temp == NULL) return -1; dtd->scaffSize *= 2; } else { - temp = (CONTENT_SCAFFOLD *)MALLOC(parser, INIT_SCAFFOLD_ELEMENTS - * sizeof(CONTENT_SCAFFOLD)); + temp = MALLOC(parser, INIT_SCAFFOLD_ELEMENTS * sizeof(CONTENT_SCAFFOLD)); if (temp == NULL) return -1; dtd->scaffSize = INIT_SCAFFOLD_ELEMENTS; } dtd->scaffold = temp; } - next = dtd->scaffCount++; + next = (int)dtd->scaffCount++; me = &dtd->scaffold[next]; if (dtd->scaffLevel) { CONTENT_SCAFFOLD *parent @@ -7658,22 +8759,25 @@ build_model(XML_Parser parser) { * from -Wtype-limits on platforms where * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ #if UINT_MAX >= SIZE_MAX - if (dtd->scaffCount > (size_t)(-1) / sizeof(XML_Content)) { + if (dtd->scaffCount > SIZE_MAX / sizeof(XML_Content)) { return NULL; } - if (dtd->contentStringLen > (size_t)(-1) / sizeof(XML_Char)) { + if (dtd->contentStringLen > SIZE_MAX / sizeof(XML_Char)) { return NULL; } #endif if (dtd->scaffCount * sizeof(XML_Content) - > (size_t)(-1) - dtd->contentStringLen * sizeof(XML_Char)) { + > SIZE_MAX - dtd->contentStringLen * sizeof(XML_Char)) { return NULL; } const size_t allocsize = (dtd->scaffCount * sizeof(XML_Content) + (dtd->contentStringLen * sizeof(XML_Char))); - ret = (XML_Content *)MALLOC(parser, allocsize); + // NOTE: We are avoiding MALLOC(..) here to so that + // applications that are not using XML_FreeContentModel but plain + // free(..) or .free_fcn() to free the content model's memory are safe. + ret = parser->m_mem.malloc_fcn(allocsize); if (! ret) return NULL; @@ -7745,12 +8849,21 @@ build_model(XML_Parser parser) { const XML_Char *src; dest->name = str; src = dtd->scaffold[src_node].name; - for (;;) { - *str++ = *src; - if (! *src) - break; - src++; + + const size_t nameLen = xcslen(src) + /* null terminator*/ 1; + + // Detect and prevent integer overflow + if (nameLen > SIZE_MAX / sizeof(XML_Char)) { + // NOTE: We are avoiding FREE(..) here because the model + // is not being allocated with MALLOC(..) but with plain + // .malloc_fcn(..). + parser->m_mem.free_fcn(ret); + return NULL; } + + memcpy(str, src, nameLen * sizeof(XML_Char)); + str += nameLen; + dest->numchildren = 0; dest->children = NULL; } else { @@ -7783,6 +8896,8 @@ getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, sizeof(ELEMENT_TYPE)); if (! ret) return NULL; + if (! ret->defaultAttForName.parser) + hashTableInit(&(ret->defaultAttForName), getRootParserOf(parser, NULL)); if (ret->name != name) poolDiscard(&dtd->pool); else { @@ -7794,23 +8909,25 @@ getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, } static XML_Char * -copyString(const XML_Char *s, const XML_Memory_Handling_Suite *memsuite) { - size_t charsRequired = 0; - XML_Char *result; - +copyString(const XML_Char *s, XML_Parser parser) { /* First determine how long the string is */ - while (s[charsRequired] != 0) { - charsRequired++; - } - /* Include the terminator */ - charsRequired++; + const size_t charsRequired = xcslen(s) + /*null terminator*/ 1; + + /* Detect and prevent integer overflow */ + if (charsRequired > SIZE_MAX / sizeof(XML_Char)) + return NULL; + + const size_t bytesRequired = charsRequired * sizeof(XML_Char); /* Now allocate space for the copy */ - result = memsuite->malloc_fcn(charsRequired * sizeof(XML_Char)); + XML_Char *const result = MALLOC(parser, bytesRequired); + if (result == NULL) return NULL; + /* Copy the original into place */ - memcpy(result, s, charsRequired * sizeof(XML_Char)); + memcpy(result, s, bytesRequired); + return result; } @@ -7825,10 +8942,10 @@ accountingGetCurrentAmplification(XML_Parser rootParser) { + rootParser->m_accounting.countBytesIndirect; const float amplificationFactor = rootParser->m_accounting.countBytesDirect - ? (countBytesOutput + ? ((float)countBytesOutput / (float)(rootParser->m_accounting.countBytesDirect)) - : ((lenOfShortestInclude - + rootParser->m_accounting.countBytesIndirect) + : ((float)(lenOfShortestInclude + + rootParser->m_accounting.countBytesIndirect) / (float)lenOfShortestInclude); assert(! rootParser->m_parentParser); return amplificationFactor; @@ -7977,15 +9094,25 @@ entityTrackingReportStats(XML_Parser rootParser, ENTITY *entity, const char *const entityName = entity->name; # endif + const bool limitingWanted = rootParser->m_entity_stats.debugLevel < 2; + const int maxLimitedDepth = 10; // somewhat arbitrary + const int candidateIndentDepth + = (int)rootParser->m_entity_stats.currentDepth - 1; + const bool limitingNeeded + = limitingWanted && (candidateIndentDepth > maxLimitedDepth); + const char *const ellipisOrEmpty = limitingNeeded ? " [..] " : ""; + const int indentDepth + = limitingNeeded ? (maxLimitedDepth - /* make space for ellipis */ 2) + : candidateIndentDepth; + fprintf( stderr, - "expat: Entities(%p): Count %9u, depth %2u/%2u %*s%s%s; %s length %d (xmlparse.c:%d)\n", + "expat: Entities(%p): Count %9u, depth %2u/%2u %*s%s%s%s; %s length %d (xmlparse.c:%d)\n", (void *)rootParser, rootParser->m_entity_stats.countEverOpened, rootParser->m_entity_stats.currentDepth, - rootParser->m_entity_stats.maximumDepthSeen, - (rootParser->m_entity_stats.currentDepth - 1) * 2, "", - entity->is_param ? "%" : "&", entityName, action, entity->textLen, - sourceLine); + rootParser->m_entity_stats.maximumDepthSeen, indentDepth * 2, "", + ellipisOrEmpty, entity->is_param ? "%" : "&", entityName, action, + entity->textLen, sourceLine); } static void @@ -8012,6 +9139,8 @@ entityTrackingOnClose(XML_Parser originParser, ENTITY *entity, int sourceLine) { rootParser->m_entity_stats.currentDepth--; } +#endif /* XML_GE == 1 */ + static XML_Parser getRootParserOf(XML_Parser parser, unsigned int *outLevelDiff) { XML_Parser rootParser = parser; @@ -8027,6 +9156,8 @@ getRootParserOf(XML_Parser parser, unsigned int *outLevelDiff) { return rootParser; } +#if XML_GE == 1 + const char * unsignedCharToPrintable(unsigned char c) { switch (c) { @@ -8542,11 +9673,13 @@ unsignedCharToPrintable(unsigned char c) { return "\\xFE"; case 255: return "\\xFF"; + // LCOV_EXCL_START default: assert(0); /* never gets here */ return "dead code"; } assert(0); /* never gets here */ + // LCOV_EXCL_STOP } #endif /* XML_GE == 1 */ diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c index 2c48bf408679538..dbf4a2d1b8f7080 100644 --- a/Modules/expat/xmlrole.c +++ b/Modules/expat/xmlrole.c @@ -12,10 +12,11 @@ Copyright (c) 2002-2006 Karl Waclawek Copyright (c) 2002-2003 Fred L. Drake, Jr. Copyright (c) 2005-2009 Steven Solie - Copyright (c) 2016-2023 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2017 Rhodri James Copyright (c) 2019 David Loffredo Copyright (c) 2021 Donghee Na + Copyright (c) 2025 Alfonso Gregory Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -36,6 +37,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #include "expat_config.h" @@ -46,7 +49,6 @@ # include "winconfig.h" #endif -#include "expat_external.h" #include "internal.h" #include "xmlrole.h" #include "ascii.h" @@ -119,9 +121,8 @@ static const char KW_SYSTEM[] # define setTopLevel(state) ((state)->handler = internalSubset) #endif /* not XML_DTD */ -typedef int PTRCALL PROLOG_HANDLER(PROLOG_STATE *state, int tok, - const char *ptr, const char *end, - const ENCODING *enc); +typedef int PROLOG_HANDLER(PROLOG_STATE *state, int tok, const char *ptr, + const char *end, const ENCODING *enc); static PROLOG_HANDLER prolog0, prolog1, prolog2, doctype0, doctype1, doctype2, doctype3, doctype4, doctype5, internalSubset, entity0, entity1, entity2, @@ -135,9 +136,9 @@ static PROLOG_HANDLER prolog0, prolog1, prolog2, doctype0, doctype1, doctype2, #endif /* XML_DTD */ declClose, error; -static int FASTCALL common(PROLOG_STATE *state, int tok); +static int common(PROLOG_STATE *state, int tok); -static int PTRCALL +static int prolog0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -168,7 +169,7 @@ prolog0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int prolog1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -200,7 +201,7 @@ prolog1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int prolog2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -220,7 +221,7 @@ prolog2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int doctype0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -237,7 +238,7 @@ doctype0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int doctype1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -263,7 +264,7 @@ doctype1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int doctype2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -279,7 +280,7 @@ doctype2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int doctype3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -295,7 +296,7 @@ doctype3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int doctype4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -314,7 +315,7 @@ doctype4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int doctype5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -330,7 +331,7 @@ doctype5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int internalSubset(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -375,7 +376,7 @@ internalSubset(PROLOG_STATE *state, int tok, const char *ptr, const char *end, #ifdef XML_DTD -static int PTRCALL +static int externalSubset0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { state->handler = externalSubset1; @@ -384,7 +385,7 @@ externalSubset0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return externalSubset1(state, tok, ptr, end, enc); } -static int PTRCALL +static int externalSubset1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -412,7 +413,7 @@ externalSubset1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, #endif /* XML_DTD */ -static int PTRCALL +static int entity0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -431,7 +432,7 @@ entity0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -447,7 +448,7 @@ entity1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -471,7 +472,7 @@ entity2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -487,7 +488,7 @@ entity3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -503,7 +504,7 @@ entity4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -522,7 +523,7 @@ entity5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity6(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -539,7 +540,7 @@ entity6(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity7(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -563,7 +564,7 @@ entity7(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity8(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -579,7 +580,7 @@ entity8(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity9(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -595,7 +596,7 @@ entity9(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity10(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -611,7 +612,7 @@ entity10(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int notation0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -627,7 +628,7 @@ notation0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int notation1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -647,7 +648,7 @@ notation1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int notation2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -663,7 +664,7 @@ notation2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int notation3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -680,7 +681,7 @@ notation3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int notation4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -700,7 +701,7 @@ notation4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -717,7 +718,7 @@ attlist0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -737,7 +738,7 @@ attlist1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -767,7 +768,7 @@ attlist2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -785,7 +786,7 @@ attlist3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -804,7 +805,7 @@ attlist4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -820,7 +821,7 @@ attlist5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist6(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -836,7 +837,7 @@ attlist6(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist7(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -856,7 +857,7 @@ attlist7(PROLOG_STATE *state, int tok, const char *ptr, const char *end, } /* default value */ -static int PTRCALL +static int attlist8(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -886,7 +887,7 @@ attlist8(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist9(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -902,7 +903,7 @@ attlist9(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -919,7 +920,7 @@ element0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -945,7 +946,7 @@ element1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -979,7 +980,7 @@ element2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1003,7 +1004,7 @@ element3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1020,7 +1021,7 @@ element4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1040,7 +1041,7 @@ element5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element6(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1069,7 +1070,7 @@ element6(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element7(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1118,7 +1119,7 @@ element7(PROLOG_STATE *state, int tok, const char *ptr, const char *end, #ifdef XML_DTD -static int PTRCALL +static int condSect0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -1138,7 +1139,7 @@ condSect0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int condSect1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1155,7 +1156,7 @@ condSect1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int condSect2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1173,7 +1174,7 @@ condSect2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, #endif /* XML_DTD */ -static int PTRCALL +static int declClose(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1209,7 +1210,7 @@ declClose(PROLOG_STATE *state, int tok, const char *ptr, const char *end, * * LCOV_EXCL_START */ -static int PTRCALL +static int error(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(state); @@ -1221,7 +1222,7 @@ error(PROLOG_STATE *state, int tok, const char *ptr, const char *end, } /* LCOV_EXCL_STOP */ -static int FASTCALL +static int common(PROLOG_STATE *state, int tok) { #ifdef XML_DTD if (! state->documentEntity && tok == XML_TOK_PARAM_ENTITY_REF) diff --git a/Modules/expat/xmlrole.h b/Modules/expat/xmlrole.h index a7904274c91d4ec..f4aba4079f5cdfa 100644 --- a/Modules/expat/xmlrole.h +++ b/Modules/expat/xmlrole.h @@ -10,7 +10,7 @@ Copyright (c) 2000 Clark Cooper Copyright (c) 2002 Karl Waclawek Copyright (c) 2002 Fred L. Drake, Jr. - Copyright (c) 2017-2024 Sebastian Pipping + Copyright (c) 2017-2026 Sebastian Pipping Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -31,22 +31,14 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifndef XmlRole_INCLUDED -#define XmlRole_INCLUDED 1 - -#ifdef __VMS -/* 0 1 2 3 0 1 2 3 - 1234567890123456789012345678901 1234567890123456789012345678901 */ -# define XmlPrologStateInitExternalEntity XmlPrologStateInitExternalEnt -#endif +# define XmlRole_INCLUDED 1 -#include "xmltok.h" - -#ifdef __cplusplus -extern "C" { -#endif +# include "xmltok.h" enum { XML_ROLE_ERROR = -1, @@ -107,36 +99,32 @@ enum { XML_ROLE_CONTENT_ELEMENT_PLUS, XML_ROLE_PI, XML_ROLE_COMMENT, -#ifdef XML_DTD +# ifdef XML_DTD XML_ROLE_TEXT_DECL, XML_ROLE_IGNORE_SECT, XML_ROLE_INNER_PARAM_ENTITY_REF, -#endif /* XML_DTD */ +# endif /* XML_DTD */ XML_ROLE_PARAM_ENTITY_REF }; typedef struct prolog_state { - int(PTRCALL *handler)(struct prolog_state *state, int tok, const char *ptr, - const char *end, const ENCODING *enc); + int (*handler)(struct prolog_state *state, int tok, const char *ptr, + const char *end, const ENCODING *enc); unsigned level; int role_none; -#ifdef XML_DTD +# ifdef XML_DTD unsigned includeLevel; int documentEntity; int inEntityValue; -#endif /* XML_DTD */ +# endif /* XML_DTD */ } PROLOG_STATE; void XmlPrologStateInit(PROLOG_STATE *state); -#ifdef XML_DTD +# ifdef XML_DTD void XmlPrologStateInitExternalEntity(PROLOG_STATE *state); -#endif /* XML_DTD */ - -#define XmlTokenRole(state, tok, ptr, end, enc) \ - (((state)->handler)(state, tok, ptr, end, enc)) +# endif /* XML_DTD */ -#ifdef __cplusplus -} -#endif +# define XmlTokenRole(state, tok, ptr, end, enc) \ + (((state)->handler)(state, tok, ptr, end, enc)) #endif /* not XmlRole_INCLUDED */ diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index 29a66d72ceea5e5..2f837acc4cbf72a 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -12,7 +12,7 @@ Copyright (c) 2002 Greg Stein Copyright (c) 2002-2016 Karl Waclawek Copyright (c) 2005-2009 Steven Solie - Copyright (c) 2016-2024 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2016 Pascal Cuoq Copyright (c) 2016 Don Lewis Copyright (c) 2017 Rhodri James @@ -24,6 +24,10 @@ Copyright (c) 2022 Martin Ettl Copyright (c) 2022 Sean McBride Copyright (c) 2023 Hanno Böck + Copyright (c) 2025 Alfonso Gregory + Copyright (c) 2026 Nick Begg + Copyright (c) 2026 Kartik Kenchi + Copyright (c) 2026 Afonso Januário Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -44,6 +48,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #include "expat_config.h" @@ -56,8 +62,8 @@ # include "winconfig.h" #endif -#include "expat_external.h" #include "internal.h" +#include "fallthrough.h" #include "xmltok.h" #include "nametab.h" @@ -131,20 +137,20 @@ : ((p)[1] & 0x80) == 0 \ || ((*p) == 0xF4 ? (p)[1] > 0x8F : ((p)[1] & 0xC0) == 0xC0))) -static int PTRFASTCALL +static int isNever(const ENCODING *enc, const char *p) { UNUSED_P(enc); UNUSED_P(p); return 0; } -static int PTRFASTCALL +static int utf8_isName2(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_GET_NAMING2(namePages, (const unsigned char *)p); } -static int PTRFASTCALL +static int utf8_isName3(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_GET_NAMING3(namePages, (const unsigned char *)p); @@ -152,13 +158,13 @@ utf8_isName3(const ENCODING *enc, const char *p) { #define utf8_isName4 isNever -static int PTRFASTCALL +static int utf8_isNmstrt2(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_GET_NAMING2(nmstrtPages, (const unsigned char *)p); } -static int PTRFASTCALL +static int utf8_isNmstrt3(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_GET_NAMING3(nmstrtPages, (const unsigned char *)p); @@ -166,19 +172,19 @@ utf8_isNmstrt3(const ENCODING *enc, const char *p) { #define utf8_isNmstrt4 isNever -static int PTRFASTCALL +static int utf8_isInvalid2(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_INVALID2((const unsigned char *)p); } -static int PTRFASTCALL +static int utf8_isInvalid3(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_INVALID3((const unsigned char *)p); } -static int PTRFASTCALL +static int utf8_isInvalid4(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_INVALID4((const unsigned char *)p); @@ -187,36 +193,20 @@ utf8_isInvalid4(const ENCODING *enc, const char *p) { struct normal_encoding { ENCODING enc; unsigned char type[256]; -#ifdef XML_MIN_SIZE - int(PTRFASTCALL *byteType)(const ENCODING *, const char *); - int(PTRFASTCALL *isNameMin)(const ENCODING *, const char *); - int(PTRFASTCALL *isNmstrtMin)(const ENCODING *, const char *); - int(PTRFASTCALL *byteToAscii)(const ENCODING *, const char *); - int(PTRCALL *charMatches)(const ENCODING *, const char *, int); -#endif /* XML_MIN_SIZE */ - int(PTRFASTCALL *isName2)(const ENCODING *, const char *); - int(PTRFASTCALL *isName3)(const ENCODING *, const char *); - int(PTRFASTCALL *isName4)(const ENCODING *, const char *); - int(PTRFASTCALL *isNmstrt2)(const ENCODING *, const char *); - int(PTRFASTCALL *isNmstrt3)(const ENCODING *, const char *); - int(PTRFASTCALL *isNmstrt4)(const ENCODING *, const char *); - int(PTRFASTCALL *isInvalid2)(const ENCODING *, const char *); - int(PTRFASTCALL *isInvalid3)(const ENCODING *, const char *); - int(PTRFASTCALL *isInvalid4)(const ENCODING *, const char *); + int (*isName2)(const ENCODING *, const char *); + int (*isName3)(const ENCODING *, const char *); + int (*isName4)(const ENCODING *, const char *); + int (*isNmstrt2)(const ENCODING *, const char *); + int (*isNmstrt3)(const ENCODING *, const char *); + int (*isNmstrt4)(const ENCODING *, const char *); + int (*isInvalid2)(const ENCODING *, const char *); + int (*isInvalid3)(const ENCODING *, const char *); + int (*isInvalid4)(const ENCODING *, const char *); }; #define AS_NORMAL_ENCODING(enc) ((const struct normal_encoding *)(enc)) -#ifdef XML_MIN_SIZE - -# define STANDARD_VTABLE(E) \ - E##byteType, E##isNameMin, E##isNmstrtMin, E##byteToAscii, E##charMatches, - -#else - -# define STANDARD_VTABLE(E) /* as nothing */ - -#endif +#define STANDARD_VTABLE(E) /* as nothing */ #define NORMAL_VTABLE(E) \ E##isName2, E##isName3, E##isName4, E##isNmstrt2, E##isNmstrt3, \ @@ -227,80 +217,33 @@ struct normal_encoding { /* isNmstrt2 */ NULL, /* isNmstrt3 */ NULL, /* isNmstrt4 */ NULL, \ /* isInvalid2 */ NULL, /* isInvalid3 */ NULL, /* isInvalid4 */ NULL -static int FASTCALL checkCharRefNumber(int result); +#define UTF16_NULL_VTABLE(E) NULL_VTABLE + +static int checkCharRefNumber(int result); #include "xmltok_impl.h" #include "ascii.h" -#ifdef XML_MIN_SIZE -# define sb_isNameMin isNever -# define sb_isNmstrtMin isNever -#endif - -#ifdef XML_MIN_SIZE -# define MINBPC(enc) ((enc)->minBytesPerChar) -#else /* minimum bytes per character */ -# define MINBPC(enc) 1 -#endif +#define MINBPC(enc) 1 #define SB_BYTE_TYPE(enc, p) \ (((const struct normal_encoding *)(enc))->type[(unsigned char)*(p)]) -#ifdef XML_MIN_SIZE -static int PTRFASTCALL -sb_byteType(const ENCODING *enc, const char *p) { - return SB_BYTE_TYPE(enc, p); -} -# define BYTE_TYPE(enc, p) (AS_NORMAL_ENCODING(enc)->byteType(enc, p)) -#else -# define BYTE_TYPE(enc, p) SB_BYTE_TYPE(enc, p) -#endif +#define BYTE_TYPE(enc, p) SB_BYTE_TYPE(enc, p) -#ifdef XML_MIN_SIZE -# define BYTE_TO_ASCII(enc, p) (AS_NORMAL_ENCODING(enc)->byteToAscii(enc, p)) -static int PTRFASTCALL -sb_byteToAscii(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return *p; -} -#else -# define BYTE_TO_ASCII(enc, p) (*(p)) -#endif +#define BYTE_TO_ASCII(enc, p) (*(p)) #define IS_NAME_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isName##n(enc, p)) #define IS_NMSTRT_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isNmstrt##n(enc, p)) -#ifdef XML_MIN_SIZE -# define IS_INVALID_CHAR(enc, p, n) \ - (AS_NORMAL_ENCODING(enc)->isInvalid##n \ - && AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p)) -#else -# define IS_INVALID_CHAR(enc, p, n) \ - (AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p)) -#endif +#define IS_INVALID_CHAR(enc, p, n) \ + (AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p)) -#ifdef XML_MIN_SIZE -# define IS_NAME_CHAR_MINBPC(enc, p) \ - (AS_NORMAL_ENCODING(enc)->isNameMin(enc, p)) -# define IS_NMSTRT_CHAR_MINBPC(enc, p) \ - (AS_NORMAL_ENCODING(enc)->isNmstrtMin(enc, p)) -#else -# define IS_NAME_CHAR_MINBPC(enc, p) (0) -# define IS_NMSTRT_CHAR_MINBPC(enc, p) (0) -#endif +#define IS_NAME_CHAR_MINBPC(enc, p) (0) +#define IS_NMSTRT_CHAR_MINBPC(enc, p) (0) -#ifdef XML_MIN_SIZE -# define CHAR_MATCHES(enc, p, c) \ - (AS_NORMAL_ENCODING(enc)->charMatches(enc, p, c)) -static int PTRCALL -sb_charMatches(const ENCODING *enc, const char *p, int c) { - UNUSED_P(enc); - return *p == c; -} -#else /* c is an ASCII character */ -# define CHAR_MATCHES(enc, p, c) (*(p) == (c)) -#endif +#define CHAR_MATCHES(enc, p, c) (*(p) == (c)) #define PREFIX(ident) normal_##ident #define XML_TOK_IMPL_C @@ -363,7 +306,7 @@ _INTERNAL_trim_to_complete_utf8_characters(const char *from, *fromLimRef = fromLim; } -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result utf8_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, char **toP, const char *toLim) { bool input_incomplete = false; @@ -402,7 +345,7 @@ utf8_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, return XML_CONVERT_COMPLETED; } -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result utf8_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim, unsigned short **toP, const unsigned short *toLim) { enum XML_Convert_Result res = XML_CONVERT_COMPLETED; @@ -500,7 +443,7 @@ static const struct normal_encoding internal_utf8_encoding }, STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)}; -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result latin1_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, char **toP, const char *toLim) { UNUSED_P(enc); @@ -523,7 +466,7 @@ latin1_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, } } -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result latin1_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim, unsigned short **toP, const unsigned short *toLim) { UNUSED_P(enc); @@ -558,7 +501,7 @@ static const struct normal_encoding latin1_encoding }, STANDARD_VTABLE(sb_) NULL_VTABLE}; -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result ascii_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, char **toP, const char *toLim) { UNUSED_P(enc); @@ -593,7 +536,7 @@ static const struct normal_encoding ascii_encoding }, STANDARD_VTABLE(sb_) NULL_VTABLE}; -static int PTRFASTCALL +static int unicode_byte_type(char hi, char lo) { switch ((unsigned char)hi) { /* 0xD800-0xDBFF first 16-bit code unit or high surrogate (W1) */ @@ -620,7 +563,7 @@ unicode_byte_type(char hi, char lo) { } #define DEFINE_UTF16_TO_UTF8(E) \ - static enum XML_Convert_Result PTRCALL E##toUtf8( \ + static enum XML_Convert_Result E##toUtf8( \ const ENCODING *enc, const char **fromP, const char *fromLim, \ char **toP, const char *toLim) { \ const char *from = *fromP; \ @@ -641,7 +584,7 @@ unicode_byte_type(char hi, char lo) { *(*toP)++ = lo; \ break; \ } \ - /* fall through */ \ + EXPAT_FALLTHROUGH; \ case 0x1: \ case 0x2: \ case 0x3: \ @@ -697,15 +640,16 @@ unicode_byte_type(char hi, char lo) { } #define DEFINE_UTF16_TO_UTF16(E) \ - static enum XML_Convert_Result PTRCALL E##toUtf16( \ + static enum XML_Convert_Result E##toUtf16( \ const ENCODING *enc, const char **fromP, const char *fromLim, \ unsigned short **toP, const unsigned short *toLim) { \ enum XML_Convert_Result res = XML_CONVERT_COMPLETED; \ UNUSED_P(enc); \ fromLim = *fromP + (((fromLim - *fromP) >> 1) << 1); /* shrink to even */ \ - /* Avoid copying first half only of surrogate */ \ + /* Avoid copying the first half (2 bytes) of surrogate pairs (4 bytes) */ \ if (fromLim - *fromP > ((toLim - *toP) << 1) \ - && (GET_HI(fromLim - 2) & 0xF8) == 0xD8) { \ + && /* are the last two bytes a high surrogate (0xD800-0xDBFF)? */ \ + (GET_HI(fromLim - 2) & 0xFC) == 0xD8) { \ fromLim -= 2; \ res = XML_CONVERT_INPUT_INCOMPLETE; \ } \ @@ -743,70 +687,38 @@ DEFINE_UTF16_TO_UTF16(big2_) UCS2_GET_NAMING(namePages, (unsigned char)p[1], (unsigned char)p[0]) #define LITTLE2_IS_NMSTRT_CHAR_MINBPC(p) \ UCS2_GET_NAMING(nmstrtPages, (unsigned char)p[1], (unsigned char)p[0]) +/* A 4-byte UTF-16 character is a surrogate pair; byteType only reports BT_LEAD4 + for a high surrogate, so the pair is invalid unless the second unit is a low + surrogate (U+DC00..U+DFFF, i.e. high byte 0xDC..0xDF). */ +#define LITTLE2_IS_INVALID_CHAR(p, n) \ + ((n) == 4 && ((unsigned char)(p)[3] & 0xFC) != 0xDC) -#ifdef XML_MIN_SIZE - -static int PTRFASTCALL -little2_byteType(const ENCODING *enc, const char *p) { - return LITTLE2_BYTE_TYPE(enc, p); -} - -static int PTRFASTCALL -little2_byteToAscii(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return LITTLE2_BYTE_TO_ASCII(p); -} - -static int PTRCALL -little2_charMatches(const ENCODING *enc, const char *p, int c) { - UNUSED_P(enc); - return LITTLE2_CHAR_MATCHES(p, c); -} - -static int PTRFASTCALL -little2_isNameMin(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return LITTLE2_IS_NAME_CHAR_MINBPC(p); -} - -static int PTRFASTCALL -little2_isNmstrtMin(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return LITTLE2_IS_NMSTRT_CHAR_MINBPC(p); -} - -# undef VTABLE -# define VTABLE VTABLE1, little2_toUtf8, little2_toUtf16 +#undef PREFIX +#define PREFIX(ident) little2_##ident +#define MINBPC(enc) 2 +/* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */ +#define BYTE_TYPE(enc, p) LITTLE2_BYTE_TYPE(enc, p) +#define BYTE_TO_ASCII(enc, p) LITTLE2_BYTE_TO_ASCII(p) +#define CHAR_MATCHES(enc, p, c) LITTLE2_CHAR_MATCHES(p, c) +#define IS_NAME_CHAR(enc, p, n) 0 +#define IS_NAME_CHAR_MINBPC(enc, p) LITTLE2_IS_NAME_CHAR_MINBPC(p) +#define IS_NMSTRT_CHAR(enc, p, n) (0) +#define IS_NMSTRT_CHAR_MINBPC(enc, p) LITTLE2_IS_NMSTRT_CHAR_MINBPC(p) +#define IS_INVALID_CHAR(enc, p, n) LITTLE2_IS_INVALID_CHAR(p, n) -#else /* not XML_MIN_SIZE */ +#define XML_TOK_IMPL_C +#include "xmltok_impl.c" +#undef XML_TOK_IMPL_C -# undef PREFIX -# define PREFIX(ident) little2_##ident -# define MINBPC(enc) 2 -/* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */ -# define BYTE_TYPE(enc, p) LITTLE2_BYTE_TYPE(enc, p) -# define BYTE_TO_ASCII(enc, p) LITTLE2_BYTE_TO_ASCII(p) -# define CHAR_MATCHES(enc, p, c) LITTLE2_CHAR_MATCHES(p, c) -# define IS_NAME_CHAR(enc, p, n) 0 -# define IS_NAME_CHAR_MINBPC(enc, p) LITTLE2_IS_NAME_CHAR_MINBPC(p) -# define IS_NMSTRT_CHAR(enc, p, n) (0) -# define IS_NMSTRT_CHAR_MINBPC(enc, p) LITTLE2_IS_NMSTRT_CHAR_MINBPC(p) - -# define XML_TOK_IMPL_C -# include "xmltok_impl.c" -# undef XML_TOK_IMPL_C - -# undef MINBPC -# undef BYTE_TYPE -# undef BYTE_TO_ASCII -# undef CHAR_MATCHES -# undef IS_NAME_CHAR -# undef IS_NAME_CHAR_MINBPC -# undef IS_NMSTRT_CHAR -# undef IS_NMSTRT_CHAR_MINBPC -# undef IS_INVALID_CHAR - -#endif /* not XML_MIN_SIZE */ +#undef MINBPC +#undef BYTE_TYPE +#undef BYTE_TO_ASCII +#undef CHAR_MATCHES +#undef IS_NAME_CHAR +#undef IS_NAME_CHAR_MINBPC +#undef IS_NMSTRT_CHAR +#undef IS_NMSTRT_CHAR_MINBPC +#undef IS_INVALID_CHAR #ifdef XML_NS @@ -822,7 +734,7 @@ static const struct normal_encoding little2_encoding_ns # include "asciitab.h" # include "latin1tab.h" }, - STANDARD_VTABLE(little2_) NULL_VTABLE}; + STANDARD_VTABLE(little2_) UTF16_NULL_VTABLE(little2_)}; #endif @@ -840,7 +752,7 @@ static const struct normal_encoding little2_encoding #undef BT_COLON #include "latin1tab.h" }, - STANDARD_VTABLE(little2_) NULL_VTABLE}; + STANDARD_VTABLE(little2_) UTF16_NULL_VTABLE(little2_)}; #if BYTEORDER != 4321 @@ -852,7 +764,7 @@ static const struct normal_encoding internal_little2_encoding_ns # include "iasciitab.h" # include "latin1tab.h" }, - STANDARD_VTABLE(little2_) NULL_VTABLE}; + STANDARD_VTABLE(little2_) UTF16_NULL_VTABLE(little2_)}; # endif @@ -864,7 +776,7 @@ static const struct normal_encoding internal_little2_encoding # undef BT_COLON # include "latin1tab.h" }, - STANDARD_VTABLE(little2_) NULL_VTABLE}; + STANDARD_VTABLE(little2_) UTF16_NULL_VTABLE(little2_)}; #endif @@ -876,70 +788,38 @@ static const struct normal_encoding internal_little2_encoding UCS2_GET_NAMING(namePages, (unsigned char)p[0], (unsigned char)p[1]) #define BIG2_IS_NMSTRT_CHAR_MINBPC(p) \ UCS2_GET_NAMING(nmstrtPages, (unsigned char)p[0], (unsigned char)p[1]) +/* A 4-byte UTF-16 character is a surrogate pair; byteType only reports BT_LEAD4 + for a high surrogate, so the pair is invalid unless the second unit is a low + surrogate (U+DC00..U+DFFF, i.e. high byte 0xDC..0xDF). */ +#define BIG2_IS_INVALID_CHAR(p, n) \ + ((n) == 4 && ((unsigned char)(p)[2] & 0xFC) != 0xDC) -#ifdef XML_MIN_SIZE - -static int PTRFASTCALL -big2_byteType(const ENCODING *enc, const char *p) { - return BIG2_BYTE_TYPE(enc, p); -} - -static int PTRFASTCALL -big2_byteToAscii(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return BIG2_BYTE_TO_ASCII(p); -} - -static int PTRCALL -big2_charMatches(const ENCODING *enc, const char *p, int c) { - UNUSED_P(enc); - return BIG2_CHAR_MATCHES(p, c); -} - -static int PTRFASTCALL -big2_isNameMin(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return BIG2_IS_NAME_CHAR_MINBPC(p); -} - -static int PTRFASTCALL -big2_isNmstrtMin(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return BIG2_IS_NMSTRT_CHAR_MINBPC(p); -} - -# undef VTABLE -# define VTABLE VTABLE1, big2_toUtf8, big2_toUtf16 +#undef PREFIX +#define PREFIX(ident) big2_##ident +#define MINBPC(enc) 2 +/* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */ +#define BYTE_TYPE(enc, p) BIG2_BYTE_TYPE(enc, p) +#define BYTE_TO_ASCII(enc, p) BIG2_BYTE_TO_ASCII(p) +#define CHAR_MATCHES(enc, p, c) BIG2_CHAR_MATCHES(p, c) +#define IS_NAME_CHAR(enc, p, n) 0 +#define IS_NAME_CHAR_MINBPC(enc, p) BIG2_IS_NAME_CHAR_MINBPC(p) +#define IS_NMSTRT_CHAR(enc, p, n) (0) +#define IS_NMSTRT_CHAR_MINBPC(enc, p) BIG2_IS_NMSTRT_CHAR_MINBPC(p) +#define IS_INVALID_CHAR(enc, p, n) BIG2_IS_INVALID_CHAR(p, n) -#else /* not XML_MIN_SIZE */ +#define XML_TOK_IMPL_C +#include "xmltok_impl.c" +#undef XML_TOK_IMPL_C -# undef PREFIX -# define PREFIX(ident) big2_##ident -# define MINBPC(enc) 2 -/* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */ -# define BYTE_TYPE(enc, p) BIG2_BYTE_TYPE(enc, p) -# define BYTE_TO_ASCII(enc, p) BIG2_BYTE_TO_ASCII(p) -# define CHAR_MATCHES(enc, p, c) BIG2_CHAR_MATCHES(p, c) -# define IS_NAME_CHAR(enc, p, n) 0 -# define IS_NAME_CHAR_MINBPC(enc, p) BIG2_IS_NAME_CHAR_MINBPC(p) -# define IS_NMSTRT_CHAR(enc, p, n) (0) -# define IS_NMSTRT_CHAR_MINBPC(enc, p) BIG2_IS_NMSTRT_CHAR_MINBPC(p) - -# define XML_TOK_IMPL_C -# include "xmltok_impl.c" -# undef XML_TOK_IMPL_C - -# undef MINBPC -# undef BYTE_TYPE -# undef BYTE_TO_ASCII -# undef CHAR_MATCHES -# undef IS_NAME_CHAR -# undef IS_NAME_CHAR_MINBPC -# undef IS_NMSTRT_CHAR -# undef IS_NMSTRT_CHAR_MINBPC -# undef IS_INVALID_CHAR - -#endif /* not XML_MIN_SIZE */ +#undef MINBPC +#undef BYTE_TYPE +#undef BYTE_TO_ASCII +#undef CHAR_MATCHES +#undef IS_NAME_CHAR +#undef IS_NAME_CHAR_MINBPC +#undef IS_NMSTRT_CHAR +#undef IS_NMSTRT_CHAR_MINBPC +#undef IS_INVALID_CHAR #ifdef XML_NS @@ -955,7 +835,7 @@ static const struct normal_encoding big2_encoding_ns # include "asciitab.h" # include "latin1tab.h" }, - STANDARD_VTABLE(big2_) NULL_VTABLE}; + STANDARD_VTABLE(big2_) UTF16_NULL_VTABLE(big2_)}; #endif @@ -973,7 +853,7 @@ static const struct normal_encoding big2_encoding #undef BT_COLON #include "latin1tab.h" }, - STANDARD_VTABLE(big2_) NULL_VTABLE}; + STANDARD_VTABLE(big2_) UTF16_NULL_VTABLE(big2_)}; #if BYTEORDER != 1234 @@ -985,7 +865,7 @@ static const struct normal_encoding internal_big2_encoding_ns # include "iasciitab.h" # include "latin1tab.h" }, - STANDARD_VTABLE(big2_) NULL_VTABLE}; + STANDARD_VTABLE(big2_) UTF16_NULL_VTABLE(big2_)}; # endif @@ -997,13 +877,13 @@ static const struct normal_encoding internal_big2_encoding # undef BT_COLON # include "latin1tab.h" }, - STANDARD_VTABLE(big2_) NULL_VTABLE}; + STANDARD_VTABLE(big2_) UTF16_NULL_VTABLE(big2_)}; #endif #undef PREFIX -static int FASTCALL +static int streqci(const char *s1, const char *s2) { for (;;) { char c1 = *s1++; @@ -1024,7 +904,7 @@ streqci(const char *s1, const char *s2) { return 1; } -static void PTRCALL +static void initUpdatePosition(const ENCODING *enc, const char *ptr, const char *end, POSITION *pos) { UNUSED_P(enc); @@ -1042,7 +922,7 @@ toAscii(const ENCODING *enc, const char *ptr, const char *end) { return buf[0]; } -static int FASTCALL +static int isSpace(int c) { switch (c) { case 0x20: @@ -1148,6 +1028,34 @@ static const char KW_yes[] = {ASCII_y, ASCII_e, ASCII_s, '\0'}; static const char KW_no[] = {ASCII_n, ASCII_o, '\0'}; +static const char KW_1_dot[] = {ASCII_1, ASCII_PERIOD, '\0'}; + +/* Checks a version pseudo-attribute value against the VersionNum production. + XML 1.0 Fourth Edition only allows the literal "1.0", but the Fifth + Edition relaxed this to "1." followed by one or more digits, since Expat + only implements 1.0 itself but plans to track the Fifth Edition's laxer + grammar here so that "1.1" and similar aren't rejected only to have that + rejection reverted later. Returns true for a value matching "1.[0-9]+". + val/valEnd bound the value itself; valEnd is the upper bound used when + decoding the individual characters between them. */ +static bool +checkXmlDeclVersionNum(const ENCODING *enc, const char *val, + const char *valEnd) { + if (valEnd - val < 2 * enc->minBytesPerChar + || ! XmlNameMatchesAscii(enc, val, val + 2 * enc->minBytesPerChar, + KW_1_dot)) + return false; + val += 2 * enc->minBytesPerChar; + if (val == valEnd) + return false; + for (; val != valEnd; val += enc->minBytesPerChar) { + int c = toAscii(enc, val, valEnd); + if (c < ASCII_0 || c > ASCII_9) + return false; + } + return true; +} + static int doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *, const char *, const char *), @@ -1175,6 +1083,22 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *, const char *, *versionPtr = val; if (versionEndPtr) *versionEndPtr = ptr; + /* The version number must not be empty; VersionNum requires at least + one character. The encoding and standalone pseudo-attributes below + already reject an empty value, so keep version consistent. */ + if (val == ptr - enc->minBytesPerChar) { + *badPtr = val; + return 0; + } + /* Expat implements XML 1.0 only, so any version outside the "1.0"/"1.x" + family is rejected. Following the Fifth Edition's VersionNum + production (rather than the Fourth Edition's exact "1.0") avoids + rejecting "1.1" now only to have to revert that once Expat tracks + the newer edition. */ + if (! checkXmlDeclVersionNum(enc, val, ptr - enc->minBytesPerChar)) { + *badPtr = val; + return 0; + } if (! parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) { *badPtr = ptr; return 0; @@ -1229,7 +1153,7 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *, const char *, return 1; } -static int FASTCALL +static int checkCharRefNumber(int result) { switch (result >> 8) { case 0xD8: @@ -1253,7 +1177,7 @@ checkCharRefNumber(int result) { return result; } -int FASTCALL +int XmlUtf8Encode(int c, char *buf) { enum { /* minN is minimum legal resulting value for N byte sequence */ @@ -1289,7 +1213,7 @@ XmlUtf8Encode(int c, char *buf) { return 0; /* LCOV_EXCL_LINE: this case too is eliminated before calling */ } -int FASTCALL +int XmlUtf16Encode(int charNum, unsigned short *buf) { if (charNum < 0) return 0; @@ -1321,7 +1245,7 @@ XmlSizeOfUnknownEncoding(void) { return sizeof(struct unknown_encoding); } -static int PTRFASTCALL +static int unknown_isName(const ENCODING *enc, const char *p) { const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc); int c = uenc->convert(uenc->userData, p); @@ -1330,7 +1254,7 @@ unknown_isName(const ENCODING *enc, const char *p) { return UCS2_GET_NAMING(namePages, c >> 8, c & 0xFF); } -static int PTRFASTCALL +static int unknown_isNmstrt(const ENCODING *enc, const char *p) { const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc); int c = uenc->convert(uenc->userData, p); @@ -1339,14 +1263,14 @@ unknown_isNmstrt(const ENCODING *enc, const char *p) { return UCS2_GET_NAMING(nmstrtPages, c >> 8, c & 0xFF); } -static int PTRFASTCALL +static int unknown_isInvalid(const ENCODING *enc, const char *p) { const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc); int c = uenc->convert(uenc->userData, p); return (c & ~0xFFFF) || checkCharRefNumber(c) < 0; } -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result unknown_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, char **toP, const char *toLim) { const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc); @@ -1376,7 +1300,7 @@ unknown_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, } } -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result unknown_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim, unsigned short **toP, const unsigned short *toLim) { const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc); @@ -1398,7 +1322,7 @@ unknown_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim, } ENCODING * -XmlInitUnknownEncoding(void *mem, int *table, CONVERTER convert, +XmlInitUnknownEncoding(void *mem, const int *table, CONVERTER convert, void *userData) { int i; struct unknown_encoding *e = (struct unknown_encoding *)mem; @@ -1500,7 +1424,7 @@ static const char KW_UTF_16LE[] = {ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, ASCII_L, ASCII_E, '\0'}; -static int FASTCALL +static int getEncodingIndex(const char *name) { static const char *const encodingNames[] = { KW_ISO_8859_1, KW_US_ASCII, KW_UTF_8, KW_UTF_16, KW_UTF_16BE, KW_UTF_16LE, @@ -1557,7 +1481,7 @@ initScan(const ENCODING *const *encodingTable, const INIT_ENCODING *enc, case 0xEF: /* possibly first byte of UTF-8 BOM */ if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC && state == XML_CONTENT_STATE) break; - /* fall through */ + EXPAT_FALLTHROUGH; case 0x00: case 0x3C: return XML_TOK_PARTIAL; @@ -1661,7 +1585,7 @@ initScan(const ENCODING *const *encodingTable, const INIT_ENCODING *enc, # undef ns ENCODING * -XmlInitUnknownEncodingNS(void *mem, int *table, CONVERTER convert, +XmlInitUnknownEncodingNS(void *mem, const int *table, CONVERTER convert, void *userData) { ENCODING *enc = XmlInitUnknownEncoding(mem, table, convert, userData); if (enc) diff --git a/Modules/expat/xmltok.h b/Modules/expat/xmltok.h index c51fce1ec1518be..bf641af6d4f77f0 100644 --- a/Modules/expat/xmltok.h +++ b/Modules/expat/xmltok.h @@ -10,7 +10,7 @@ Copyright (c) 2000 Clark Cooper Copyright (c) 2002 Fred L. Drake, Jr. Copyright (c) 2002-2005 Karl Waclawek - Copyright (c) 2016-2024 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2017 Rhodri James Licensed under the MIT license: @@ -32,121 +32,121 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifndef XmlTok_INCLUDED -#define XmlTok_INCLUDED 1 +# define XmlTok_INCLUDED 1 -#ifdef __cplusplus -extern "C" { -#endif +# include // uint64_t /* The following token may be returned by XmlContentTok */ -#define XML_TOK_TRAILING_RSQB \ - -5 /* ] or ]] at the end of the scan; might be \ - start of illegal ]]> sequence */ +# define XML_TOK_TRAILING_RSQB \ + -5 /* ] or ]] at the end of the scan; might be \ + start of illegal ]]> sequence */ /* The following tokens may be returned by both XmlPrologTok and XmlContentTok. */ -#define XML_TOK_NONE -4 /* The string to be scanned is empty */ -#define XML_TOK_TRAILING_CR \ - -3 /* A CR at the end of the scan; \ - might be part of CRLF sequence */ -#define XML_TOK_PARTIAL_CHAR -2 /* only part of a multibyte sequence */ -#define XML_TOK_PARTIAL -1 /* only part of a token */ -#define XML_TOK_INVALID 0 +# define XML_TOK_NONE -4 /* The string to be scanned is empty */ +# define XML_TOK_TRAILING_CR \ + -3 /* A CR at the end of the scan; \ + might be part of CRLF sequence */ +# define XML_TOK_PARTIAL_CHAR -2 /* only part of a multibyte sequence */ +# define XML_TOK_PARTIAL -1 /* only part of a token */ +# define XML_TOK_INVALID 0 /* The following tokens are returned by XmlContentTok; some are also returned by XmlAttributeValueTok, XmlEntityTok, XmlCdataSectionTok. */ -#define XML_TOK_START_TAG_WITH_ATTS 1 -#define XML_TOK_START_TAG_NO_ATTS 2 -#define XML_TOK_EMPTY_ELEMENT_WITH_ATTS 3 /* empty element tag */ -#define XML_TOK_EMPTY_ELEMENT_NO_ATTS 4 -#define XML_TOK_END_TAG 5 -#define XML_TOK_DATA_CHARS 6 -#define XML_TOK_DATA_NEWLINE 7 -#define XML_TOK_CDATA_SECT_OPEN 8 -#define XML_TOK_ENTITY_REF 9 -#define XML_TOK_CHAR_REF 10 /* numeric character reference */ +# define XML_TOK_START_TAG_WITH_ATTS 1 +# define XML_TOK_START_TAG_NO_ATTS 2 +# define XML_TOK_EMPTY_ELEMENT_WITH_ATTS 3 /* empty element tag */ +# define XML_TOK_EMPTY_ELEMENT_NO_ATTS 4 +# define XML_TOK_END_TAG 5 +# define XML_TOK_DATA_CHARS 6 +# define XML_TOK_DATA_NEWLINE 7 +# define XML_TOK_CDATA_SECT_OPEN 8 +# define XML_TOK_ENTITY_REF 9 +# define XML_TOK_CHAR_REF 10 /* numeric character reference */ /* The following tokens may be returned by both XmlPrologTok and XmlContentTok. */ -#define XML_TOK_PI 11 /* processing instruction */ -#define XML_TOK_XML_DECL 12 /* XML decl or text decl */ -#define XML_TOK_COMMENT 13 -#define XML_TOK_BOM 14 /* Byte order mark */ +# define XML_TOK_PI 11 /* processing instruction */ +# define XML_TOK_XML_DECL 12 /* XML decl or text decl */ +# define XML_TOK_COMMENT 13 +# define XML_TOK_BOM 14 /* Byte order mark */ /* The following tokens are returned only by XmlPrologTok */ -#define XML_TOK_PROLOG_S 15 -#define XML_TOK_DECL_OPEN 16 /* */ -#define XML_TOK_NAME 18 -#define XML_TOK_NMTOKEN 19 -#define XML_TOK_POUND_NAME 20 /* #name */ -#define XML_TOK_OR 21 /* | */ -#define XML_TOK_PERCENT 22 -#define XML_TOK_OPEN_PAREN 23 -#define XML_TOK_CLOSE_PAREN 24 -#define XML_TOK_OPEN_BRACKET 25 -#define XML_TOK_CLOSE_BRACKET 26 -#define XML_TOK_LITERAL 27 -#define XML_TOK_PARAM_ENTITY_REF 28 -#define XML_TOK_INSTANCE_START 29 +# define XML_TOK_PROLOG_S 15 +# define XML_TOK_DECL_OPEN 16 /* */ +# define XML_TOK_NAME 18 +# define XML_TOK_NMTOKEN 19 +# define XML_TOK_POUND_NAME 20 /* #name */ +# define XML_TOK_OR 21 /* | */ +# define XML_TOK_PERCENT 22 +# define XML_TOK_OPEN_PAREN 23 +# define XML_TOK_CLOSE_PAREN 24 +# define XML_TOK_OPEN_BRACKET 25 +# define XML_TOK_CLOSE_BRACKET 26 +# define XML_TOK_LITERAL 27 +# define XML_TOK_PARAM_ENTITY_REF 28 +# define XML_TOK_INSTANCE_START 29 /* The following occur only in element type declarations */ -#define XML_TOK_NAME_QUESTION 30 /* name? */ -#define XML_TOK_NAME_ASTERISK 31 /* name* */ -#define XML_TOK_NAME_PLUS 32 /* name+ */ -#define XML_TOK_COND_SECT_OPEN 33 /* */ -#define XML_TOK_CLOSE_PAREN_QUESTION 35 /* )? */ -#define XML_TOK_CLOSE_PAREN_ASTERISK 36 /* )* */ -#define XML_TOK_CLOSE_PAREN_PLUS 37 /* )+ */ -#define XML_TOK_COMMA 38 +# define XML_TOK_NAME_QUESTION 30 /* name? */ +# define XML_TOK_NAME_ASTERISK 31 /* name* */ +# define XML_TOK_NAME_PLUS 32 /* name+ */ +# define XML_TOK_COND_SECT_OPEN 33 /* */ +# define XML_TOK_CLOSE_PAREN_QUESTION 35 /* )? */ +# define XML_TOK_CLOSE_PAREN_ASTERISK 36 /* )* */ +# define XML_TOK_CLOSE_PAREN_PLUS 37 /* )+ */ +# define XML_TOK_COMMA 38 /* The following token is returned only by XmlAttributeValueTok */ -#define XML_TOK_ATTRIBUTE_VALUE_S 39 +# define XML_TOK_ATTRIBUTE_VALUE_S 39 /* The following token is returned only by XmlCdataSectionTok */ -#define XML_TOK_CDATA_SECT_CLOSE 40 +# define XML_TOK_CDATA_SECT_CLOSE 40 /* With namespace processing this is returned by XmlPrologTok for a name with a colon. */ -#define XML_TOK_PREFIXED_NAME 41 +# define XML_TOK_PREFIXED_NAME 41 -#ifdef XML_DTD -# define XML_TOK_IGNORE_SECT 42 -#endif /* XML_DTD */ +# ifdef XML_DTD +# define XML_TOK_IGNORE_SECT 42 +# endif /* XML_DTD */ -#ifdef XML_DTD -# define XML_N_STATES 4 -#else /* not XML_DTD */ -# define XML_N_STATES 3 -#endif /* not XML_DTD */ +# ifdef XML_DTD +# define XML_N_STATES 4 +# else /* not XML_DTD */ +# define XML_N_STATES 3 +# endif /* not XML_DTD */ -#define XML_PROLOG_STATE 0 -#define XML_CONTENT_STATE 1 -#define XML_CDATA_SECTION_STATE 2 -#ifdef XML_DTD -# define XML_IGNORE_SECTION_STATE 3 -#endif /* XML_DTD */ +# define XML_PROLOG_STATE 0 +# define XML_CONTENT_STATE 1 +# define XML_CDATA_SECTION_STATE 2 +# ifdef XML_DTD +# define XML_IGNORE_SECTION_STATE 3 +# endif /* XML_DTD */ -#define XML_N_LITERAL_TYPES 2 -#define XML_ATTRIBUTE_VALUE_LITERAL 0 -#define XML_ENTITY_VALUE_LITERAL 1 +# define XML_N_LITERAL_TYPES 2 +# define XML_ATTRIBUTE_VALUE_LITERAL 0 +# define XML_ENTITY_VALUE_LITERAL 1 /* The size of the buffer passed to XmlUtf8Encode must be at least this. */ -#define XML_UTF8_ENCODE_MAX 4 +# define XML_UTF8_ENCODE_MAX 4 /* The size of the buffer passed to XmlUtf16Encode must be at least this. */ -#define XML_UTF16_ENCODE_MAX 2 +# define XML_UTF16_ENCODE_MAX 2 typedef struct position { /* first line and first column are 0 not 1 */ - XML_Size lineNumber; - XML_Size columnNumber; + uint64_t lineNumber; + uint64_t columnNumber; } POSITION; typedef struct { @@ -159,41 +159,40 @@ typedef struct { struct encoding; typedef struct encoding ENCODING; -typedef int(PTRCALL *SCANNER)(const ENCODING *, const char *, const char *, - const char **); +typedef int (*SCANNER)(const ENCODING *, const char *, const char *, + const char **); enum XML_Convert_Result { XML_CONVERT_COMPLETED = 0, XML_CONVERT_INPUT_INCOMPLETE = 1, - XML_CONVERT_OUTPUT_EXHAUSTED - = 2 /* and therefore potentially input remaining as well */ + XML_CONVERT_OUTPUT_EXHAUSTED = 2 /* and therefore potentially input remaining + as well */ }; struct encoding { SCANNER scanners[XML_N_STATES]; SCANNER literalScanners[XML_N_LITERAL_TYPES]; - int(PTRCALL *nameMatchesAscii)(const ENCODING *, const char *, const char *, - const char *); - int(PTRFASTCALL *nameLength)(const ENCODING *, const char *); - const char *(PTRFASTCALL *skipS)(const ENCODING *, const char *); - int(PTRCALL *getAtts)(const ENCODING *enc, const char *ptr, int attsMax, - ATTRIBUTE *atts); - int(PTRFASTCALL *charRefNumber)(const ENCODING *enc, const char *ptr); - int(PTRCALL *predefinedEntityName)(const ENCODING *, const char *, - const char *); - void(PTRCALL *updatePosition)(const ENCODING *, const char *ptr, - const char *end, POSITION *); - int(PTRCALL *isPublicId)(const ENCODING *enc, const char *ptr, - const char *end, const char **badPtr); - enum XML_Convert_Result(PTRCALL *utf8Convert)(const ENCODING *enc, - const char **fromP, - const char *fromLim, char **toP, - const char *toLim); - enum XML_Convert_Result(PTRCALL *utf16Convert)(const ENCODING *enc, - const char **fromP, - const char *fromLim, - unsigned short **toP, - const unsigned short *toLim); + int (*nameMatchesAscii)(const ENCODING *, const char *, const char *, + const char *); + int (*nameLength)(const ENCODING *, const char *); + const char *(*skipS)(const ENCODING *, const char *); + int (*getAtts)(const ENCODING *enc, const char *ptr, int attsMax, + ATTRIBUTE *atts); + int (*charRefNumber)(const ENCODING *enc, const char *ptr); + int (*predefinedEntityName)(const ENCODING *, const char *, const char *); + void (*updatePosition)(const ENCODING *, const char *ptr, const char *end, + POSITION *); + int (*isPublicId)(const ENCODING *enc, const char *ptr, const char *end, + const char **badPtr); + enum XML_Convert_Result (*utf8Convert)(const ENCODING *enc, + const char **fromP, + const char *fromLim, char **toP, + const char *toLim); + enum XML_Convert_Result (*utf16Convert)(const ENCODING *enc, + const char **fromP, + const char *fromLim, + unsigned short **toP, + const unsigned short *toLim); int minBytesPerChar; char isUtf8; char isUtf16; @@ -220,63 +219,63 @@ struct encoding { the prolog outside literals, comments and processing instructions. */ -#define XmlTok(enc, state, ptr, end, nextTokPtr) \ - (((enc)->scanners[state])(enc, ptr, end, nextTokPtr)) +# define XmlTok(enc, state, ptr, end, nextTokPtr) \ + (((enc)->scanners[state])(enc, ptr, end, nextTokPtr)) -#define XmlPrologTok(enc, ptr, end, nextTokPtr) \ - XmlTok(enc, XML_PROLOG_STATE, ptr, end, nextTokPtr) +# define XmlPrologTok(enc, ptr, end, nextTokPtr) \ + XmlTok(enc, XML_PROLOG_STATE, ptr, end, nextTokPtr) -#define XmlContentTok(enc, ptr, end, nextTokPtr) \ - XmlTok(enc, XML_CONTENT_STATE, ptr, end, nextTokPtr) +# define XmlContentTok(enc, ptr, end, nextTokPtr) \ + XmlTok(enc, XML_CONTENT_STATE, ptr, end, nextTokPtr) -#define XmlCdataSectionTok(enc, ptr, end, nextTokPtr) \ - XmlTok(enc, XML_CDATA_SECTION_STATE, ptr, end, nextTokPtr) +# define XmlCdataSectionTok(enc, ptr, end, nextTokPtr) \ + XmlTok(enc, XML_CDATA_SECTION_STATE, ptr, end, nextTokPtr) -#ifdef XML_DTD +# ifdef XML_DTD -# define XmlIgnoreSectionTok(enc, ptr, end, nextTokPtr) \ - XmlTok(enc, XML_IGNORE_SECTION_STATE, ptr, end, nextTokPtr) +# define XmlIgnoreSectionTok(enc, ptr, end, nextTokPtr) \ + XmlTok(enc, XML_IGNORE_SECTION_STATE, ptr, end, nextTokPtr) -#endif /* XML_DTD */ +# endif /* XML_DTD */ /* This is used for performing a 2nd-level tokenization on the content of a literal that has already been returned by XmlTok. */ -#define XmlLiteralTok(enc, literalType, ptr, end, nextTokPtr) \ - (((enc)->literalScanners[literalType])(enc, ptr, end, nextTokPtr)) +# define XmlLiteralTok(enc, literalType, ptr, end, nextTokPtr) \ + (((enc)->literalScanners[literalType])(enc, ptr, end, nextTokPtr)) -#define XmlAttributeValueTok(enc, ptr, end, nextTokPtr) \ - XmlLiteralTok(enc, XML_ATTRIBUTE_VALUE_LITERAL, ptr, end, nextTokPtr) +# define XmlAttributeValueTok(enc, ptr, end, nextTokPtr) \ + XmlLiteralTok(enc, XML_ATTRIBUTE_VALUE_LITERAL, ptr, end, nextTokPtr) -#define XmlEntityValueTok(enc, ptr, end, nextTokPtr) \ - XmlLiteralTok(enc, XML_ENTITY_VALUE_LITERAL, ptr, end, nextTokPtr) +# define XmlEntityValueTok(enc, ptr, end, nextTokPtr) \ + XmlLiteralTok(enc, XML_ENTITY_VALUE_LITERAL, ptr, end, nextTokPtr) -#define XmlNameMatchesAscii(enc, ptr1, end1, ptr2) \ - (((enc)->nameMatchesAscii)(enc, ptr1, end1, ptr2)) +# define XmlNameMatchesAscii(enc, ptr1, end1, ptr2) \ + (((enc)->nameMatchesAscii)(enc, ptr1, end1, ptr2)) -#define XmlNameLength(enc, ptr) (((enc)->nameLength)(enc, ptr)) +# define XmlNameLength(enc, ptr) (((enc)->nameLength)(enc, ptr)) -#define XmlSkipS(enc, ptr) (((enc)->skipS)(enc, ptr)) +# define XmlSkipS(enc, ptr) (((enc)->skipS)(enc, ptr)) -#define XmlGetAttributes(enc, ptr, attsMax, atts) \ - (((enc)->getAtts)(enc, ptr, attsMax, atts)) +# define XmlGetAttributes(enc, ptr, attsMax, atts) \ + (((enc)->getAtts)(enc, ptr, attsMax, atts)) -#define XmlCharRefNumber(enc, ptr) (((enc)->charRefNumber)(enc, ptr)) +# define XmlCharRefNumber(enc, ptr) (((enc)->charRefNumber)(enc, ptr)) -#define XmlPredefinedEntityName(enc, ptr, end) \ - (((enc)->predefinedEntityName)(enc, ptr, end)) +# define XmlPredefinedEntityName(enc, ptr, end) \ + (((enc)->predefinedEntityName)(enc, ptr, end)) -#define XmlUpdatePosition(enc, ptr, end, pos) \ - (((enc)->updatePosition)(enc, ptr, end, pos)) +# define XmlUpdatePosition(enc, ptr, end, pos) \ + (((enc)->updatePosition)(enc, ptr, end, pos)) -#define XmlIsPublicId(enc, ptr, end, badPtr) \ - (((enc)->isPublicId)(enc, ptr, end, badPtr)) +# define XmlIsPublicId(enc, ptr, end, badPtr) \ + (((enc)->isPublicId)(enc, ptr, end, badPtr)) -#define XmlUtf8Convert(enc, fromP, fromLim, toP, toLim) \ - (((enc)->utf8Convert)(enc, fromP, fromLim, toP, toLim)) +# define XmlUtf8Convert(enc, fromP, fromLim, toP, toLim) \ + (((enc)->utf8Convert)(enc, fromP, fromLim, toP, toLim)) -#define XmlUtf16Convert(enc, fromP, fromLim, toP, toLim) \ - (((enc)->utf16Convert)(enc, fromP, fromLim, toP, toLim)) +# define XmlUtf16Convert(enc, fromP, fromLim, toP, toLim) \ + (((enc)->utf16Convert)(enc, fromP, fromLim, toP, toLim)) typedef struct { ENCODING initEnc; @@ -293,13 +292,13 @@ int XmlInitEncoding(INIT_ENCODING *p, const ENCODING **encPtr, const char *name); const ENCODING *XmlGetUtf8InternalEncoding(void); const ENCODING *XmlGetUtf16InternalEncoding(void); -int FASTCALL XmlUtf8Encode(int charNumber, char *buf); -int FASTCALL XmlUtf16Encode(int charNumber, unsigned short *buf); +int XmlUtf8Encode(int charNumber, char *buf); +int XmlUtf16Encode(int charNumber, unsigned short *buf); int XmlSizeOfUnknownEncoding(void); typedef int(XMLCALL *CONVERTER)(void *userData, const char *p); -ENCODING *XmlInitUnknownEncoding(void *mem, int *table, CONVERTER convert, +ENCODING *XmlInitUnknownEncoding(void *mem, const int *table, CONVERTER convert, void *userData); int XmlParseXmlDeclNS(int isGeneralTextEntity, const ENCODING *enc, @@ -312,10 +311,7 @@ int XmlInitEncodingNS(INIT_ENCODING *p, const ENCODING **encPtr, const char *name); const ENCODING *XmlGetUtf8InternalEncodingNS(void); const ENCODING *XmlGetUtf16InternalEncodingNS(void); -ENCODING *XmlInitUnknownEncodingNS(void *mem, int *table, CONVERTER convert, - void *userData); -#ifdef __cplusplus -} -#endif +ENCODING *XmlInitUnknownEncodingNS(void *mem, const int *table, + CONVERTER convert, void *userData); #endif /* not XmlTok_INCLUDED */ diff --git a/Modules/expat/xmltok_impl.c b/Modules/expat/xmltok_impl.c index 239a2d06c4512ce..7900e94701e2768 100644 --- a/Modules/expat/xmltok_impl.c +++ b/Modules/expat/xmltok_impl.c @@ -1,4 +1,4 @@ -/* This file is included (from xmltok.c, 1-3 times depending on XML_MIN_SIZE)! +/* This file is included (from xmltok.c, 3 times)! __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -10,13 +10,15 @@ Copyright (c) 2000 Clark Cooper Copyright (c) 2002 Fred L. Drake, Jr. Copyright (c) 2002-2016 Karl Waclawek - Copyright (c) 2016-2022 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2017 Rhodri James Copyright (c) 2018 Benjamin Peterson Copyright (c) 2018 Anton Maklakov Copyright (c) 2019 David Loffredo Copyright (c) 2020 Boris Kolpackov Copyright (c) 2022 Martin Ettl + Copyright (c) 2026 Nick Begg + Copyright (c) 2026 Kartik Kenchi Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -37,14 +39,12 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifdef XML_TOK_IMPL_C -# ifndef IS_INVALID_CHAR // i.e. for UTF-16 and XML_MIN_SIZE not defined -# define IS_INVALID_CHAR(enc, ptr, n) (0) -# endif - # define INVALID_LEAD_CASE(n, ptr, nextTokPtr) \ case BT_LEAD##n: \ if (end - ptr < n) \ @@ -83,7 +83,7 @@ *nextTokPtr = ptr; \ return XML_TOK_INVALID; \ } \ - /* fall through */ \ + EXPAT_FALLTHROUGH; \ case BT_NMSTRT: \ case BT_HEX: \ case BT_DIGIT: \ @@ -112,7 +112,7 @@ *nextTokPtr = ptr; \ return XML_TOK_INVALID; \ } \ - /* fall through */ \ + EXPAT_FALLTHROUGH; \ case BT_NMSTRT: \ case BT_HEX: \ ptr += MINBPC(enc); \ @@ -141,7 +141,7 @@ /* ptr points to character following "= end) @@ -427,7 +427,7 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr, const char *end, /* ptr points to character following "= end) @@ -898,7 +898,7 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_INVALID; } } - /* fall through */ + EXPAT_FALLTHROUGH; case BT_AMP: case BT_LT: case BT_NONXML: @@ -919,7 +919,7 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end, /* ptr points to character following "%" */ -static int PTRCALL +static int PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { REQUIRE_CHAR(enc, ptr, end); @@ -949,7 +949,7 @@ PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_PARTIAL; } -static int PTRCALL +static int PREFIX(scanPoundName)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { REQUIRE_CHAR(enc, ptr, end); @@ -979,7 +979,7 @@ PREFIX(scanPoundName)(const ENCODING *enc, const char *ptr, const char *end, return -XML_TOK_POUND_NAME; } -static int PTRCALL +static int PREFIX(scanLit)(int open, const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { while (HAS_CHAR(enc, ptr, end)) { @@ -1013,7 +1013,7 @@ PREFIX(scanLit)(int open, const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_PARTIAL; } -static int PTRCALL +static int PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { int tok; @@ -1059,7 +1059,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, /* indicate that this might be part of a CR/LF pair */ return -XML_TOK_PROLOG_S; } - /* fall through */ + EXPAT_FALLTHROUGH; case BT_S: case BT_LF: for (;;) { @@ -1074,7 +1074,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, /* don't split CR/LF pair */ if (ptr + MINBPC(enc) != end) break; - /* fall through */ + EXPAT_FALLTHROUGH; default: *nextTokPtr = ptr; return XML_TOK_PROLOG_S; @@ -1189,7 +1189,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, tok = XML_TOK_NMTOKEN; break; } - /* fall through */ + EXPAT_FALLTHROUGH; default: *nextTokPtr = ptr; return XML_TOK_INVALID; @@ -1257,7 +1257,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, return -tok; } -static int PTRCALL +static int PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { const char *start; @@ -1326,7 +1326,7 @@ PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_DATA_CHARS; } -static int PTRCALL +static int PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { const char *start; @@ -1393,7 +1393,7 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr, const char *end, # ifdef XML_DTD -static int PTRCALL +static int PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { int level = 0; @@ -1445,7 +1445,7 @@ PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr, const char *end, # endif /* XML_DTD */ -static int PTRCALL +static int PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end, const char **badPtr) { ptr += MINBPC(enc); @@ -1484,7 +1484,7 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end, case BT_NMSTRT: if (! (BYTE_TO_ASCII(enc, ptr) & ~0x7f)) break; - /* fall through */ + EXPAT_FALLTHROUGH; default: switch (BYTE_TO_ASCII(enc, ptr)) { case 0x24: /* $ */ @@ -1505,7 +1505,7 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end, first attsMax attributes are stored in atts. */ -static int PTRCALL +static int PREFIX(getAtts)(const ENCODING *enc, const char *ptr, int attsMax, ATTRIBUTE *atts) { enum { other, inName, inValue } state = inName; @@ -1598,7 +1598,7 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr, int attsMax, /* not reached */ } -static int PTRFASTCALL +static int PREFIX(charRefNumber)(const ENCODING *enc, const char *ptr) { int result = 0; /* skip &# */ @@ -1656,7 +1656,7 @@ PREFIX(charRefNumber)(const ENCODING *enc, const char *ptr) { return checkCharRefNumber(result); } -static int PTRCALL +static int PREFIX(predefinedEntityName)(const ENCODING *enc, const char *ptr, const char *end) { UNUSED_P(enc); @@ -1710,7 +1710,7 @@ PREFIX(predefinedEntityName)(const ENCODING *enc, const char *ptr, return 0; } -static int PTRCALL +static int PREFIX(nameMatchesAscii)(const ENCODING *enc, const char *ptr1, const char *end1, const char *ptr2) { UNUSED_P(enc); @@ -1729,7 +1729,7 @@ PREFIX(nameMatchesAscii)(const ENCODING *enc, const char *ptr1, return ptr1 == end1; } -static int PTRFASTCALL +static int PREFIX(nameLength)(const ENCODING *enc, const char *ptr) { const char *start = ptr; for (;;) { @@ -1759,7 +1759,7 @@ PREFIX(nameLength)(const ENCODING *enc, const char *ptr) { } } -static const char *PTRFASTCALL +static const char * PREFIX(skipS)(const ENCODING *enc, const char *ptr) { for (;;) { switch (BYTE_TYPE(enc, ptr)) { @@ -1774,7 +1774,7 @@ PREFIX(skipS)(const ENCODING *enc, const char *ptr) { } } -static void PTRCALL +static void PREFIX(updatePosition)(const ENCODING *enc, const char *ptr, const char *end, POSITION *pos) { while (HAS_CHAR(enc, ptr, end)) { diff --git a/Modules/expat/xmltok_impl.h b/Modules/expat/xmltok_impl.h index 3469c4ae138c95a..13a08899da3b1a5 100644 --- a/Modules/expat/xmltok_impl.h +++ b/Modules/expat/xmltok_impl.h @@ -29,6 +29,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ enum { diff --git a/Modules/expat/xmltok_ns.c b/Modules/expat/xmltok_ns.c index fbdd3e3c7b79996..d9f69abb92ef39f 100644 --- a/Modules/expat/xmltok_ns.c +++ b/Modules/expat/xmltok_ns.c @@ -11,7 +11,8 @@ Copyright (c) 2002 Greg Stein Copyright (c) 2002 Fred L. Drake, Jr. Copyright (c) 2002-2006 Karl Waclawek - Copyright (c) 2017-2021 Sebastian Pipping + Copyright (c) 2017-2026 Sebastian Pipping + Copyright (c) 2025 Alfonso Gregory Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -32,6 +33,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifdef XML_TOK_NS_C @@ -61,14 +64,14 @@ static const ENCODING *const NS(encodings)[] = { &ns(utf8_encoding).enc /* NO_ENC */ }; -static int PTRCALL +static int NS(initScanProlog)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { return initScan(NS(encodings), (const INIT_ENCODING *)enc, XML_PROLOG_STATE, ptr, end, nextTokPtr); } -static int PTRCALL +static int NS(initScanContent)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { return initScan(NS(encodings), (const INIT_ENCODING *)enc, XML_CONTENT_STATE, @@ -98,13 +101,13 @@ NS(findEncoding)(const ENCODING *enc, const char *ptr, const char *end) { int i; XmlUtf8Convert(enc, &ptr, end, &p, p + ENCODING_MAX - 1); if (ptr != end) - return 0; + return NULL; *p = 0; if (streqci(buf, KW_UTF_16) && enc->minBytesPerChar == 2) return enc; i = getEncodingIndex(buf); if (i == UNKNOWN_ENC) - return 0; + return NULL; return NS(encodings)[i]; } diff --git a/PCbuild/_elementtree.vcxproj b/PCbuild/_elementtree.vcxproj index 511e26bb60dbd86..5a49a9553ed8893 100644 --- a/PCbuild/_elementtree.vcxproj +++ b/PCbuild/_elementtree.vcxproj @@ -74,19 +74,23 @@ + + - + + + diff --git a/PCbuild/_elementtree.vcxproj.filters b/PCbuild/_elementtree.vcxproj.filters index 6acdf35846ab149..b8a02b047dd5840 100644 --- a/PCbuild/_elementtree.vcxproj.filters +++ b/PCbuild/_elementtree.vcxproj.filters @@ -24,6 +24,12 @@ Header Files + + Header Files + + + Header Files + Header Files @@ -33,7 +39,7 @@ Header Files - + Header Files @@ -48,6 +54,9 @@ Header Files + + Header Files + Header Files @@ -59,6 +68,9 @@ Source Files + + Source Files + Source Files diff --git a/setup.py b/setup.py index 927572c949cbe78..ba2e81f415697e4 100644 --- a/setup.py +++ b/setup.py @@ -1508,7 +1508,8 @@ class db_found(Exception): pass ('XML_POOR_ENTROPY', '1'), ] expat_lib = [] - expat_sources = ['expat/xmlparse.c', + expat_sources = ['expat/xcs.c', + 'expat/xmlparse.c', 'expat/xmlrole.c', 'expat/xmltok.c'] expat_depends = ['expat/ascii.h', @@ -1519,6 +1520,7 @@ class db_found(Exception): pass 'expat/internal.h', 'expat/latin1tab.h', 'expat/utf8tab.h', + 'expat/xcs.h', 'expat/xmlrole.h', 'expat/xmltok.h', 'expat/xmltok_impl.h' From 784022998c921aac983502fa9c6850189daff261 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Wed, 28 Oct 2026 10:32:36 -0500 Subject: [PATCH 14/23] gh-156293: Do not call the SNI callback through a released SSLContext (CVE-2026-19445) Backport of python/cpython gh-156293 (87665c9b1f; 3.10: ec44b5a3258) to 2.7.18.x. set_servername_callback() registered the SSLContext as a borrowed pointer in the SSL_CTX's servername arg, and _servername_callback() used that pointer to find the Python callback. When the callback switches the connection to another context (sslsock.context = other), the original SSLContext can be deallocated while its SSL_CTX lives on as the connection's session_ctx. The callback then reads freed memory: right after the Python callback returns (if it dropped the last reference and raised), or on a later ClientHello such as one after a HelloRetryRequest. On 2.7.18.15, a callback that switches context, drops it and raises segfaults the server. Under captured stderr it reports "in ignored", read from the freed object. _servername_callback() now finds the context through the SSL object's app data (the PySSLSocket and its current, live context), and holds strong references to that context and its callback while they are used. set_servername_callback() no longer registers the servername arg, and context_dealloc() unregisters the servername callback before freeing the SSL_CTX. As upstream, after the callback switches contexts, any further ClientHello on that connection goes to the new context's callback. 2.7.18.x: the legacy set_servername_callback()/set_hostname is the only API (there is no sni_callback property); no Py_NewRef; context_dealloc's unregister is guarded by HAVE_SNI like the rest of the SNI code. 2.7 has no MemoryBIO, so the tests run the handshake over a socketpair with the client in a thread. The "callback raises" test also checks that the unraisable error names the callback, which is what fails on 2.7.18.15 (""). The HelloRetryRequest test passes on 2.7.18.15 in this release build; it is kept, as upstream, as a regression guard. Co-Authored-By: Claude Opus 5.5 --- Lib/test/test_ssl.py | 98 ++++++++++++++++++++++++++++++++++++++++++++ Modules/_ssl.c | 41 +++++++++++++----- 2 files changed, 129 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 2381baa847cd7bd..57ee6c37f0ce7e9 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -3304,6 +3304,104 @@ def cb_wrong_return_type(ssl_sock, server_name, initial_context): self.assertEqual(cm.exception.reason, 'TLSV1_ALERT_INTERNAL_ERROR') self.assertIn("TypeError", stderr.getvalue()) + def sni_handshake_released_context(self, sni_cb, ecdh_curve=None): + # Run a handshake over a socketpair whose server side was wrapped + # by a context that only the C-level SSL object references once + # sni_cb has switched the connection to another context. + # Returns (server socket, server handshake error or None, + # weakref to the released context). + def make_server(sock): + dispatch_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + dispatch_ctx.load_cert_chain(SIGNED_CERTFILE) + if ecdh_curve is not None: + dispatch_ctx.set_ecdh_curve(ecdh_curve) + dispatch_ctx.set_servername_callback(sni_cb) + server = dispatch_ctx.wrap_socket( + sock, server_side=True, do_handshake_on_connect=False) + return server, weakref.ref(dispatch_ctx) + + s1, s2 = [socket.socket(_sock=s) for s in socket.socketpair()] + self.addCleanup(s1.close) + self.addCleanup(s2.close) + server, dispatch_ref = make_server(s1) + client_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + client = client_ctx.wrap_socket(s2, server_hostname='localhost', + do_handshake_on_connect=False) + self.addCleanup(client.close) + self.addCleanup(server.close) + client_thread = threading.Thread(target=self._quiet_handshake, + args=(client,)) + client_thread.start() + error = None + try: + server.do_handshake() + except ssl.SSLError as e: + error = e + finally: + if error is not None: + s1.shutdown(socket.SHUT_RDWR) + client_thread.join() + gc.collect() + return server, error, dispatch_ref + + @staticmethod + def _quiet_handshake(sock): + try: + sock.do_handshake() + except (ssl.SSLError, socket.error): + pass + + @needs_sni + def test_sni_callback_context_released_and_callback_raises(self): + # CVE-2026-19445: the callback switches the connection to another + # context, drops the last reference to the context that carries + # it, and raises. The C callback must not touch that context + # after the Python callback returned. + leaf_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + leaf_ctx.load_cert_chain(SIGNED_CERTFILE) + + def sni_cb(ssl_sock, server_name, ctx): + ssl_sock.context = leaf_ctx + del ctx + raise LookupError("no certificate for %r" % (server_name,)) + + with support.captured_stderr() as stderr: + server, error, dispatch_ref = \ + self.sni_handshake_released_context(sni_cb) + self.assertIsInstance(error, ssl.SSLError) + self.assertIn("LookupError", stderr.getvalue()) + # The unraisable error names the callback. Read through the + # freed context it was (and writing that to a real stderr + # crashed). + self.assertIn("sni_cb", stderr.getvalue()) + self.assertIs(server.context, leaf_ctx) + self.assertIsNone(dispatch_ref()) + + @needs_sni + @unittest.skipUnless(ssl.HAS_TLSv1_3, "needs TLS 1.3") + def test_sni_callback_context_released_before_second_client_hello(self): + # CVE-2026-19445: after the callback switched the connection to + # another context, the server forces a HelloRetryRequest (the + # client offers an X25519 key share first, the server accepts + # only P-384). The second ClientHello must not reach the + # deallocated context through the original SSL_CTX. + leaf_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + leaf_ctx.load_cert_chain(SIGNED_CERTFILE) + calls = [] + + def sni_cb(ssl_sock, server_name, ctx): + calls.append(server_name) + ssl_sock.context = leaf_ctx + + server, error, dispatch_ref = \ + self.sni_handshake_released_context(sni_cb, "secp384r1") + self.assertIsNone(error) + self.assertIsNone(dispatch_ref()) + self.assertGreaterEqual(len(calls), 1) + self.assertEqual(calls[0], 'localhost') + self.assertIs(server.context, leaf_ctx) + self.assertIsNotNone(server.cipher()) + def test_read_write_after_close_raises_valuerror(self): context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) context.verify_mode = ssl.CERT_REQUIRED diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 0df84021d3814a6..c8f07f92b2622e3 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -2413,6 +2413,11 @@ context_dealloc(PySSLContext *self) /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(self); context_clear(self); +#if HAVE_SNI && !defined(OPENSSL_NO_TLSEXT) + /* The SSL_CTX may outlive this object as the session_ctx of sockets that + were switched to another context; leave no Python callback behind. */ + SSL_CTX_set_tlsext_servername_callback(self->ctx, NULL); +#endif SSL_CTX_free(self->ctx); #if HAVE_NPN PyMem_FREE(self->npn_protocols); @@ -3430,29 +3435,42 @@ static int _servername_callback(SSL *s, int *al, void *args) { int ret; - PySSLContext *ssl_ctx = (PySSLContext *) args; + PySSLContext *ssl_ctx; PySSLSocket *ssl; PyObject *servername_o; PyObject *servername_idna; PyObject *result; /* The high-level ssl.SSLSocket object */ PyObject *ssl_socket; + PyObject *sni_cb; const char *servername = SSL_get_servername(s, TLSEXT_NAMETYPE_host_name); #ifdef WITH_THREAD PyGILState_STATE gstate = PyGILState_Ensure(); #endif - if (ssl_ctx->set_hostname == NULL) { - /* remove race condition in this the call back while if removing the - * callback is in progress */ + /* Do not use the SSL_CTX's servername arg to find the context: it is a + borrowed pointer to whichever _SSLContext installed the callback, and + that object may already be gone while OpenSSL still reaches this + callback through the connection's session_ctx (e.g. on the second + ClientHello after a HelloRetryRequest, once the callback has switched + the socket to another context). The socket's current context is + always alive; hold strong references to it and to the callback while + they are used here. */ + (void)args; + ssl = SSL_get_app_data(s); + assert(ssl != NULL && PySSLSocket_Check(ssl)); + ssl_ctx = ssl->ctx; + Py_INCREF(ssl_ctx); + sni_cb = ssl_ctx->set_hostname; + if (sni_cb == NULL) { + Py_DECREF(ssl_ctx); #ifdef WITH_THREAD PyGILState_Release(gstate); #endif return SSL_TLSEXT_ERR_OK; } + Py_INCREF(sni_cb); - ssl = SSL_get_app_data(s); - assert(PySSLSocket_Check(ssl)); if (ssl->ssl_sock == NULL) { ssl_socket = Py_None; } else { @@ -3464,7 +3482,7 @@ _servername_callback(SSL *s, int *al, void *args) } if (servername == NULL) { - result = PyObject_CallFunctionObjArgs(ssl_ctx->set_hostname, ssl_socket, + result = PyObject_CallFunctionObjArgs(sni_cb, ssl_socket, Py_None, ssl_ctx, NULL); } else { @@ -3480,14 +3498,14 @@ _servername_callback(SSL *s, int *al, void *args) goto error; } Py_DECREF(servername_o); - result = PyObject_CallFunctionObjArgs(ssl_ctx->set_hostname, ssl_socket, + result = PyObject_CallFunctionObjArgs(sni_cb, ssl_socket, servername_idna, ssl_ctx, NULL); Py_DECREF(servername_idna); } Py_DECREF(ssl_socket); if (result == NULL) { - PyErr_WriteUnraisable(ssl_ctx->set_hostname); + PyErr_WriteUnraisable(sni_cb); *al = SSL_AD_HANDSHAKE_FAILURE; ret = SSL_TLSEXT_ERR_ALERT_FATAL; } @@ -3506,6 +3524,8 @@ _servername_callback(SSL *s, int *al, void *args) Py_DECREF(result); } + Py_DECREF(sni_cb); + Py_DECREF(ssl_ctx); #ifdef WITH_THREAD PyGILState_Release(gstate); #endif @@ -3513,6 +3533,8 @@ _servername_callback(SSL *s, int *al, void *args) error: Py_DECREF(ssl_socket); + Py_DECREF(sni_cb); + Py_DECREF(ssl_ctx); *al = SSL_AD_INTERNAL_ERROR; ret = SSL_TLSEXT_ERR_ALERT_FATAL; #ifdef WITH_THREAD @@ -3555,7 +3577,6 @@ set_servername_callback(PySSLContext *self, PyObject *args) Py_INCREF(cb); self->set_hostname = cb; SSL_CTX_set_tlsext_servername_callback(self->ctx, _servername_callback); - SSL_CTX_set_tlsext_servername_arg(self->ctx, self); } Py_RETURN_NONE; #else From 4ea11bd2711230b3bfb5b66c060953b3429ddc93 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Wed, 28 Oct 2026 10:35:34 -0500 Subject: [PATCH 15/23] gh-149079: Fix O(n^2) canonical ordering in unicodedata.normalize() (CVE-2026-3276) Backport of python/cpython gh-149079 (991224b1e8; 3.10: db744c0776c) to 2.7.18.x. nfd_nfkd(), which every normalization form goes through, put combining characters into canonical order with an in-place insertion sort. A long run of combining marks in alternating canonical combining class order made it quadratic. On 2.7.18.15, NFC of u"a" + u"\u0300\u0327" * n takes 0.22 s for n=20000, 1.6 s for n=50000 and 19.9 s for n=200000. Each run of non-starters is now sorted on its own: insertion sort for runs shorter than 20, otherwise a stable counting sort on the combining class through a reusable buffer, and runs already in order are skipped. The same input now takes 0.001 s, 0.004 s and 0.015 s. 2.7.18.x: the sort works on the Py_UNICODE result buffer instead of PyUnicode_READ/WRITE. On narrow (UCS-2) builds it therefore sees UTF-16 code units, as the old sort did; surrogates have combining class 0, so they end runs as before. C89 declarations. The buffer grows through PyMem_Realloc rather than upstream's PyMem_Resize, which in 2.7 overwrites its pointer argument and would leak the old buffer on failure. Tests: upstream's test_long_combining_mark_run and test_combining_mark_run_fast_paths, plus test_combining_mark_run_is_linear (a 100000-mark run through the counting sort). On 20000 random strings for each of three seeds, mostly combining marks with runs of 1-200, NFC/NFD/NFKC/NFKD from both unicodedata and ucd_3_2_0 hash identically to 2.7.18.15. Co-Authored-By: Claude Opus 5.5 --- Lib/test/test_unicodedata.py | 38 ++++++++++++ Modules/unicodedata.c | 115 +++++++++++++++++++++++++++++------ 2 files changed, 135 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 11f2cda82099868..cd406d90ce3269b 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -204,6 +204,44 @@ def test_issue10254(self): b = u'C\u0338' * 20 + u'\xC7' self.assertEqual(self.db.normalize('NFC', a), b) + def test_long_combining_mark_run(self): + # gh-149079 (CVE-2026-3276): avoid quadratic canonical ordering. + payload = u"a" + (u"\u0300\u0327" * 32) + nfd = u"a" + (u"\u0327" * 32) + (u"\u0300" * 32) + nfc = u"\u00e0" + (u"\u0327" * 32) + (u"\u0300" * 31) + + self.assertEqual(self.db.normalize("NFD", payload), nfd) + self.assertEqual(self.db.normalize("NFKD", payload), nfd) + self.assertEqual(self.db.normalize("NFC", payload), nfc) + self.assertEqual(self.db.normalize("NFKC", payload), nfc) + + def test_combining_mark_run_fast_paths(self): + # gh-149079: cover short runs and already-sorted long runs. + short_payload = u"a" + (u"\u0300\u0327" * 9) + u"\u0300" + short_nfd = u"a" + (u"\u0327" * 9) + (u"\u0300" * 10) + short_nfc = u"\u00e0" + (u"\u0327" * 9) + (u"\u0300" * 9) + long_sorted = u"a" + (u"\u0327" * 30) + (u"\u0300" * 30) + long_sorted_nfc = u"\u00e0" + (u"\u0327" * 30) + (u"\u0300" * 29) + + self.assertEqual(self.db.normalize("NFD", short_payload), short_nfd) + self.assertEqual(self.db.normalize("NFKD", short_payload), short_nfd) + self.assertEqual(self.db.normalize("NFC", short_payload), short_nfc) + self.assertEqual(self.db.normalize("NFKC", short_payload), short_nfc) + self.assertEqual(self.db.normalize("NFD", long_sorted), long_sorted) + self.assertEqual(self.db.normalize("NFKD", long_sorted), long_sorted) + self.assertEqual(self.db.normalize("NFC", long_sorted), long_sorted_nfc) + self.assertEqual(self.db.normalize("NFKC", long_sorted), long_sorted_nfc) + + def test_combining_mark_run_is_linear(self): + # gh-149079 (CVE-2026-3276): a run of 100000 combining marks in + # alternating canonical combining class order took about 1.5 s with + # the quadratic sort (and 20 s for 400000); it now takes a few + # milliseconds. This checks the counting sort on a long run. + n = 50000 + payload = u"a" + (u"\u0300\u0327" * n) + nfd = u"a" + (u"\u0327" * n) + (u"\u0300" * n) + self.assertEqual(self.db.normalize("NFD", payload), nfd) + def test_issue29456(self): # Fix #29456 u1176_str_a = u'\u1100\u1176\u11a8' diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index df6ffe343c3f1c5..aedf41f7d44f4d1 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -493,6 +493,56 @@ get_decomp_record(PyObject *self, Py_UCS4 code, int *index, int *prefix, int *co #define NCount (VCount*TCount) #define SCount (LCount*NCount) +/* Small combining runs are usually cheaper with insertion sort. */ +#define CANONICAL_ORDERING_COUNTING_SORT_THRESHOLD 20 + +static void +canonical_ordering_sort_insertion(Py_UNICODE *start, Py_UNICODE *end) +{ + Py_UNICODE *i, *j; + + for (i = start + 1; i < end; i++) { + Py_UNICODE code = *i; + unsigned char combining = _getrecord_ex(code)->combining; + + j = i; + while (j > start) { + if (_getrecord_ex(j[-1])->combining <= combining) + break; + *j = j[-1]; + j--; + } + if (j != i) + *j = code; + } +} + +static void +canonical_ordering_sort_counting(Py_UNICODE *start, Py_UNICODE *end, + Py_UNICODE *sortbuf) +{ + Py_ssize_t counts[256]; + Py_ssize_t total = 0; + Py_UNICODE *i; + int c; + + memset(counts, 0, sizeof(counts)); + for (i = start; i < end; i++) + counts[_getrecord_ex(*i)->combining]++; + + for (c = 0; c < 256; c++) { + Py_ssize_t count = counts[c]; + counts[c] = total; + total += count; + } + + /* Reuse counts[] as the next output slot for each CCC; this is a + stable sort, as canonical ordering requires. */ + for (i = start; i < end; i++) + sortbuf[counts[_getrecord_ex(*i)->combining]++] = *i; + memcpy(start, sortbuf, (end - start) * sizeof(Py_UNICODE)); +} + static PyObject* nfd_nfkd(PyObject *self, PyObject *input, int k) { @@ -503,6 +553,8 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) Py_ssize_t space, isize; int index, prefix, count, stackptr; unsigned char prev, cur; + Py_UNICODE *sortbuf = NULL; + Py_ssize_t sortbuflen = 0; stackptr = 0; isize = PyUnicode_GET_SIZE(input); @@ -581,31 +633,58 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) /* Drop overallocation. Cannot fail. */ PyUnicode_Resize(&result, PyUnicode_GET_SIZE(result) - space); - /* Sort canonically. */ + /* Sort each consecutive combining-character run canonically. The + old in-place insertion sort was O(n**2) in the run length + (gh-149079, CVE-2026-3276). */ i = PyUnicode_AS_UNICODE(result); - prev = _getrecord_ex(*i)->combining; end = i + PyUnicode_GET_SIZE(result); - for (i++; i < end; i++) { - cur = _getrecord_ex(*i)->combining; - if (prev == 0 || cur == 0 || prev <= cur) { - prev = cur; + while (i < end) { + Py_UNICODE *run_start; + Py_ssize_t run_length; + int needs_sort = 0; + + prev = _getrecord_ex(*i)->combining; + if (prev == 0) { + i++; continue; } - /* Non-canonical order. Need to switch *i with previous. */ - o = i - 1; - while (1) { - Py_UNICODE tmp = o[1]; - o[1] = o[0]; - o[0] = tmp; - o--; - if (o < PyUnicode_AS_UNICODE(result)) - break; - prev = _getrecord_ex(*o)->combining; - if (prev == 0 || prev <= cur) + + run_start = i++; + while (i < end) { + cur = _getrecord_ex(*i)->combining; + if (cur == 0) break; + if (prev > cur) + needs_sort = 1; + prev = cur; + i++; } - prev = _getrecord_ex(*i)->combining; + if (!needs_sort) + continue; + + run_length = i - run_start; + if (run_length < CANONICAL_ORDERING_COUNTING_SORT_THRESHOLD) { + canonical_ordering_sort_insertion(run_start, i); + continue; + } + + if (run_length > sortbuflen) { + Py_UNICODE *new_sortbuf = NULL; + if ((size_t)run_length <= PY_SSIZE_T_MAX / sizeof(Py_UNICODE)) + new_sortbuf = (Py_UNICODE *)PyMem_Realloc( + sortbuf, run_length * sizeof(Py_UNICODE)); + if (new_sortbuf == NULL) { + PyErr_NoMemory(); + PyMem_Free(sortbuf); + Py_DECREF(result); + return NULL; + } + sortbuf = new_sortbuf; + sortbuflen = run_length; + } + canonical_ordering_sort_counting(run_start, i, sortbuf); } + PyMem_Free(sortbuf); return result; } From 386508c6c1d4060db83d563aea31f3ad7e09a6a4 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Wed, 28 Oct 2026 10:38:42 -0500 Subject: [PATCH 16/23] gh-118486: Support mkdir(mode=0700) on Windows (CVE-2024-4030) Backport of python/cpython gh-118486 (eb29e2f590; 3.10: c8f868dc52f) to 2.7.18.x. On Windows os.mkdir() ignored its mode, so tempfile.mkdtemp(), which calls os.mkdir(name, 0700), created directories that inherit the parent's ACL. Under a shared parent other local users can read the "private" temporary directory. posix_mkdir() now handles mode 0700 on Windows by creating the directory with a security descriptor whose protected discretionary ACL only grants full control (inherited by files and subdirectories) to SYSTEM, Administrators and the owner: "D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA) (A;OICI;FA;;;OW)". Other modes are ignored, as before. The same mode check on POSIX is unchanged. 2.7.18.x: 2.7's posix_mkdir() has a wide (CreateDirectoryW) and a narrow (CreateDirectoryA) path, and mkdtemp() normally takes the narrow one for str names, so the descriptor is built by a shared helper and applied on both. C89 declarations. ConvertStringSecurityDescriptorToSecurityDescriptorW is in advapi32, which pythoncore already links (_winreg). The docstring and Doc/library/os.rst describe the 0700 handling as upstream does. Tests (Windows only, ported from upstream): test_os.MakedirTests. test_win32_mkdir_700 checks the SDDL with cacls.exe, and test_tempfile.test_mkdtemp.test_mode_win32 checks with icacls.exe that BUILTIN\Users has no access. Not built or run on Windows yet: the Windows code was syntax-checked with gcc -std=c89 -pedantic against stub declarations, and the Linux build and test_os/test_tempfile pass. Co-Authored-By: Claude Opus 5.5 --- Doc/library/os.rst | 7 ++++ Lib/test/test_os.py | 16 +++++++-- Lib/test/test_tempfile.py | 27 +++++++++++++++ Modules/posixmodule.c | 73 +++++++++++++++++++++++++++++++++++++-- 4 files changed, 118 insertions(+), 5 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 5bfa35186e97542..5d63d21793dc0b1 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1233,11 +1233,18 @@ Files and Directories platform-dependent. On some platforms, they are ignored and you should call :func:`chmod` explicitly to set them. + On Windows, a *mode* of ``0700`` is specifically handled to apply access + control to the new directory such that only the current user and + administrators have access. Other values of *mode* are ignored. + It is also possible to create temporary directories; see the :mod:`tempfile` module's :func:`tempfile.mkdtemp` function. Availability: Unix, Windows. + .. versionchanged:: 2.7.18.16 + Windows now handles a *mode* of ``0700``. + .. function:: makedirs(path[, mode]) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index dcd486e2b1856dd..25cb19435fdc410 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -509,8 +509,20 @@ def test_makedir(self): 'dir5', 'dir6') os.makedirs(path) - - + @unittest.skipUnless(os.name == 'nt', "requires Windows") + def test_win32_mkdir_700(self): + # gh-118486 (CVE-2024-4030): mode 0700 gets an ACL that only allows + # SYSTEM, Administrators and the owner. + path = os.path.abspath(os.path.join(test_support.TESTFN, 'dir')) + os.mkdir(path, 0700) + try: + out = subprocess.check_output(["cacls.exe", path, "/s"]) + finally: + os.rmdir(path) + self.assertEqual( + out.strip(), + '%s "D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)"' % path, + ) def tearDown(self): path = os.path.join(test_support.TESTFN, 'dir1', 'dir2', 'dir3', diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 2efb8362fe1bd5c..f3387d77e994ac1 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -649,6 +649,33 @@ def test_mode(self): finally: os.rmdir(dir) + @unittest.skipUnless(os.name == "nt", "Only on Windows.") + def test_mode_win32(self): + # gh-118486 (CVE-2024-4030): use icacls.exe to extract the users with + # some level of access. Main thing we are testing is that the + # BUILTIN\Users group has no access. The exact ACL is going to vary + # based on which user is running the test. + import subprocess + dir = self.do_create() + try: + out = subprocess.check_output(["icacls.exe", dir]).lower() + finally: + os.rmdir(dir) + + dir = dir.lower() + users = set() + for line in out.strip().splitlines(): + acl = None + # First line of result includes our directory + if line.startswith(dir): + acl = line[len(dir):].strip() + elif line and line[:1].isspace(): + acl = line.strip() + if acl: + users.add(acl.partition(":")[0]) + + self.assertNotIn(r"BUILTIN\Users".lower(), users) + def test_collision_with_existing_file(self): # mkdtemp tries another name when a file with # the chosen name already exists diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index e5d108228f62ef1..b26e87b99b107ec 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -278,6 +278,7 @@ extern int lstat(const char *, struct stat *); #include #include #include /* for ShellExecute() */ +#include /* for ConvertStringSecurityDescriptorToSecurityDescriptorW() */ #define popen _popen #define pclose _pclose #endif /* _MSC_VER */ @@ -2564,7 +2565,52 @@ posix__getfullpathname(PyObject *self, PyObject *args) PyDoc_STRVAR(posix_mkdir__doc__, "mkdir(path [, mode=0777])\n\n\ -Create a directory."); +Create a directory.\n\ +\n\ +On Windows, a mode of 0700 is specifically handled to apply access control\n\ +to the new directory such that only the current user and administrators\n\ +have access. Other values of mode are ignored."); + +#ifdef MS_WINDOWS +/* gh-118486 (CVE-2024-4030): for mode 0700, build a security descriptor + whose protected discretionary ACL (D:P) only has inheritable (OICI) + full-control (FA) entries for SYSTEM (SY), Administrators (BA) and the + owner (OW). Returns 0 or a Windows error code. On success with mode + 0700, *pSecAttr points to secAttr and secAttr->lpSecurityDescriptor + must be released with LocalFree(). */ +static DWORD +win32_mkdir_security(int mode, SECURITY_ATTRIBUTES *secAttr, + SECURITY_ATTRIBUTES **pSecAttr) +{ + ULONG sdSize; + + ZeroMemory(secAttr, sizeof(*secAttr)); + secAttr->nLength = sizeof(*secAttr); + *pSecAttr = NULL; + if (mode != 0700) + return 0; + if (!ConvertStringSecurityDescriptorToSecurityDescriptorW( + L"D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)", + SDDL_REVISION_1, + &secAttr->lpSecurityDescriptor, + &sdSize)) + return GetLastError(); + *pSecAttr = secAttr; + return 0; +} + +/* Release the descriptor from win32_mkdir_security(). Returns 0 or a + Windows error code; uncommonly, LocalFree() returns non-NULL on error + and reports the error code through GetLastError(). */ +static DWORD +win32_mkdir_security_free(SECURITY_ATTRIBUTES *secAttr) +{ + if (secAttr->lpSecurityDescriptor && + LocalFree(secAttr->lpSecurityDescriptor)) + return GetLastError(); + return 0; +} +#endif /* MS_WINDOWS */ static PyObject * posix_mkdir(PyObject *self, PyObject *args) @@ -2575,10 +2621,21 @@ posix_mkdir(PyObject *self, PyObject *args) #ifdef MS_WINDOWS Py_UNICODE *wpath; + DWORD error; + SECURITY_ATTRIBUTES secAttr; + SECURITY_ATTRIBUTES *pSecAttr; + if (PyArg_ParseTuple(args, "u|i:mkdir", &wpath, &mode)) { + res = 0; Py_BEGIN_ALLOW_THREADS - res = CreateDirectoryW(wpath, NULL); + error = win32_mkdir_security(mode, &secAttr, &pSecAttr); + if (!error) { + res = CreateDirectoryW(wpath, pSecAttr); + error = win32_mkdir_security_free(&secAttr); + } Py_END_ALLOW_THREADS + if (error) + return PyErr_SetFromWindowsErr(error); if (!res) return win32_error_unicode("mkdir", wpath); Py_INCREF(Py_None); @@ -2590,9 +2647,19 @@ posix_mkdir(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "et|i:mkdir", Py_FileSystemDefaultEncoding, &path, &mode)) return NULL; + res = 0; Py_BEGIN_ALLOW_THREADS - res = CreateDirectoryA(path, NULL); + error = win32_mkdir_security(mode, &secAttr, &pSecAttr); + if (!error) { + res = CreateDirectoryA(path, pSecAttr); + error = win32_mkdir_security_free(&secAttr); + } Py_END_ALLOW_THREADS + if (error) { + PyErr_SetFromWindowsErr(error); + PyMem_Free(path); + return NULL; + } if (!res) { win32_error("mkdir", path); PyMem_Free(path); From 403699fae6330ee8267a1a983443916658ca31c7 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Wed, 28 Oct 2026 11:22:00 -0500 Subject: [PATCH 17/23] Define XML_POOR_ENTROPY for the Windows _elementtree build Expat 2.9.0 moved the rand_s() entropy code out of xmlparse.c into random_rand_s.c/.h, which xmlparse.c includes under "_WIN32 && ! XML_POOR_ENTROPY". CPython does not vendor that file; it builds Expat with XML_POOR_ENTROPY instead, as setup.py already does on POSIX and as 3.10.22.x's PCbuild/_elementtree.vcxproj does. _elementtree.vcxproj compiles the vendored xmlparse.c, so without the define the Windows build fails with "random_rand_s.h: No such file or directory". Python sets the hash salt itself, so Expat's own entropy source is not used. pyexpat.vcxproj does not compile the vendored sources (it links the AS libexpat), so it needs no change. Co-Authored-By: Claude Opus 5.5 --- PCbuild/_elementtree.vcxproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PCbuild/_elementtree.vcxproj b/PCbuild/_elementtree.vcxproj index 5a49a9553ed8893..64586e173f8cb34 100644 --- a/PCbuild/_elementtree.vcxproj +++ b/PCbuild/_elementtree.vcxproj @@ -62,7 +62,7 @@ ..\Modules\expat;%(AdditionalIncludeDirectories) - _CRT_SECURE_NO_WARNINGS;USE_PYEXPAT_CAPI;XML_STATIC;%(PreprocessorDefinitions) + _CRT_SECURE_NO_WARNINGS;USE_PYEXPAT_CAPI;XML_POOR_ENTROPY;XML_STATIC;%(PreprocessorDefinitions) 0x1D100000 From fb6672021a956a99e19c15137605c4b1b0ff7ee7 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Wed, 28 Oct 2026 10:28:53 -0500 Subject: [PATCH 18/23] gh-149018: Give Expat parsers a 16-byte random hash salt (CVE-2026-7210) Backport of python/cpython gh-149018 (24b8f12544; 3.10: ea70712d1a8) to 2.7.18.x. pyexpat and cElementTree seed every Expat parser's hash-flooding protection with XML_SetHashSalt(parser, _Py_HashSecret.prefix). On 2.7 _Py_HashSecret is all zeros unless hash randomization (-R) is on. A salt of 0 makes Expat generate its own, and our build defines XML_POOR_ENTROPY, so it comes from the time and pid. EXPAT_ENTROPY_DEBUG=1 shows "fallback(8)" salts whose upper bits are the same on every run. Even with -R the salt is at most one long. With Expat >= 2.8.0, pyexpat and _elementtree now read 16 bytes from _PyOS_URandom() once at import, and pass them to every parser through XML_SetHashSalt16Bytes(). As upstream, the function is added to the end of the PyExpat_CAPI struct, which is NULL for Expat < 2.8.0. If the OS random source fails, or with an older Expat, the old XML_SetHashSalt() path is used. 2.7.18.x: upstream reads the salt from a new 16-byte expat member of _Py_HashSecret. 2.7's _Py_HashSecret_t is two longs, and changing it would alter that exported struct, so each module keeps its own random salt instead. _elementtree uses the 16-byte path only through the pyexpat CAPI (USE_PYEXPAT_CAPI, which setup.py and the Windows project both define). Needs Expat >= 2.8.0 (the bundled copy is 2.9.0). On Windows, pyexpat links the external AS libexpat, which must then also be >= 2.8.0 (the current platform build uses 2.8.4). The new test_pyexpat.HashSaltTest runs pyexpat and cElementTree under EXPAT_ENTROPY_DEBUG=1 and requires every parser's salt to be "explicit(16)". With bundled Expat 2.9.0 and no fix, both tests fail with fallback(8). With the fix, both pass. Both are skipped for Expat < 2.8.0. Co-Authored-By: Claude Opus 5.5 --- Include/pyexpat.h | 3 +++ Lib/test/test_pyexpat.py | 27 ++++++++++++++++++++++++++- Modules/_elementtree.c | 14 ++++++++++++++ Modules/pyexpat.c | 29 ++++++++++++++++++++++++++++- 4 files changed, 71 insertions(+), 2 deletions(-) diff --git a/Include/pyexpat.h b/Include/pyexpat.h index 3fc5fa54da6376c..29f117eeef59359 100644 --- a/Include/pyexpat.h +++ b/Include/pyexpat.h @@ -45,6 +45,9 @@ struct PyExpat_CAPI void (*SetUserData)(XML_Parser parser, void *userData); /* might be none for expat < 2.1.0 */ int (*SetHashSalt)(XML_Parser parser, unsigned long hash_salt); + /* might be NULL for expat < 2.8.0 */ + XML_Bool (*SetHashSalt16Bytes)( + XML_Parser parser, const uint8_t entropy[16]); /* always add new stuff to the end! */ }; diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index 0fe1c6575d4c9de..8c586b3b6f2d36a 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -678,6 +678,30 @@ def resolve_entity(context, base, system_id, public_id): self.assertEqual(handler_call_args, [("bar", "baz")]) +@unittest.skipIf(expat.version_info < (2, 8, 0), + "XML_SetHashSalt16Bytes needs expat >= 2.8.0") +class HashSaltTest(unittest.TestCase): + # gh-149018 (CVE-2026-7210): parsers get 16 bytes of OS entropy as their + # hash salt instead of the 0 from _Py_HashSecret (without -R), which made + # expat fall back to its own time-based salt. EXPAT_ENTROPY_DEBUG makes + # expat report on stderr where each parser's salt came from. + def check_salt(self, code): + from test.script_helper import assert_python_ok + rc, out, err = assert_python_ok('-c', code, EXPAT_ENTROPY_DEBUG='1') + lines = [l for l in err.splitlines() if l.startswith('expat: Entropy')] + self.assertTrue(lines, err) + for line in lines: + self.assertIn('explicit(16)', line) + + def test_pyexpat(self): + self.check_salt("import pyexpat\n" + "pyexpat.ParserCreate().Parse('', True)") + + def test_celementtree(self): + self.check_salt("import xml.etree.cElementTree as ET\n" + "ET.fromstring('')") + + def test_main(): run_unittest(SetAttributeTest, ParseTest, @@ -689,7 +713,8 @@ def test_main(): sf1296433Test, ChardataBufferTest, MalformedInputText, - ForeignDTDTests) + ForeignDTDTests, + HashSaltTest) if __name__ == "__main__": test_main() diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index bbaf99410485920..7cbfa69f81f672c 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -2059,6 +2059,9 @@ statichere PyTypeObject TreeBuilder_Type = { #include "pyexpat.h" static struct PyExpat_CAPI* expat_capi; #define EXPAT(func) (expat_capi->func) +/* Hash salt for SetHashSalt16Bytes(), see gh-149018 and pyexpat.c. */ +static unsigned char expat_hash_salt16[16]; +static int expat_hash_salt16_set = 0; #else #define EXPAT(func) (XML_##func) #endif @@ -2592,6 +2595,13 @@ xmlparser(PyObject* self_, PyObject* args, PyObject* kw) PyErr_NoMemory(); return NULL; } +#if defined(USE_PYEXPAT_CAPI) + /* Prefer 16 bytes of entropy, only expat >= 2.8.0. See gh-149018. */ + if (expat_hash_salt16_set && EXPAT(SetHashSalt16Bytes) != NULL) { + EXPAT(SetHashSalt16Bytes)(self->parser, expat_hash_salt16); + } + else +#endif /* expat < 2.1.0 has no XML_SetHashSalt() */ if (EXPAT(SetHashSalt) != NULL) { EXPAT(SetHashSalt)(self->parser, @@ -3209,6 +3219,10 @@ init_elementtree(void) expat_capi->MICRO_VERSION != XML_MICRO_VERSION) expat_capi = NULL; } + if (_PyOS_URandom(expat_hash_salt16, sizeof(expat_hash_salt16)) == 0) + expat_hash_salt16_set = 1; + else + PyErr_Clear(); #endif elementtree_parseerror_obj = PyErr_NewException( diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index a1a870d418e8b3b..038eec834e89118 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -8,6 +8,15 @@ #define XML_COMBINED_VERSION (10000*XML_MAJOR_VERSION+100*XML_MINOR_VERSION+XML_MICRO_VERSION) +#if XML_COMBINED_VERSION >= 20800 +/* Process-wide hash salt for XML_SetHashSalt16Bytes(), read from the OS + random source at import (gh-149018). 2.7 has no 16-byte expat secret + in _Py_HashSecret, and _Py_HashSecret is all zeros unless hash + randomization (-R) is enabled. */ +static uint8_t expat_hash_salt16[16]; +static int expat_hash_salt16_set = 0; +#endif + #ifndef PyDoc_STRVAR /* @@ -1318,7 +1327,14 @@ newxmlparseobject(char *encoding, char *namespace_separator, PyObject *intern) Py_DECREF(self); return NULL; } -#if XML_COMBINED_VERSION >= 20100 +#if XML_COMBINED_VERSION >= 20800 + /* Prefer 16 bytes of entropy, only expat >= 2.8.0. See gh-149018. */ + if (expat_hash_salt16_set) + XML_SetHashSalt16Bytes(self->itself, expat_hash_salt16); + else + XML_SetHashSalt(self->itself, + (unsigned long)_Py_HashSecret.prefix); +#elif XML_COMBINED_VERSION >= 20100 /* This feature was added upstream in libexpat 2.1.0. */ XML_SetHashSalt(self->itself, (unsigned long)_Py_HashSecret.prefix); @@ -1843,6 +1859,12 @@ MODULE_INITFUNC(void) if (errmod_name == NULL) return; +#if XML_COMBINED_VERSION >= 20800 + if (_PyOS_URandom(expat_hash_salt16, sizeof(expat_hash_salt16)) == 0) + expat_hash_salt16_set = 1; + else + PyErr_Clear(); +#endif modelmod_name = PyString_FromString(MODULE_NAME ".model"); if (modelmod_name == NULL) return; @@ -2045,6 +2067,11 @@ MODULE_INITFUNC(void) #else capi.SetHashSalt = NULL; #endif +#if XML_COMBINED_VERSION >= 20800 + capi.SetHashSalt16Bytes = XML_SetHashSalt16Bytes; +#else + capi.SetHashSalt16Bytes = NULL; +#endif /* export using capsule */ capi_object = PyCapsule_New(&capi, PyExpat_CAPSULE_NAME, NULL); From 47fbde7b88ef5ba9ae40d2c1bb1916f4f1c3b8b2 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Wed, 28 Oct 2026 11:25:26 -0500 Subject: [PATCH 19/23] bz2: Make BZ2File unusable for reading after a data error (CVE-2026-9669) Review follow-up. In 2.7, BZ2File is C code over BZ2_bzRead(), which calls BZ2_bzDecompress() on the same stream on every call. After a read fails, the file stayed in MODE_READ, so the next read, readline, readlines, iteration or forward seek re-entered the errored stream. That is the out-of-bounds write this change closes for BZ2Decompressor. In 3.x, BZ2File is built on BZ2Decompressor, so upstream's fix already covers it. BZ2FileObject now records the libbz2 error wherever a read path gets one (Util_CatchBZ2ReadError). read(), readline(), readlines() and iteration then raise ValueError("BZ2File is unusable after a previous error"). seek() reopens the stream first when an error was recorded, and every backwards seek reopens it anyway, so seeking reads from a fresh stream and clears the error. The rewind code moves into Util_BZ2FileRewind(), shared by both cases. test_bz2.BZ2FileTest.test_read_after_data_error writes the crafted stream plus 4000 trailing bytes. It checks that every read method raises ValueError after the first IOError, and that seek(0), seek(10) and seek(0, 2) start over and hit the same IOError instead of reusing the stream. Without this change the second read re-enters libbz2 and fails with IOError again. The crafted stream is now a module constant shared with the decompressor test. Co-Authored-By: Claude Opus 5.5 --- Lib/test/test_bz2.py | 39 ++++++++++++--- Modules/bz2module.c | 115 ++++++++++++++++++++++++++++++++----------- 2 files changed, 119 insertions(+), 35 deletions(-) diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index 1077da4d721da3a..64c65a2490dd316 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -45,6 +45,16 @@ def decompress(self, data): return bz2.decompress(data) +# gh-150599: a crafted stream on which BZ2_bzDecompress() fails with +# BZ_DATA_ERROR; calling it again on the same stream could write out of +# bounds. +DATA_ERROR_STREAM = binascii.unhexlify( + "425a6839314159265359000000000000007fffff000000000000000000000000" + "00000000000000000000000000000000000000e0370000000000000000000000" + "000000000000000000000000000000000000000000000000000083f3" +) + + class BZ2FileTest(BaseTest): "Test BZ2File type miscellaneous methods." @@ -338,6 +348,28 @@ def testInitNonExistentFile(self): self.assertRaises(IOError, f.__init__, "non-existent-file") del f + def test_read_after_data_error(self): + # gh-150599: BZ2File reads with BZ2_bzRead(), which re-enters + # BZ2_bzDecompress() on the same stream; after a data error the + # file refuses further reads until a seek() reopens the stream. + with open(self.filename, "wb") as f: + f.write(DATA_ERROR_STREAM + b"\x00" * 4000) + def readers(bz2f): + return [bz2f.read, lambda: bz2f.read(10), bz2f.readline, + bz2f.readlines, lambda: next(iter(bz2f))] + for i in range(5): + with BZ2File(self.filename) as bz2f: + with self.assertRaisesRegexp(IOError, "invalid data stream"): + bz2f.read() + with self.assertRaisesRegexp(ValueError, "previous error"): + readers(bz2f)[i]() + # seek() starts over from a new stream, which fails again. + for offset, whence in ((0, 0), (10, 0), (0, 2)): + with self.assertRaisesRegexp(IOError, + "invalid data stream"): + bz2f.seek(offset, whence) + bz2f.read() + class BZ2CompressorTest(BaseTest): def testCompress(self): # "Test BZ2Compressor.compress()/flush()" @@ -421,14 +453,9 @@ def testEOFError(self): def test_decompress_after_data_error(self): # gh-150599: re-entering BZ2_bzDecompress() after an error could # write out of bounds; the decompressor is now unusable instead. - data = binascii.unhexlify( - "425a6839314159265359000000000000007fffff000000000000000000000000" - "00000000000000000000000000000000000000e0370000000000000000000000" - "000000000000000000000000000000000000000000000000000083f3" - ) bzd = BZ2Decompressor() with self.assertRaisesRegexp(IOError, "invalid data stream"): - bzd.decompress(data) + bzd.decompress(DATA_ERROR_STREAM) with self.assertRaisesRegexp(ValueError, "previous error"): bzd.decompress(b'\x00' * 18) diff --git a/Modules/bz2module.c b/Modules/bz2module.c index c68d84783ad14d8..312cd8f5721c967 100644 --- a/Modules/bz2module.c +++ b/Modules/bz2module.c @@ -102,6 +102,8 @@ typedef struct { int mode; Py_off_t pos; Py_off_t size; + int bzerror; /* Sticky decompression error, see + Util_CatchBZ2ReadError() */ #ifdef WITH_THREAD PyThread_type_lock lock; #endif @@ -204,6 +206,29 @@ Util_CatchBZ2Error(int bzerror) return ret; } +/* BZ2_bzRead() calls BZ2_bzDecompress() on the same stream every time, and + re-entering BZ2_bzDecompress() after an error can write out of bounds + (gh-150599, CVE-2026-9669). A read error therefore makes the file + unusable for reading until a backwards seek() reopens the stream. */ +static int +Util_CatchBZ2ReadError(BZ2FileObject *f, int bzerror) +{ + if (bzerror != BZ_OK && bzerror != BZ_STREAM_END) + f->bzerror = bzerror; + return Util_CatchBZ2Error(bzerror); +} + +static int +Util_CheckBZ2ReadError(BZ2FileObject *f) +{ + if (f->bzerror) { + PyErr_SetString(PyExc_ValueError, + "BZ2File is unusable after a previous error"); + return -1; + } + return 0; +} + #if BUFSIZ < 8192 #define SMALLCHUNK 8192 #else @@ -299,7 +324,7 @@ Util_GetLine(BZ2FileObject *f, int n) f->mode = MODE_READ_EOF; break; } else if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); + Util_CatchBZ2ReadError(f, bzerror); Py_DECREF(v); return NULL; } @@ -438,7 +463,7 @@ Util_ReadAhead(BZ2FileObject *f, int bufsize) f->size = f->pos; f->mode = MODE_READ_EOF; } else if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); + Util_CatchBZ2ReadError(f, bzerror); Util_DropReadAhead(f); return -1; } @@ -536,6 +561,9 @@ BZ2File_read(BZ2FileObject *self, PyObject *args) if (check_iterbuffered(self)) goto cleanup; + if (Util_CheckBZ2ReadError(self)) + goto cleanup; + if (bytesrequested < 0) buffersize = Util_NewBufferSize((size_t)0); else @@ -565,7 +593,7 @@ BZ2File_read(BZ2FileObject *self, PyObject *args) self->mode = MODE_READ_EOF; break; } else if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); + Util_CatchBZ2ReadError(self, bzerror); Py_DECREF(ret); ret = NULL; goto cleanup; @@ -625,6 +653,9 @@ BZ2File_readline(BZ2FileObject *self, PyObject *args) if (check_iterbuffered(self)) goto cleanup; + if (Util_CheckBZ2ReadError(self)) + goto cleanup; + if (sizehint == 0) ret = PyString_FromString(""); else @@ -686,6 +717,9 @@ BZ2File_readlines(BZ2FileObject *self, PyObject *args) if (check_iterbuffered(self)) goto cleanup; + if (Util_CheckBZ2ReadError(self)) + goto cleanup; + if ((list = PyList_New(0)) == NULL) goto cleanup; @@ -705,7 +739,7 @@ BZ2File_readlines(BZ2FileObject *self, PyObject *args) } shortread = 1; } else if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); + Util_CatchBZ2ReadError(self, bzerror); error: Py_DECREF(list); list = NULL; @@ -996,6 +1030,41 @@ BZ2File_writelines(BZ2FileObject *self, PyObject *seq) #undef CHUNKSIZE } +/* Close the read stream and reopen it at the start of the file. */ +static int +Util_BZ2FileRewind(BZ2FileObject *self) +{ + PyObject *ret; + int bzerror; + + BZ2_bzReadClose(&bzerror, self->fp); + if (self->fp) { + PyFile_DecUseCount((PyFileObject *)self->file); + self->fp = NULL; + } + if (bzerror != BZ_OK) { + Util_CatchBZ2Error(bzerror); + return -1; + } + ret = PyObject_CallMethod(self->file, "seek", "(i)", 0); + if (!ret) + return -1; + Py_DECREF(ret); + self->pos = 0; + self->fp = BZ2_bzReadOpen(&bzerror, PyFile_AsFile(self->file), + 0, 0, NULL, 0); + if (self->fp) + PyFile_IncUseCount((PyFileObject *)self->file); + if (bzerror != BZ_OK) { + Util_CatchBZ2Error(bzerror); + return -1; + } + self->mode = MODE_READ; + /* A fresh stream: earlier read errors no longer apply. */ + self->bzerror = 0; + return 0; +} + PyDoc_STRVAR(BZ2File_seek__doc__, "seek(offset [, whence]) -> None\n\ \n\ @@ -1053,6 +1122,11 @@ BZ2File_seek(BZ2FileObject *self, PyObject *args) goto cleanup; } + /* After a read error the stream must not be decompressed again + (gh-150599): start over from a freshly opened one. */ + if (self->bzerror && Util_BZ2FileRewind(self) < 0) + goto cleanup; + if (where == 2) { if (self->size == -1) { assert(self->mode != MODE_READ_EOF); @@ -1069,7 +1143,7 @@ BZ2File_seek(BZ2FileObject *self, PyObject *args) if (bzerror == BZ_STREAM_END) { break; } else if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); + Util_CatchBZ2ReadError(self, bzerror); goto cleanup; } } @@ -1090,30 +1164,8 @@ BZ2File_seek(BZ2FileObject *self, PyObject *args) offset -= self->pos; } else { /* we cannot move back, so rewind the stream */ - BZ2_bzReadClose(&bzerror, self->fp); - if (self->fp) { - PyFile_DecUseCount((PyFileObject *)self->file); - self->fp = NULL; - } - if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - goto cleanup; - } - ret = PyObject_CallMethod(self->file, "seek", "(i)", 0); - if (!ret) - goto cleanup; - Py_DECREF(ret); - ret = NULL; - self->pos = 0; - self->fp = BZ2_bzReadOpen(&bzerror, PyFile_AsFile(self->file), - 0, 0, NULL, 0); - if (self->fp) - PyFile_IncUseCount((PyFileObject *)self->file); - if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); + if (Util_BZ2FileRewind(self) < 0) goto cleanup; - } - self->mode = MODE_READ; } if (offset <= 0 || self->mode == MODE_READ_EOF) @@ -1140,7 +1192,7 @@ BZ2File_seek(BZ2FileObject *self, PyObject *args) self->mode = MODE_READ_EOF; break; } else if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); + Util_CatchBZ2ReadError(self, bzerror); goto cleanup; } if (bytesread == offset) @@ -1392,6 +1444,7 @@ BZ2File_init(BZ2FileObject *self, PyObject *args, PyObject *kwargs) int mode_char = 0; self->size = -1; + self->bzerror = 0; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|sii:BZ2File", kwlist, &name, &mode, &buffering, @@ -1533,6 +1586,10 @@ BZ2File_iternext(BZ2FileObject *self) "I/O operation on closed file"); return NULL; } + if (self->mode == MODE_READ && Util_CheckBZ2ReadError(self)) { + RELEASE_LOCK(self); + return NULL; + } ret = Util_ReadAheadGetLineSkip(self, 0, READAHEAD_BUFSIZE); RELEASE_LOCK(self); if (ret == NULL || PyString_GET_SIZE(ret) == 0) { From eb736f002ccf326d68e4c7f1a90f37d1c3696f05 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Wed, 28 Oct 2026 11:27:01 -0500 Subject: [PATCH 20/23] gh-150743: Limit trailer lines and interim responses read by httplib Backport of python/cpython gh-150743 (41cc78a7a4; 3.10: 1790e584142), GHSA-w4q2-g22w-6fr4, to 2.7.18.x. Review follow-up to the CVE-2021-3737 fix in this PR, which bounded the header lines inside one interim response but not these two sibling loops: - HTTPResponse.begin() skipped any number of "100 Continue" responses. It now stops after _MAXINTERIMRESPONSES (100) and raises HTTPException("got more than 100 interim responses"). - _read_chunked() discarded any number of trailer lines after the last chunk. It now raises HTTPException("got more than 100 trailers") past _MAXHEADERS. A server streaming either one forever kept the client reading indefinitely, and a socket timeout cannot catch that because data keeps arriving. 2.7.18.x: % formatting instead of f-strings, and assertRaisesRegexp. 2.7 has no chunked fixtures in test_httplib, so the tests build their own bodies. test_chunked_trailers is added to show that a few trailers are still accepted. test_too_many_interim_responses and test_chunked_too_many_trailers fail on the previous head of this branch and pass here. Co-Authored-By: Claude Opus 5.5 --- Lib/httplib.py | 20 +++++++++++- Lib/test/test_httplib.py | 66 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/Lib/httplib.py b/Lib/httplib.py index bdc6cb22fcae371..5b38653c63058d2 100644 --- a/Lib/httplib.py +++ b/Lib/httplib.py @@ -219,6 +219,13 @@ # maximum amount of headers accepted _MAXHEADERS = 100 +# maximal number of interim (1xx) responses tolerated before the final +# response. Real servers send at most a few; without a bound, a server +# streaming "100 Continue" responses would hang getresponse() forever. +# A socket timeout cannot detect that as data keeps arriving within every +# timeout window. +_MAXINTERIMRESPONSES = 100 + # Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2) # # VCHAR = %x21-7E @@ -448,7 +455,7 @@ def begin(self): return # read until we get a non-100 response - while True: + for _ in range(_MAXINTERIMRESPONSES): version, status, reason = self._read_status() if status != CONTINUE: break @@ -466,6 +473,9 @@ def begin(self): raise HTTPException("got more than %d headers" % _MAXHEADERS) if self.debuglevel > 0: print "header:", skip + else: + raise HTTPException("got more than %d interim responses" + % _MAXINTERIMRESPONSES) self.status = status self.reason = reason.strip() @@ -666,6 +676,7 @@ def _read_chunked(self, amt): # read and discard trailer up to the CRLF terminator ### note: we shouldn't have any trailers! + trailers_read = 0 while True: line = self.fp.readline(_MAXLINE + 1) if len(line) > _MAXLINE: @@ -676,6 +687,13 @@ def _read_chunked(self, amt): break if line == '\r\n': break + # Bound the trailer count just as response headers are bounded. + # A server streaming trailer lines forever would otherwise hang + # the client; a socket timeout cannot detect that as data keeps + # arriving within every timeout window. + trailers_read += 1 + if trailers_read > _MAXHEADERS: + raise HTTPException("got more than %d trailers" % _MAXHEADERS) # we read everything; close the "file" self.close() diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 8f39725acaaabcd..4d1d5c912a8b529 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -701,6 +701,72 @@ def test_headers_after_100(self): self.assertEqual(resp.status, 200) self.assertEqual(resp.read(), 'Hello') + def test_too_many_interim_responses(self): + # gh-150743 (CVE-2021-3737 follow-up): a server streaming + # "100 Continue" responses forever must not hang getresponse(). + body = ('HTTP/1.1 100 Continue\r\n\r\n' + * (httplib._MAXINTERIMRESPONSES + 1)) + resp = httplib.HTTPResponse(FakeSocket(body)) + with self.assertRaisesRegexp(httplib.HTTPException, + "got more than 100 interim responses"): + resp.begin() + + def test_multiple_interim_responses(self): + # A reasonable number of interim responses before the final + # response is skipped as before. + body = ( + 'HTTP/1.1 100 Continue\r\n\r\n' * 3 + + 'HTTP/1.1 200 OK\r\n' + 'Content-Length: 5\r\n' + '\r\n' + 'hello' + ) + resp = httplib.HTTPResponse(FakeSocket(body), method="GET") + resp.begin() + self.assertEqual(resp.status, 200) + self.assertEqual(resp.read(), 'hello') + resp.close() + + def test_chunked_trailers(self): + # A few trailer lines after the last chunk are read and discarded. + body = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + '5\r\nhello\r\n0\r\n' + 'X-Trailer1: 1\r\nX-Trailer2: 2\r\n\r\n' + ) + resp = httplib.HTTPResponse(FakeSocket(body), method="GET") + resp.begin() + self.assertEqual(resp.read(), 'hello') + resp.close() + + def test_chunked_too_many_trailers(self): + # gh-150743 (CVE-2021-3737 follow-up): a response streaming endless + # trailer lines must raise, not hang. + body = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + '5\r\nhello\r\n0\r\n' + + ''.join('X-Trailer%d: %d\r\n' % (i, i) + for i in range(httplib._MAXHEADERS + 1)) + + '\r\n' + ) + # An unbounded read() reaches the trailers via the final 0 chunk. + resp = httplib.HTTPResponse(FakeSocket(body), method="GET") + resp.begin() + with self.assertRaisesRegexp(httplib.HTTPException, + "got more than 100 trailers"): + resp.read() + resp.close() + + # A bounded read(amt) larger than the body hits the same limit. + resp = httplib.HTTPResponse(FakeSocket(body), method="GET") + resp.begin() + with self.assertRaisesRegexp(httplib.HTTPException, + "got more than 100 trailers"): + resp.read(6) + resp.close() + def test_overflowing_chunked_line(self): body = ( 'HTTP/1.1 200 OK\r\n' From 85ba67cd14cfbaa556a448d4d3f0226c0bc3e90b Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Wed, 28 Oct 2026 12:27:39 -0500 Subject: [PATCH 21/23] Review follow-ups for 2.7.18.16: _socket debug link, SNI None ref, sddl.h Three small fixes raised in the reviews of #106 and #108: - socketmodule.c: remove socket_parse_timeout(). It came in with the CVE-2023-40217 getblocking() backport (47374069a24, first released in 2.7.18.10), has no callers, and calls _PyTime_FromSecondsObject(), which 2.7 never builds (Python/pytime.c is not compiled). Optimized builds drop the unused static function, but an -O0 or --with-pydebug build leaves an undefined symbol in _socket, so _socket (and ssl) fail to import. - _ssl.c: _servername_callback() set ssl_socket to Py_None without a reference when the SSL object has no Python socket, and the error path then did Py_DECREF(ssl_socket). Take the reference. This predates #106; it is not reachable through ssl.SSLSocket, which always sets ssl_sock. - posixmodule.c: include under MS_WINDOWS, as upstream does, instead of inside the _MSC_VER block, so it matches the MS_WINDOWS helpers that call ConvertStringSecurityDescriptorToSecurityDescriptorW (#108). Correction to the #108 commit message (386508c6c1d): it cites upstream's gh-118486 commit as eb29e2f590. The original main commit is 81939dad77 (GH-118488, 2024-05-02), followed by 8ed5466795 (OWNER RIGHTS) and 8af84b503d (gh-118773, the SDDL string the port uses). Co-Authored-By: Claude Opus 5.5 --- Modules/_ssl.c | 2 ++ Modules/posixmodule.c | 5 ++++- Modules/socketmodule.c | 41 ----------------------------------------- 3 files changed, 6 insertions(+), 42 deletions(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c index c8f07f92b2622e3..232fff61af61e63 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -3472,7 +3472,9 @@ _servername_callback(SSL *s, int *al, void *args) Py_INCREF(sni_cb); if (ssl->ssl_sock == NULL) { + /* The error path below releases ssl_socket. */ ssl_socket = Py_None; + Py_INCREF(ssl_socket); } else { ssl_socket = PyWeakref_GetObject(ssl->ssl_sock); Py_INCREF(ssl_socket); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index b26e87b99b107ec..4ad7de27c6206ec 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -278,11 +278,14 @@ extern int lstat(const char *, struct stat *); #include #include #include /* for ShellExecute() */ -#include /* for ConvertStringSecurityDescriptorToSecurityDescriptorW() */ #define popen _popen #define pclose _pclose #endif /* _MSC_VER */ +#ifdef MS_WINDOWS +#include /* for ConvertStringSecurityDescriptorToSecurityDescriptorW() */ +#endif + #if defined(PYCC_VACPP) && defined(PYOS_OS2) #include #endif /* OS2 */ diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index b145daf5c88b6df..69472cf2d95e9c4 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -1836,47 +1836,6 @@ PyDoc_STRVAR(getblocking_doc, Returns True if socket is in blocking mode, or False if it\n\ is in non-blocking mode."); -static int -socket_parse_timeout(_PyTime_t *timeout, PyObject *timeout_obj) -{ -#ifdef MS_WINDOWS - struct timeval tv; -#endif -#ifndef HAVE_POLL - _PyTime_t ms; -#endif - int overflow = 0; - - if (timeout_obj == Py_None) { - *timeout = _PyTime_FromSeconds(-1); - return 0; - } - - if (_PyTime_FromSecondsObject(timeout, - timeout_obj, _PyTime_ROUND_TIMEOUT) < 0) - return -1; - - if (*timeout < 0) { - PyErr_SetString(PyExc_ValueError, "Timeout value out of range"); - return -1; - } - -#ifdef MS_WINDOWS - overflow |= (_PyTime_AsTimeval(*timeout, &tv, _PyTime_ROUND_TIMEOUT) < 0); -#endif -#ifndef HAVE_POLL - ms = _PyTime_AsMilliseconds(*timeout, _PyTime_ROUND_TIMEOUT); - overflow |= (ms > INT_MAX); -#endif - if (overflow) { - PyErr_SetString(PyExc_OverflowError, - "timeout doesn't fit into C timeval"); - return -1; - } - - return 0; -} - /* s.settimeout(timeout) method. Argument: None -- no timeout, blocking mode; same as setblocking(True) 0.0 -- non-blocking mode; same as setblocking(False) From b9b77dbe0f98c1aa2082d6f1ee418f2848025668 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Wed, 28 Oct 2026 12:41:29 -0500 Subject: [PATCH 22/23] test_ssl: drop the third-party mock import Review follow-up (#111). test_ssl.py imported "from mock import Mock" (added with an earlier ssl backport, 37df9f76475, and in 2.7.18.14), but mock is not part of the 2.7 standard library, so on a plain build the whole module failed with "ImportError: No module named mock" and none of test_ssl ran. Mock itself was unused; the only mock use, mock.patch.dict(os.environ) in test_keylog_env, referred to a name the file never imported. Use support.EnvironmentVarGuard there instead and remove the import. (test_keylog_env is skipped on 2.7, whose _ssl has no keylog callback.) Co-Authored-By: Claude Opus 5.5 --- Lib/test/test_ssl.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 57ee6c37f0ce7e9..8c689f7aaa32768 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -6,8 +6,6 @@ from test import test_support as support from test.script_helper import assert_python_ok import asyncore -# import unittest.mock -from mock import Mock from test import support import re import socket @@ -3577,8 +3575,8 @@ def test_keylog_filename(self): "test is not compatible with ignore_environment") def test_keylog_env(self): self.addCleanup(support.unlink, support.TESTFN) - with mock.patch.dict(os.environ): - os.environ['SSLKEYLOGFILE'] = support.TESTFN + with support.EnvironmentVarGuard() as env: + env['SSLKEYLOGFILE'] = support.TESTFN self.assertEqual(os.environ['SSLKEYLOGFILE'], support.TESTFN) ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) From 60c3e21e033b62c089443d4ce260c3b8fe74152a Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Wed, 28 Oct 2026 14:34:04 -0500 Subject: [PATCH 23/23] 2.7.18.16 Release Bump PY_VERSION to 2.7.18.16 and add release notes for the security fixes merged into 2.7.18.16 (#92-#108, #111): tarfile (CVE-2026-11972), csv.Sniffer (CVE-2026-18503), ftplib.ftpcp (CVE-2026-8328), httplib (CVE-2026-103995, CVE-2021-3737 and GHSA-w4q2-g22w-6fr4), SimpleHTTPServer (CVE-2021-28861), mimetypes on Windows (CVE-2024-3220), pyexpat (CVE-2026-4224), bz2 (CVE-2026-9669), ElementPath (CVE-2026-6879), urllib2 (CVE-2026-15806), HTMLParser (CVE-2026-15308), bundled Expat 2.9.0 (30 CVEs), the Expat hash salt (CVE-2026-7210), the ssl SNI callback (CVE-2026-19445), unicodedata (CVE-2026-3276) and os.mkdir on Windows (CVE-2024-4030), plus the review follow-up build fixes. Co-Authored-By: Claude Opus 5.5 --- Include/patchlevel.h | 2 +- Misc/NEWS.d/2.7.18.16.rst | 258 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 259 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/2.7.18.16.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 2a6d2f141b71328..7d32ce5e2f0144d 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -27,7 +27,7 @@ #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "2.7.18.15" +#define PY_VERSION "2.7.18.16" /*--end constants--*/ /* Subversion Revision number of this file (not of the repository). Empty diff --git a/Misc/NEWS.d/2.7.18.16.rst b/Misc/NEWS.d/2.7.18.16.rst new file mode 100644 index 000000000000000..58a4d70473933e1 --- /dev/null +++ b/Misc/NEWS.d/2.7.18.16.rst @@ -0,0 +1,258 @@ +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as0116 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +Fix a hang in tarfile streaming mode (CVE-2026-11972) + +Seeking forward in a :mod:`tarfile` stream opened in ``r|`` mode stopped +only at the target offset, so a member header declaring a huge size made it +loop forever at end of file. It now stops at EOF and raises +:exc:`tarfile.ReadError` ("unexpected end of data"). + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as0216 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +Fix quadratic time in csv.Sniffer on quoted fields (CVE-2026-18503) + +The regular expression :class:`csv.Sniffer` uses to guess the quote character +backtracked quadratically on long quoted fields. Python 2.7's :mod:`re` has +no possessive quantifiers, so the upstream fix is expressed with an +equivalent lookahead and backreference. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as0316 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +ftplib.ftpcp() no longer trusts the host in a PASV reply (CVE-2026-8328) + +:func:`ftplib.ftpcp` connected the target server to the host address given +in the source server's ``PASV`` reply, letting a malicious server direct the +transfer to another host. It now uses the source server's actual peer +address, as :class:`ftplib.FTP` already does for its own transfers. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as0416 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +Reject malformed Content-Length and chunk sizes in httplib (CVE-2026-103995) + +:mod:`httplib` now accepts only decimal digits (with optional surrounding +whitespace) in ``Content-Length``, and only hexadecimal digits for a chunk +size. Values such as ``+5``, ``-0`` or ``0x10``, which other HTTP +implementations parse differently, are rejected or ignored instead of being +used. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as0516 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +Limit what httplib reads around interim responses and trailers (CVE-2021-3737) + +:class:`httplib.HTTPResponse` now reads at most 100 header lines inside a +``100 Continue`` response, skips at most 100 interim (1xx) responses, and +discards at most 100 trailer lines after a chunked body +(GHSA-w4q2-g22w-6fr4). Each limit raises :exc:`httplib.HTTPException` when +exceeded, instead of reading for as long as the server keeps sending. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as0616 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +Fix an open redirect in SimpleHTTPServer (CVE-2021-28861) + +A request path starting with ``//`` could make +:class:`SimpleHTTPServer.SimpleHTTPRequestHandler` redirect to another +host (``Location: //evil.example/...``). Leading slashes are now collapsed +before the path is used. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as0716 +.. release date: 2026-10-28 +.. original section: Windows +.. section: Security + +mimetypes no longer reads files from fixed paths on Windows (CVE-2024-3220) + +On Windows, :mod:`mimetypes` no longer reads the Unix ``knownfiles`` paths +(such as ``C:\etc\mime.types``), which any local user may be able to +create. The registry is still used. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as0816 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +Fix a crash in pyexpat on deeply nested content models (CVE-2026-4224) + +Converting a deeply nested DTD content model for an ``ElementDeclHandler`` +recursed without limit in C and could overflow the stack. It now raises +:exc:`RuntimeError` when the recursion limit is reached. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as0916 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +bz2 objects can no longer be reused after a data error (CVE-2026-9669) + +Calling into libbz2 again after it reported an error could write out of +bounds. :class:`bz2.BZ2Decompressor` and, for reading, :class:`bz2.BZ2File` +now raise :exc:`ValueError` once an error has occurred. Seeking a +:class:`bz2.BZ2File` starts over from a newly opened stream. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as1016 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +Fix quadratic time in ElementPath index predicates (CVE-2026-6879) + +``[1]``, ``[last()]`` and similar index predicates in +:mod:`xml.etree.ElementTree` path expressions re-scanned the parent for every +element, which was quadratic in the number of siblings. Results are now +computed once per parent. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as1116 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +urllib2 password managers no longer send https credentials over http (CVE-2026-15806) + +Credentials added to :class:`urllib2.HTTPPasswordMgr` for an ``https`` URL +were also returned for the same host over ``http``. The URL scheme is now +part of the match; an entry given without a scheme still matches any +scheme. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as1216 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +Fix quadratic time in HTMLParser incremental parsing (CVE-2026-15308) + +:class:`HTMLParser.HTMLParser` copied its whole buffer on every +:meth:`~HTMLParser.HTMLParser.feed` call while a construct was still open, +so feeding a document in small pieces was quadratic. Data is now buffered +until it can be parsed. As upstream, events after such a construct may be +reported later, but none are lost. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as1316 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +Update the bundled Expat to 2.9.0 + +The Expat in ``Modules/expat`` is updated from 2.6.4 to 2.9.0, fixing +CVE-2024-8176, CVE-2025-59375, CVE-2026-24515, CVE-2026-25210, +CVE-2026-32776, CVE-2026-32777, CVE-2026-32778, CVE-2026-41080, +CVE-2026-45186, CVE-2026-50219, CVE-2026-56131, CVE-2026-56132, +CVE-2026-56403 to CVE-2026-56412, CVE-2026-66046, CVE-2026-72522, +CVE-2026-76641, CVE-2026-76956, CVE-2026-76957, CVE-2026-77214, +CVE-2026-93990 and CVE-2026-102633. On Windows, :mod:`pyexpat` links the +external Expat library, which should be 2.9.0 as well. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as1416 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +Give Expat parsers a random 16-byte hash salt (CVE-2026-7210) + +:mod:`pyexpat` and :mod:`xml.etree.cElementTree` now seed every parser's +hash-flooding protection with 16 bytes from the operating system's random +source (``XML_SetHashSalt16Bytes``), instead of a value that is zero unless +hash randomization is enabled, which made Expat fall back to a weak +time-based salt. Requires Expat 2.8.0 or later. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as1516 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +Fix a use-after-free in the ssl SNI callback (CVE-2026-19445) + +A servername callback that switched the connection to another +:class:`ssl.SSLContext` while the original context was released could make +the next callback read freed memory and crash the server. The callback now +finds its context through the connection and keeps it alive while in use. +After a switch, later ClientHello messages go to the new context's +callback. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as1616 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +Fix quadratic time in unicodedata.normalize() (CVE-2026-3276) + +Canonical ordering of a long run of combining characters in alternating +order was quadratic. Each run is now sorted separately, with a linear +counting sort for long runs. The results are unchanged. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as1716 +.. release date: 2026-10-28 +.. original section: Windows +.. section: Security + +os.mkdir(path, 0700) restricts access on Windows (CVE-2024-4030) + +On Windows, :func:`os.mkdir` ignored its *mode*, so directories created by +:func:`tempfile.mkdtemp` were readable by other users. A mode of ``0700`` +now creates the directory with an access control list that allows only +SYSTEM, Administrators and the owner. Other modes are still ignored. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as1816 +.. release date: 2026-10-28 +.. original section: Build +.. section: Build + +Fix building _socket without optimization + +Removed an unused function from ``Modules/socketmodule.c`` that referenced +a symbol Python 2.7 does not build, so ``-O0`` and ``--with-pydebug`` builds +produced a ``_socket`` module (and :mod:`ssl`) that failed to import. +``test_ssl`` no longer needs the third-party ``mock`` package.