diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 5bfa35186e9754..5d63d21793dc0b 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1233,11 +1233,18 @@ Files and Directories platform-dependent. On some platforms, they are ignored and you should call :func:`chmod` explicitly to set them. + On Windows, a *mode* of ``0700`` is specifically handled to apply access + control to the new directory such that only the current user and + administrators have access. Other values of *mode* are ignored. + It is also possible to create temporary directories; see the :mod:`tempfile` module's :func:`tempfile.mkdtemp` function. Availability: Unix, Windows. + .. versionchanged:: 2.7.18.16 + Windows now handles a *mode* of ``0700``. + .. function:: makedirs(path[, mode]) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 2a6d2f141b7132..7d32ce5e2f0144 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -27,7 +27,7 @@ #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "2.7.18.15" +#define PY_VERSION "2.7.18.16" /*--end constants--*/ /* Subversion Revision number of this file (not of the repository). Empty diff --git a/Include/pyexpat.h b/Include/pyexpat.h index 3fc5fa54da6376..29f117eeef5935 100644 --- a/Include/pyexpat.h +++ b/Include/pyexpat.h @@ -45,6 +45,9 @@ struct PyExpat_CAPI void (*SetUserData)(XML_Parser parser, void *userData); /* might be none for expat < 2.1.0 */ int (*SetHashSalt)(XML_Parser parser, unsigned long hash_salt); + /* might be NULL for expat < 2.8.0 */ + XML_Bool (*SetHashSalt16Bytes)( + XML_Parser parser, const uint8_t entropy[16]); /* always add new stuff to the end! */ }; diff --git a/Lib/BaseHTTPServer.py b/Lib/BaseHTTPServer.py index 3df3323a97eae6..b580de0681f0de 100644 --- a/Lib/BaseHTTPServer.py +++ b/Lib/BaseHTTPServer.py @@ -287,6 +287,13 @@ def parse_request(self): return False self.command, self.path, self.request_version = command, path, version + # gh-87389: The purpose of replacing '//' with '/' is to protect + # against open redirect attacks possibly triggered if the path starts + # with '//' because http clients treat //path as an absolute URI + # without scheme (similar to http://path) rather than a path. + if self.path.startswith('//'): + self.path = '/' + self.path.lstrip('/') # Reduce to a single / + # Examine the headers and look for a Connection directive self.headers = self.MessageClass(self.rfile, 0) diff --git a/Lib/HTMLParser.py b/Lib/HTMLParser.py index 85b980750e32ba..d037ec3bb2a709 100644 --- a/Lib/HTMLParser.py +++ b/Lib/HTMLParser.py @@ -106,6 +106,9 @@ def reset(self): self.lasttag = '???' self.interesting = interesting_normal self.cdata_elem = None + self._pending = [] + self._pending_len = 0 + self._parse_threshold = 1 markupbase.ParserBase.reset(self) def feed(self, data): @@ -114,11 +117,36 @@ def feed(self, data): Call this as often as you want, with as little or as much text as you want (may include '\n'). """ - self.rawdata = self.rawdata + data - self.goahead(0) + # Accumulate new data in a list and only join and parse it once + # enough has piled up. Rescanning an unparsed buffer (e.g. an + # unterminated tag) and concatenating onto it on every call would + # both be quadratic in the input size. + self._pending_len += len(data) + if self._pending_len < self._parse_threshold: + self._pending.append(data) + else: + if not self._pending: + self.rawdata += data + else: + self._pending.append(data) + self.rawdata += ''.join(self._pending) + del self._pending[:] + self._pending_len = 0 + n = len(self.rawdata) + self.goahead(0) + if len(self.rawdata) < n: + # Some data was parsed; resume on the next call. + self._parse_threshold = 1 + else: + # Nothing was parsed; wait until the buffer doubles. + self._parse_threshold = len(self.rawdata) def close(self): """Handle any buffered data.""" + if self._pending: + self.rawdata += ''.join(self._pending) + del self._pending[:] + self._pending_len = 0 self.goahead(1) def error(self, message): diff --git a/Lib/csv.py b/Lib/csv.py index 70c53ae7791e34..4ad565f5bff38d 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -214,12 +214,18 @@ def _guess_quote_and_delimiter(self, data, delimiters): this way. """ + # The body of a quoted field ends at the first quote which is + # not doubled, as it does for a reader. A lazy ".*?" scans to + # the end of the sample instead, from every start: quadratically. + # 2.7's re has no possessive "*+", so the body is matched + # atomically with a lookahead and a backreference. + body = r'(?=(?P(?:(?P=quote){2}|(?!(?P=quote)).)*))(?P=body)' matches = [] - for restr in ('(?P[^\w\n"\'])(?P ?)(?P["\']).*?(?P=quote)(?P=delim)', # ,".*?", - '(?:^|\n)(?P["\']).*?(?P=quote)(?P[^\w\n"\'])(?P ?)', # ".*?", - '(?P[^\w\n"\'])(?P ?)(?P["\']).*?(?P=quote)(?:$|\n)', # ,".*?" - '(?:^|\n)(?P["\']).*?(?P=quote)(?:$|\n)'): # ".*?" (no delim, no space) - regexp = re.compile(restr, re.DOTALL | re.MULTILINE) + for restr in (r'(?P[^\w\n"\'])(?P ?)(?P["\'])%s(?P=quote)(?P=delim)', # ,"...", + r'(?:^|\n)(?P["\'])%s(?P=quote)(?P[^\w\n"\'])(?P ?)', # "...", + r'(?P[^\w\n"\'])(?P ?)(?P["\'])%s(?P=quote)(?:$|\n)', # ,"..." + r'(?:^|\n)(?P["\'])%s(?P=quote)(?:$|\n)'): # "..." (no delim, no space) + regexp = re.compile(restr % body, re.DOTALL | re.MULTILINE) matches = regexp.findall(data) if matches: break diff --git a/Lib/ftplib.py b/Lib/ftplib.py index 7c772e6ee5ee16..cd9e87cb6267b8 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -939,7 +939,16 @@ def ftpcp(source, sourcename, target, targetname = '', type = 'I'): type = 'TYPE ' + type source.voidcmd(type) target.voidcmd(type) - sourcehost, sourceport = parse227(source.sendcmd('PASV')) + # Don't trust the IPv4 address the source server advertises in its PASV + # reply: a malicious source could otherwise point the target's data + # connection at an arbitrary host (SSRF). A caller that needs the old + # behavior can set trust_server_pasv_ipv4_address on the source FTP + # object. See FTP.makepasv(), which applies the same rule. + untrusted_host, sourceport = parse227(source.sendcmd('PASV')) + if source.trust_server_pasv_ipv4_address: + sourcehost = untrusted_host + else: + sourcehost = source.sock.getpeername()[0] target.sendport(sourcehost, sourceport) # RFC 959: the user must "listen" [...] BEFORE sending the # transfer request. diff --git a/Lib/httplib.py b/Lib/httplib.py index 9c8c5eedccb04a..b5effd21bf11b0 100644 --- a/Lib/httplib.py +++ b/Lib/httplib.py @@ -219,6 +219,13 @@ # maximum amount of headers accepted _MAXHEADERS = 100 +# maximal number of interim (1xx) responses tolerated before the final +# response. Real servers send at most a few; without a bound, a server +# streaming "100 Continue" responses would hang getresponse() forever. +# A socket timeout cannot detect that as data keeps arriving within every +# timeout window. +_MAXINTERIMRESPONSES = 100 + # Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2) # # VCHAR = %x21-7E @@ -261,6 +268,11 @@ # to prevent http header injection. _contains_disallowed_method_pchar_re = re.compile('[\x00-\x1f]') +# RFC 9112: Content-Length = 1*DIGIT and chunk-size = 1*HEXDIG. +# int() is more permissive, so we match against the grammar before calling it. +_is_legal_content_length = re.compile(r'[0-9]+\Z').match +_is_legal_chunk_size = re.compile(r'[0-9a-fA-F]+\Z').match + # We always set the Content-Length header for these methods because some # servers will otherwise respond with a 411 _METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'} @@ -448,11 +460,12 @@ def begin(self): return # read until we get a non-100 response - while True: + for _ in range(_MAXINTERIMRESPONSES): version, status, reason = self._read_status() if status != CONTINUE: break # skip the header from the 100 response + skipped = 0 while True: skip = self.fp.readline(_MAXLINE + 1) if len(skip) > _MAXLINE: @@ -460,8 +473,14 @@ def begin(self): skip = skip.strip() if not skip: break + skipped += 1 + if skipped > _MAXHEADERS: + raise HTTPException("got more than %d headers" % _MAXHEADERS) if self.debuglevel > 0: print "header:", skip + else: + raise HTTPException("got more than %d interim responses" + % _MAXINTERIMRESPONSES) self.status = status self.reason = reason.strip() @@ -503,14 +522,12 @@ def begin(self): # do we have a Content-Length? # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked" length = self.msg.getheader('content-length') - if length and not self.chunked: - try: - self.length = int(length) - except ValueError: - self.length = None - else: - if self.length < 0: # ignore nonsensical negative lengths - self.length = None + if length is not None: + # RFC 9112, section 5.1: optional whitespace (SP / HTAB) around + # the field value is not part of the value. + length = length.strip(' \t') + if length and not self.chunked and _is_legal_content_length(length): + self.length = int(length) else: self.length = None @@ -632,7 +649,10 @@ def _read_chunked(self, amt): i = line.find(';') if i >= 0: line = line[:i] # strip chunk-extensions + line = line.rstrip() try: + if not _is_legal_chunk_size(line): + raise ValueError("invalid chunk size") chunk_left = int(line, 16) except ValueError: # close the connection as protocol synchronisation is @@ -662,6 +682,7 @@ def _read_chunked(self, amt): # read and discard trailer up to the CRLF terminator ### note: we shouldn't have any trailers! + trailers_read = 0 while True: line = self.fp.readline(_MAXLINE + 1) if len(line) > _MAXLINE: @@ -672,6 +693,13 @@ def _read_chunked(self, amt): break if line == '\r\n': break + # Bound the trailer count just as response headers are bounded. + # A server streaming trailer lines forever would otherwise hang + # the client; a socket timeout cannot detect that as data keeps + # arriving within every timeout window. + trailers_read += 1 + if trailers_read > _MAXHEADERS: + raise HTTPException("got more than %d trailers" % _MAXHEADERS) # we read everything; close the "file" self.close() diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index afc053f135ddfb..92b0a414c2e5fe 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -37,17 +37,23 @@ "add_type","read_mime_types","init" ] -knownfiles = [ - "/etc/mime.types", - "/etc/httpd/mime.types", # Mac OS X - "/etc/httpd/conf/mime.types", # Apache - "/etc/apache/mime.types", # Apache 1 - "/etc/apache2/mime.types", # Apache 2 - "/usr/local/etc/httpd/conf/mime.types", - "/usr/local/lib/netscape/mime.types", - "/usr/local/etc/httpd/conf/mime.types", # Apache 1.2 - "/usr/local/etc/mime.types", # Apache 1.3 - ] +if sys.platform == "win32": + # The default paths would resolve to the root of the current drive on Windows + # (e.g. C:\etc\mime.types), which unprivileged users can usually create. + # Only the registry is used by default. + knownfiles = [] +else: + knownfiles = [ + "/etc/mime.types", + "/etc/httpd/mime.types", # Mac OS X + "/etc/httpd/conf/mime.types", # Apache + "/etc/apache/mime.types", # Apache 1 + "/etc/apache2/mime.types", # Apache 2 + "/usr/local/etc/httpd/conf/mime.types", + "/usr/local/lib/netscape/mime.types", + "/usr/local/etc/httpd/conf/mime.types", # Apache 1.2 + "/usr/local/etc/mime.types", # Apache 1.3 + ] inited = False _db = None diff --git a/Lib/tarfile.py b/Lib/tarfile.py index d1cb4fe15ae27d..223b573f45122c 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -556,7 +556,9 @@ def seek(self, pos=0): if pos - self.pos >= 0: blocks, remainder = divmod(pos - self.pos, self.bufsize) for i in xrange(blocks): - self.read(self.bufsize) + data = self.read(self.bufsize) + if not data: + break self.read(remainder) else: raise StreamError("seeking backwards is not allowed") diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index bf5eb2055bea36..64c65a2490dd31 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -2,6 +2,7 @@ from test.test_support import TESTFN, _4G, bigmemtest, import_module, findfile import unittest +import binascii from cStringIO import StringIO import os import subprocess @@ -44,6 +45,16 @@ def decompress(self, data): return bz2.decompress(data) +# gh-150599: a crafted stream on which BZ2_bzDecompress() fails with +# BZ_DATA_ERROR; calling it again on the same stream could write out of +# bounds. +DATA_ERROR_STREAM = binascii.unhexlify( + "425a6839314159265359000000000000007fffff000000000000000000000000" + "00000000000000000000000000000000000000e0370000000000000000000000" + "000000000000000000000000000000000000000000000000000083f3" +) + + class BZ2FileTest(BaseTest): "Test BZ2File type miscellaneous methods." @@ -337,6 +348,28 @@ def testInitNonExistentFile(self): self.assertRaises(IOError, f.__init__, "non-existent-file") del f + def test_read_after_data_error(self): + # gh-150599: BZ2File reads with BZ2_bzRead(), which re-enters + # BZ2_bzDecompress() on the same stream; after a data error the + # file refuses further reads until a seek() reopens the stream. + with open(self.filename, "wb") as f: + f.write(DATA_ERROR_STREAM + b"\x00" * 4000) + def readers(bz2f): + return [bz2f.read, lambda: bz2f.read(10), bz2f.readline, + bz2f.readlines, lambda: next(iter(bz2f))] + for i in range(5): + with BZ2File(self.filename) as bz2f: + with self.assertRaisesRegexp(IOError, "invalid data stream"): + bz2f.read() + with self.assertRaisesRegexp(ValueError, "previous error"): + readers(bz2f)[i]() + # seek() starts over from a new stream, which fails again. + for offset, whence in ((0, 0), (10, 0), (0, 2)): + with self.assertRaisesRegexp(IOError, + "invalid data stream"): + bz2f.seek(offset, whence) + bz2f.read() + class BZ2CompressorTest(BaseTest): def testCompress(self): # "Test BZ2Compressor.compress()/flush()" @@ -417,6 +450,15 @@ def testEOFError(self): self.assertRaises(EOFError, bz2d.decompress, "anything") self.assertRaises(EOFError, bz2d.decompress, "") + def test_decompress_after_data_error(self): + # gh-150599: re-entering BZ2_bzDecompress() after an error could + # write out of bounds; the decompressor is now unusable instead. + bzd = BZ2Decompressor() + with self.assertRaisesRegexp(IOError, "invalid data stream"): + bzd.decompress(DATA_ERROR_STREAM) + with self.assertRaisesRegexp(ValueError, "previous error"): + bzd.decompress(b'\x00' * 18) + @bigmemtest(_4G, memuse=1.25) def testBigmem(self, size): # Issue #14398: decompression fails when output data is >=2GB. diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 88d5af7c130d5d..e4c18ffe0a3a11 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -1096,6 +1096,13 @@ def test_doublequote(self): dialect = sniffer.sniff(self.sample9) self.assertTrue(dialect.doublequote) + def test_sniff_quoted_single_column(self): + # gh-98820: this sample used to take minutes. + sniffer = csv.Sniffer() + sample = '"abcdefghijklmnopqrstuvwxyz"\n' * 10000 + with self.assertRaisesRegexp(csv.Error, "Could not determine delimiter"): + sniffer.sniff(sample, delimiters=',:|\t') + class NUL: def write(s, *args): pass diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index c7def634d6b77c..dd42483c3f8124 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -980,10 +980,67 @@ def testTimeoutDirectAccess(self): ftp.close() +class TestFtpcpSecurity(TestCase): + """ftpcp() must not trust the host a source server advertises in PASV. + + A malicious source server can otherwise redirect the target server's + data connection to an arbitrary host:port (SSRF), so ftpcp() uses the + source server's actual peer address instead, the same as FTP.makepasv(). + """ + + class _FakeSock: + def __init__(self, peer): + self.peer = peer + + def getpeername(self): + return self.peer + + class _FakeFTP: + def __init__(self, replies, peer=None, trust=False): + self.replies = replies + self.sock = TestFtpcpSecurity._FakeSock(peer) + self.trust_server_pasv_ipv4_address = trust + self.sendport_calls = [] + + def voidcmd(self, cmd): + return '200 ok' + + def voidresp(self): + return '226 ok' + + def sendcmd(self, cmd): + return self.replies.get(cmd.split()[0], '150 ok') + + def sendport(self, host, port): + self.sendport_calls.append((host, port)) + + def _make_pair(self, advertised_host, real_host, trust=False): + # PASV replies give the host as comma-separated octets, not dotted. + advertised = advertised_host.replace('.', ',') + source = self._FakeFTP( + {'PASV': '227 Entering Passive Mode (%s,1,2).' % advertised}, + peer=(real_host, 21), trust=trust) + target = self._FakeFTP({}) + return source, target + + def test_ftpcp_ignores_untrusted_pasv_host(self): + source, target = self._make_pair(advertised_host='10.0.0.5', + real_host='198.51.100.7') + ftplib.ftpcp(source, 'a', target, 'b') + self.assertEqual(target.sendport_calls, [('198.51.100.7', 258)]) + + def test_ftpcp_trust_server_pasv_ipv4_address(self): + source, target = self._make_pair(advertised_host='10.0.0.5', + real_host='198.51.100.7', trust=True) + ftplib.ftpcp(source, 'a', target, 'b') + self.assertEqual(target.sendport_calls, [('10.0.0.5', 258)]) + + def test_main(): tests = [TestFTPClass, TestTimeouts, TestIPv6Environment, - TestTLS_FTPClassMixin, TestTLS_FTPClass] + TestTLS_FTPClassMixin, TestTLS_FTPClass, + TestFtpcpSecurity] thread_info = test_support.threading_setup() try: diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py index 65c73b0eee251c..ead120c119f1d1 100644 --- a/Lib/test/test_htmlparser.py +++ b/Lib/test/test_htmlparser.py @@ -444,6 +444,52 @@ def test_unescape_function(self): +class IncrementalFeedTestCase(TestCaseBase): + + def test_incremental_no_quadratic_complexity(self): + # An unterminated construct fed in many small chunks used to take + # quadratic time, both to rescan and to concatenate the buffer. + # Now it takes a fraction of a second. + test_support.requires('cpu') + def check(prefix, chunk, suffix): + parser = HTMLParser.HTMLParser() + parser.feed(prefix) + for _ in xrange(200000): + parser.feed(chunk) + parser.feed(suffix) + parser.close() + chunk = "a" * 64 + check("") # comment + check("") # processing instruction + check("") # doctype + check("") # CDATA section + check("") # start tag + check("") # RAWTEXT element + + def test_incremental_events_preserved(self): + # Data fed while a construct is unterminated is still reported in + # full, and parsing resumes on each feed() once it makes progress. + source = "

a

c" + expected = [("starttag", "p", []), ("data", "a"), ("endtag", "p"), + ("comment", " " + "x" * 1000 + " "), + ("starttag", "b", []), ("data", "c"), ("endtag", "b")] + parser = EventCollector() + for ch in source: + parser.feed(ch) + parser.close() + self.assertEqual(parser.get_events(), expected) + # Small feeds after an unterminated construct may be held back until + # the buffer doubles, but close() always delivers them. + parser = EventCollector() + parser.feed("") + parser.feed("") + parser.close() + self.assertEqual(parser.get_events(), + [("comment", " " + "y" * 100 + " "), + ("starttag", "i", [])]) + + class AttributesTestCase(TestCaseBase): def test_attr_syntax(self): @@ -649,7 +695,8 @@ def test_broken_condcoms(self): def test_main(): - test_support.run_unittest(HTMLParserTestCase, AttributesTestCase) + test_support.run_unittest(HTMLParserTestCase, AttributesTestCase, + IncrementalFeedTestCase) if __name__ == "__main__": diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index e7ee76b0226d09..43dbaca0627a89 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -622,6 +622,46 @@ def test_negative_content_length(self): self.assertEqual(resp.read(), 'Hello\r\n') self.assertTrue(resp.isclosed()) + def test_malformed_content_length(self): + # RFC 9112: Content-Length = 1*DIGIT. Values that int() accepts but + # the grammar forbids must not be used to frame the body. + for value in ('+5', '5_0', '5 0', '0x5'): + sock = FakeSocket( + 'HTTP/1.1 200 OK\r\nContent-Length: %s\r\n\r\nHello\r\n' % value) + resp = httplib.HTTPResponse(sock, method="GET") + resp.begin() + self.assertIsNone(resp.length, value) + self.assertEqual(resp.read(), 'Hello\r\n') + resp.close() + + def test_content_length_with_whitespace(self): + # RFC 9112, section 5.1: optional whitespace (SP / HTAB) around + # the field value is not part of the value, so it must not + # prevent the Content-Length from being used to frame the body. + for value in ('7 ', '7\t', '7 \t ', ' 7 '): + sock = FakeSocket( + 'HTTP/1.1 200 OK\r\nContent-Length: %s\r\n\r\n' + 'Hello\r\nextra' % value) + resp = httplib.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.length, 7, repr(value)) + self.assertEqual(resp.read(), 'Hello\r\n') + self.assertTrue(resp.isclosed()) + resp.close() + + def test_malformed_chunk_size(self): + # RFC 9112: chunk-size = 1*HEXDIG. Reject sizes that int(_, 16) accepts + # but the grammar forbids (a sign, an "0x" prefix or leading + # whitespace). + start = 'HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n' + for size in ('-5', '+5', '0x5', ' 5'): + sock = FakeSocket(start + '%s\r\nHELLO\r\n0\r\n\r\n' % size) + resp = httplib.HTTPResponse(sock, method="GET") + resp.begin() + self.assertRaises(httplib.IncompleteRead, resp.read) + self.assertTrue(resp.isclosed(), size) + resp.close() + def test_incomplete_read(self): sock = FakeSocket('HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nHello\r\n') resp = httplib.HTTPResponse(sock, method="GET") @@ -677,6 +717,96 @@ def test_overflowing_header_line(self): resp = httplib.HTTPResponse(FakeSocket(body)) self.assertRaises(httplib.LineTooLong, resp.begin) + def test_overflowing_header_limit_after_100(self): + # CVE-2021-3737: the headers of a 100 Continue response used to be + # skipped without any limit on their number. + body = ( + 'HTTP/1.1 100 OK\r\n' + + 'r\n' * 32768 + + '\r\nHTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n' + ) + resp = httplib.HTTPResponse(FakeSocket(body)) + with self.assertRaisesRegexp(httplib.HTTPException, + "got more than 100 headers"): + resp.begin() + + def test_headers_after_100(self): + # A 100 Continue response with a few headers is still skipped. + body = ( + 'HTTP/1.1 100 Continue\r\nX-A: 1\r\nX-B: 2\r\n\r\n' + 'HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nHello' + ) + resp = httplib.HTTPResponse(FakeSocket(body)) + resp.begin() + self.assertEqual(resp.status, 200) + self.assertEqual(resp.read(), 'Hello') + + def test_too_many_interim_responses(self): + # gh-150743 (CVE-2021-3737 follow-up): a server streaming + # "100 Continue" responses forever must not hang getresponse(). + body = ('HTTP/1.1 100 Continue\r\n\r\n' + * (httplib._MAXINTERIMRESPONSES + 1)) + resp = httplib.HTTPResponse(FakeSocket(body)) + with self.assertRaisesRegexp(httplib.HTTPException, + "got more than 100 interim responses"): + resp.begin() + + def test_multiple_interim_responses(self): + # A reasonable number of interim responses before the final + # response is skipped as before. + body = ( + 'HTTP/1.1 100 Continue\r\n\r\n' * 3 + + 'HTTP/1.1 200 OK\r\n' + 'Content-Length: 5\r\n' + '\r\n' + 'hello' + ) + resp = httplib.HTTPResponse(FakeSocket(body), method="GET") + resp.begin() + self.assertEqual(resp.status, 200) + self.assertEqual(resp.read(), 'hello') + resp.close() + + def test_chunked_trailers(self): + # A few trailer lines after the last chunk are read and discarded. + body = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + '5\r\nhello\r\n0\r\n' + 'X-Trailer1: 1\r\nX-Trailer2: 2\r\n\r\n' + ) + resp = httplib.HTTPResponse(FakeSocket(body), method="GET") + resp.begin() + self.assertEqual(resp.read(), 'hello') + resp.close() + + def test_chunked_too_many_trailers(self): + # gh-150743 (CVE-2021-3737 follow-up): a response streaming endless + # trailer lines must raise, not hang. + body = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + '5\r\nhello\r\n0\r\n' + + ''.join('X-Trailer%d: %d\r\n' % (i, i) + for i in range(httplib._MAXHEADERS + 1)) + + '\r\n' + ) + # An unbounded read() reaches the trailers via the final 0 chunk. + resp = httplib.HTTPResponse(FakeSocket(body), method="GET") + resp.begin() + with self.assertRaisesRegexp(httplib.HTTPException, + "got more than 100 trailers"): + resp.read() + resp.close() + + # A bounded read(amt) larger than the body hits the same limit. + resp = httplib.HTTPResponse(FakeSocket(body), method="GET") + resp.begin() + with self.assertRaisesRegexp(httplib.HTTPException, + "got more than 100 trailers"): + resp.read(6) + resp.close() + def test_overflowing_chunked_line(self): body = ( 'HTTP/1.1 200 OK\r\n' diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 25236c6ef41ad2..9aa1b1ac43f3d7 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -351,6 +351,55 @@ def check_status_and_reason(self, response, status, data=None): if data: self.assertEqual(data, body) + def test_get_dir_redirect_location_domain_injection_bug(self): + """Ensure //evil.co/..%2f../../X does not put //evil.co/ in Location. + + //netloc/ in a Location header is a redirect to a new host. + https://github.com/python/cpython/issues/87389 + + This checks that a path resolving to a directory on our server cannot + resolve into a redirect to another server. + """ + os.mkdir(os.path.join(self.tempdir, 'existing_directory')) + url = ('/python.org/..%2f..%2f..%2f..%2f..%2f../%0a%0d/../' + + self.tempdir_name + '/existing_directory') + expected_location = url + '/' # single slash prefix, trailing slash + # Canonicalizes to /tmp/tempdir_name/existing_directory which does + # exist and is a dir, triggering the 301 redirect logic. + response = self.request(url) + self.check_status_and_reason(response, 301) + location = response.getheader('Location') + self.assertEqual(location, expected_location, msg='non-attack failed!') + + # //python.org... multi-slash prefix, no trailing slash + attack_url = '/' + url + response = self.request(attack_url) + self.check_status_and_reason(response, 301) + location = response.getheader('Location') + self.assertFalse(location.startswith('//'), msg=location) + self.assertEqual(location, expected_location, + msg='Expected Location header to start with a single / and ' + 'end with a / as this is a directory redirect.') + + # ///python.org... triple-slash prefix, no trailing slash + attack3_url = '//' + url + response = self.request(attack3_url) + self.check_status_and_reason(response, 301) + self.assertEqual(response.getheader('Location'), expected_location) + + # If the second word in the http request (Request-URI for the http + # method) is a full URI, we don't worry about it, as that'll be parsed + # and reassembled as a full URI within BaseHTTPRequestHandler.send_head + # so no errant scheme-less //netloc//evil.co/ domain mixup can happen. + attack_scheme_netloc_2slash_url = 'https://pypi.org/' + url + response = self.request(attack_scheme_netloc_2slash_url) + self.check_status_and_reason(response, 301) + location = response.getheader('Location') + # We're just ensuring that the scheme and domain make it through, if + # there are or aren't multiple slashes at the start of the path that + # follows that isn't important in this Location: header. + self.assertTrue(location.startswith('https://pypi.org/'), msg=location) + def test_get(self): #constructs the path relative to the root directory of the HTTPServer response = self.request(self.base_url + '/test') diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index e9a7216d1c6c21..2c8b7fd2137dee 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -8,6 +8,8 @@ from test import test_support # Tell it we don't know about external files: +# The default list, before it is emptied below for test isolation. +knownfiles = mimetypes.knownfiles mimetypes.knownfiles = [] mimetypes.inited = False mimetypes._default_mime_types() @@ -67,6 +69,12 @@ def test_guess_all_types(self): eq(all, []) +class KnownfilesTestCase(unittest.TestCase): + @unittest.skipIf(sys.platform == "win32", "non-Windows only") + def test_default_knownfiles(self): + self.assertIn("/etc/mime.types", knownfiles) + + @unittest.skipUnless(sys.platform.startswith("win"), "Windows only") class Win32MimeTypesTestCase(unittest.TestCase): def setUp(self): @@ -90,6 +98,10 @@ def test_registry_parsing(self): eq(db.guess_type("image.jpg"), ("image/jpeg", None)) eq(db.guess_type("image.png"), ("image/png", None)) + def test_default_knownfiles(self): + # CVE-2024-3220: the Unix paths would resolve to e.g. C:\etc\mime.types + self.assertEqual(knownfiles, []) + def test_non_latin_extension(self): import _winreg @@ -163,6 +175,7 @@ def __getattr__(self, name): def test_main(): test_support.run_unittest(MimeTypesTestCase, + KnownfilesTestCase, Win32MimeTypesTestCase ) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index dcd486e2b1856d..25cb19435fdc41 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -509,8 +509,20 @@ def test_makedir(self): 'dir5', 'dir6') os.makedirs(path) - - + @unittest.skipUnless(os.name == 'nt', "requires Windows") + def test_win32_mkdir_700(self): + # gh-118486 (CVE-2024-4030): mode 0700 gets an ACL that only allows + # SYSTEM, Administrators and the owner. + path = os.path.abspath(os.path.join(test_support.TESTFN, 'dir')) + os.mkdir(path, 0700) + try: + out = subprocess.check_output(["cacls.exe", path, "/s"]) + finally: + os.rmdir(path) + self.assertEqual( + out.strip(), + '%s "D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)"' % path, + ) def tearDown(self): path = os.path.join(test_support.TESTFN, 'dir1', 'dir2', 'dir3', diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index 72840bd3128089..abd173daab90c1 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -255,6 +255,18 @@ def test_parse_again(self): self.assertEqual(expat.ErrorString(cm.exception.code), expat.errors.XML_ERROR_FINISHED) + def test_deeply_nested_entities(self): + # CVE-2024-8176: Expat < 2.7.0 expanded nested general entities + # recursively and overflowed the C stack on a long enough chain. + n = 200000 + decl = ''.join('' % (i, i + 1) for i in range(n)) + doc = ']>&e0;' % (decl, n) + out = [] + parser = expat.ParserCreate() + parser.CharacterDataHandler = out.append + parser.Parse(doc, True) + self.assertEqual(''.join(out), 'x') + class NamespaceSeparatorTest(unittest.TestCase): def test_legal(self): # Tests that make sure we get errors when the namespace_separator value @@ -666,6 +678,57 @@ def resolve_entity(context, base, system_id, public_id): self.assertEqual(handler_call_args, [("bar", "baz")]) +class ElementDeclHandlerTest(unittest.TestCase): + def test_deeply_nested_content_model(self): + # This should raise a RuntimeError (recursion limit) and not crash. + # See https://github.com/python/cpython/issues/145986. + N = 500000 + data = ( + '\n]>\n\n' + ) + + parser = expat.ParserCreate() + parser.ElementDeclHandler = lambda _1, _2: None + with self.assertRaises(RuntimeError): + parser.Parse(data, True) + + def test_nested_content_model(self): + # A modestly nested content model is still converted. + models = [] + parser = expat.ParserCreate() + parser.ElementDeclHandler = lambda name, model: models.append(model) + parser.Parse('\n' + '\n]>\n\n', True) + self.assertEqual(len(models), 2) + self.assertEqual(len(models[0][3]), 2) + + +@unittest.skipIf(expat.version_info < (2, 8, 0), + "XML_SetHashSalt16Bytes needs expat >= 2.8.0") +class HashSaltTest(unittest.TestCase): + # gh-149018 (CVE-2026-7210): parsers get 16 bytes of OS entropy as their + # hash salt instead of the 0 from _Py_HashSecret (without -R), which made + # expat fall back to its own time-based salt. EXPAT_ENTROPY_DEBUG makes + # expat report on stderr where each parser's salt came from. + def check_salt(self, code): + from test.script_helper import assert_python_ok + rc, out, err = assert_python_ok('-c', code, EXPAT_ENTROPY_DEBUG='1') + lines = [l for l in err.splitlines() if l.startswith('expat: Entropy')] + self.assertTrue(lines, err) + for line in lines: + self.assertIn('explicit(16)', line) + + def test_pyexpat(self): + self.check_salt("import pyexpat\n" + "pyexpat.ParserCreate().Parse('
', True)") + + def test_celementtree(self): + self.check_salt("import xml.etree.cElementTree as ET\n" + "ET.fromstring('')") + + def test_main(): run_unittest(SetAttributeTest, ParseTest, @@ -677,7 +740,9 @@ def test_main(): sf1296433Test, ChardataBufferTest, MalformedInputText, - ForeignDTDTests) + ForeignDTDTests, + ElementDeclHandlerTest, + HashSaltTest) if __name__ == "__main__": test_main() diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 2381baa847cd7b..8c689f7aaa3276 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -6,8 +6,6 @@ from test import test_support as support from test.script_helper import assert_python_ok import asyncore -# import unittest.mock -from mock import Mock from test import support import re import socket @@ -3304,6 +3302,104 @@ def cb_wrong_return_type(ssl_sock, server_name, initial_context): self.assertEqual(cm.exception.reason, 'TLSV1_ALERT_INTERNAL_ERROR') self.assertIn("TypeError", stderr.getvalue()) + def sni_handshake_released_context(self, sni_cb, ecdh_curve=None): + # Run a handshake over a socketpair whose server side was wrapped + # by a context that only the C-level SSL object references once + # sni_cb has switched the connection to another context. + # Returns (server socket, server handshake error or None, + # weakref to the released context). + def make_server(sock): + dispatch_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + dispatch_ctx.load_cert_chain(SIGNED_CERTFILE) + if ecdh_curve is not None: + dispatch_ctx.set_ecdh_curve(ecdh_curve) + dispatch_ctx.set_servername_callback(sni_cb) + server = dispatch_ctx.wrap_socket( + sock, server_side=True, do_handshake_on_connect=False) + return server, weakref.ref(dispatch_ctx) + + s1, s2 = [socket.socket(_sock=s) for s in socket.socketpair()] + self.addCleanup(s1.close) + self.addCleanup(s2.close) + server, dispatch_ref = make_server(s1) + client_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + client = client_ctx.wrap_socket(s2, server_hostname='localhost', + do_handshake_on_connect=False) + self.addCleanup(client.close) + self.addCleanup(server.close) + client_thread = threading.Thread(target=self._quiet_handshake, + args=(client,)) + client_thread.start() + error = None + try: + server.do_handshake() + except ssl.SSLError as e: + error = e + finally: + if error is not None: + s1.shutdown(socket.SHUT_RDWR) + client_thread.join() + gc.collect() + return server, error, dispatch_ref + + @staticmethod + def _quiet_handshake(sock): + try: + sock.do_handshake() + except (ssl.SSLError, socket.error): + pass + + @needs_sni + def test_sni_callback_context_released_and_callback_raises(self): + # CVE-2026-19445: the callback switches the connection to another + # context, drops the last reference to the context that carries + # it, and raises. The C callback must not touch that context + # after the Python callback returned. + leaf_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + leaf_ctx.load_cert_chain(SIGNED_CERTFILE) + + def sni_cb(ssl_sock, server_name, ctx): + ssl_sock.context = leaf_ctx + del ctx + raise LookupError("no certificate for %r" % (server_name,)) + + with support.captured_stderr() as stderr: + server, error, dispatch_ref = \ + self.sni_handshake_released_context(sni_cb) + self.assertIsInstance(error, ssl.SSLError) + self.assertIn("LookupError", stderr.getvalue()) + # The unraisable error names the callback. Read through the + # freed context it was (and writing that to a real stderr + # crashed). + self.assertIn("sni_cb", stderr.getvalue()) + self.assertIs(server.context, leaf_ctx) + self.assertIsNone(dispatch_ref()) + + @needs_sni + @unittest.skipUnless(ssl.HAS_TLSv1_3, "needs TLS 1.3") + def test_sni_callback_context_released_before_second_client_hello(self): + # CVE-2026-19445: after the callback switched the connection to + # another context, the server forces a HelloRetryRequest (the + # client offers an X25519 key share first, the server accepts + # only P-384). The second ClientHello must not reach the + # deallocated context through the original SSL_CTX. + leaf_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + leaf_ctx.load_cert_chain(SIGNED_CERTFILE) + calls = [] + + def sni_cb(ssl_sock, server_name, ctx): + calls.append(server_name) + ssl_sock.context = leaf_ctx + + server, error, dispatch_ref = \ + self.sni_handshake_released_context(sni_cb, "secp384r1") + self.assertIsNone(error) + self.assertIsNone(dispatch_ref()) + self.assertGreaterEqual(len(calls), 1) + self.assertEqual(calls[0], 'localhost') + self.assertIs(server.context, leaf_ctx) + self.assertIsNotNone(server.cipher()) + def test_read_write_after_close_raises_valuerror(self): context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) context.verify_mode = ssl.CERT_REQUIRED @@ -3479,8 +3575,8 @@ def test_keylog_filename(self): "test is not compatible with ignore_environment") def test_keylog_env(self): self.addCleanup(support.unlink, support.TESTFN) - with mock.patch.dict(os.environ): - os.environ['SSLKEYLOGFILE'] = support.TESTFN + with support.EnvironmentVarGuard() as env: + env['SSLKEYLOGFILE'] = support.TESTFN self.assertEqual(os.environ['SSLKEYLOGFILE'], support.TESTFN) ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index fe66218eaa826f..8fa499f55bda64 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -592,6 +592,22 @@ def test_provoke_stream_error(self): f = self.tar.extractfile(tarinfos[0]) # read the first member self.assertRaises(tarfile.StreamError, f.read) + def test_getmembers_big_size(self): + # gh-151981: A loop in seek() for streaming files tried to read the + # declared number of blocks even at EOF + for format in (tarfile.GNU_FORMAT, tarfile.PAX_FORMAT): + tinfo = tarfile.TarInfo("huge-file") + tinfo.size = 1 << 64 + bio = io.BytesIO() + # Write header without data + bio.write(tinfo.tobuf(format)) + + # Reset & try to get contents + bio.seek(0) + with tarfile.open(fileobj=bio, mode="r|") as tar: + with self.assertRaises(tarfile.ReadError): + tar.getmembers() + def test_compare_members(self): tar1 = tarfile.open(tarname, encoding="iso8859-1") try: diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 2efb8362fe1bd5..f3387d77e994ac 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -649,6 +649,33 @@ def test_mode(self): finally: os.rmdir(dir) + @unittest.skipUnless(os.name == "nt", "Only on Windows.") + def test_mode_win32(self): + # gh-118486 (CVE-2024-4030): use icacls.exe to extract the users with + # some level of access. Main thing we are testing is that the + # BUILTIN\Users group has no access. The exact ACL is going to vary + # based on which user is running the test. + import subprocess + dir = self.do_create() + try: + out = subprocess.check_output(["icacls.exe", dir]).lower() + finally: + os.rmdir(dir) + + dir = dir.lower() + users = set() + for line in out.strip().splitlines(): + acl = None + # First line of result includes our directory + if line.startswith(dir): + acl = line[len(dir):].strip() + elif line and line[:1].isspace(): + acl = line.strip() + if acl: + users.add(acl.partition(":")[0]) + + self.assertNotIn(r"BUILTIN\Users".lower(), users) + def test_collision_with_existing_file(self): # mkdtemp tries another name when a file with # the chosen name already exists diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 11f2cda8209986..cd406d90ce3269 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -204,6 +204,44 @@ def test_issue10254(self): b = u'C\u0338' * 20 + u'\xC7' self.assertEqual(self.db.normalize('NFC', a), b) + def test_long_combining_mark_run(self): + # gh-149079 (CVE-2026-3276): avoid quadratic canonical ordering. + payload = u"a" + (u"\u0300\u0327" * 32) + nfd = u"a" + (u"\u0327" * 32) + (u"\u0300" * 32) + nfc = u"\u00e0" + (u"\u0327" * 32) + (u"\u0300" * 31) + + self.assertEqual(self.db.normalize("NFD", payload), nfd) + self.assertEqual(self.db.normalize("NFKD", payload), nfd) + self.assertEqual(self.db.normalize("NFC", payload), nfc) + self.assertEqual(self.db.normalize("NFKC", payload), nfc) + + def test_combining_mark_run_fast_paths(self): + # gh-149079: cover short runs and already-sorted long runs. + short_payload = u"a" + (u"\u0300\u0327" * 9) + u"\u0300" + short_nfd = u"a" + (u"\u0327" * 9) + (u"\u0300" * 10) + short_nfc = u"\u00e0" + (u"\u0327" * 9) + (u"\u0300" * 9) + long_sorted = u"a" + (u"\u0327" * 30) + (u"\u0300" * 30) + long_sorted_nfc = u"\u00e0" + (u"\u0327" * 30) + (u"\u0300" * 29) + + self.assertEqual(self.db.normalize("NFD", short_payload), short_nfd) + self.assertEqual(self.db.normalize("NFKD", short_payload), short_nfd) + self.assertEqual(self.db.normalize("NFC", short_payload), short_nfc) + self.assertEqual(self.db.normalize("NFKC", short_payload), short_nfc) + self.assertEqual(self.db.normalize("NFD", long_sorted), long_sorted) + self.assertEqual(self.db.normalize("NFKD", long_sorted), long_sorted) + self.assertEqual(self.db.normalize("NFC", long_sorted), long_sorted_nfc) + self.assertEqual(self.db.normalize("NFKC", long_sorted), long_sorted_nfc) + + def test_combining_mark_run_is_linear(self): + # gh-149079 (CVE-2026-3276): a run of 100000 combining marks in + # alternating canonical combining class order took about 1.5 s with + # the quadratic sort (and 20 s for 400000); it now takes a few + # milliseconds. This checks the counting sort on a long run. + n = 50000 + payload = u"a" + (u"\u0300\u0327" * n) + nfd = u"a" + (u"\u0327" * n) + (u"\u0300" * n) + self.assertEqual(self.db.normalize("NFD", payload), nfd) + def test_issue29456(self): # Fix #29456 u1176_str_a = u'\u1100\u1176\u11a8' diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 20a0f581436d64..a289bf60dc86d2 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1265,6 +1265,62 @@ def _test_basic_auth(self, opener, auth_handler, auth_header, self.assertEqual(len(http_handler.requests), 1) self.assertFalse(http_handler.requests[0].has_header(auth_header)) +class PasswordManagerSchemeTests(unittest.TestCase): + + def test_password_manager_scheme(self): + mgr = urllib2.HTTPPasswordMgr() + mgr.add_password( + "realm", "https://example.com/", "user", "password") + + self.assertEqual( + mgr.find_user_password("realm", "https://example.com/"), + ("user", "password")) + self.assertEqual( + mgr.find_user_password("realm", "http://example.com/"), + (None, None)) + # Support an authority without a scheme. + self.assertEqual( + mgr.find_user_password("realm", "example.com"), + ("user", "password")) + # An authority without a scheme continues to match any scheme. + mgr.add_password( + "realm", "schemeless.example.com", "user", "password") + for scheme in "http", "https": + self.assertEqual( + mgr.find_user_password( + "realm", "%s://schemeless.example.com/" % scheme), + ("user", "password")) + + # A network-path reference also has no scheme. + mgr.add_password( + "realm", "//network-path.example.com/", "user", "password") + self.assertEqual( + mgr.find_user_password( + "realm", "https://network-path.example.com/"), + ("user", "password")) + + def test_password_manager_default_realm_scheme(self): + mgr = urllib2.HTTPPasswordMgrWithDefaultRealm() + mgr.add_password(None, "https://example.com/", "user", "password") + self.assertEqual( + mgr.find_user_password("realm", "https://example.com/x"), + ("user", "password")) + self.assertEqual( + mgr.find_user_password("realm", "http://example.com/x"), + (None, None)) + + def test_password_manager_reduced_uri(self): + mgr = urllib2.HTTPPasswordMgr() + + self.assertEqual( + mgr.reduce_uri("http://example.com/path"), + ("example.com:80", "/path")) + self.assertTrue( + mgr.is_suburi( + ("example.com", "/path"), + ("example.com", "/path/subpath"))) + + class MiscTests(unittest.TestCase, FakeHTTPMixin): def test_build_opener(self): @@ -1485,6 +1541,7 @@ def test_main(verbose=None): OpenerDirectorTests, HandlerTests, MiscTests, + PasswordManagerSchemeTests, RequestTests) test_support.run_unittest(*tests) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index fed93f1d907f0b..119f87c9957619 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1906,6 +1906,40 @@ def test_find_xpath(self): self.assertEqual(e.find('./tag[last()-1]').attrib['class'], 'c') self.assertEqual(e.find('./tag[last()-2]').attrib['class'], 'b') + def test_find_xpath_index_no_quadratic_complexity(self): + if not isinstance(ET.Element, type): + self.skipTest("Element is not subclassable in this implementation") + + class CountingElement(ET.Element): + findall_calls = 0 + def findall(self, *args, **kwargs): + type(self).findall_calls += 1 + return super(CountingElement, self).findall(*args, **kwargs) + + def work(n, pattern): + root = CountingElement("root") + for _ in range(n): + ET.SubElement(root, "a") + CountingElement.findall_calls = 0 + root.findall(pattern) + return CountingElement.findall_calls + + for pattern in [".//a[1]", ".//a[last()]"]: + w1 = work(1024, pattern) + w2 = work(2048, pattern) + w3 = work(4096, pattern) + + self.assertGreater(w1, 0) + r1 = float(w2) / w1 + r2 = float(w3) / w2 + # Doubling N must not ~double the parent.findall calls. + # Linear-in-N call counts indicate the cache is missing. + self.assertLess( + max(r1, r2), 1.5, + msg="Possible quadratic behavior on %r: calls=%r ratios=%r" + % (pattern, (w1, w2, w3), (r1, r2)), + ) + def test_findall(self): e = ET.XML(SAMPLE_XML) e[2] = ET.XML(SAMPLE_SECTION) diff --git a/Lib/urllib2.py b/Lib/urllib2.py index 71e05bfd24eea7..9548f959004952 100644 --- a/Lib/urllib2.py +++ b/Lib/urllib2.py @@ -788,16 +788,17 @@ def add_password(self, realm, uri, user, passwd): self.passwd[realm] = {} for default_port in True, False: reduced_uri = tuple( - [self.reduce_uri(u, default_port) for u in uri]) + [self._reduce_uri_with_scheme(u, default_port) for u in uri]) self.passwd[realm][reduced_uri] = (user, passwd) def find_user_password(self, realm, authuri): domains = self.passwd.get(realm, {}) for default_port in True, False: - reduced_authuri = self.reduce_uri(authuri, default_port) + reduced_authuri = self._reduce_uri_with_scheme( + authuri, default_port) for uris, authinfo in domains.iteritems(): for uri in uris: - if self.is_suburi(uri, reduced_authuri): + if self._is_suburi_with_scheme(uri, reduced_authuri): return authinfo return None, None @@ -824,6 +825,20 @@ def reduce_uri(self, uri, default_port=True): authority = "%s:%d" % (host, dport) return authority, path + def _reduce_uri_with_scheme(self, uri, default_port=True): + # Like reduce_uri(), but keep the scheme (None for a bare authority + # or a network-path reference) so that credentials added for one + # scheme are not sent over another, e.g. https -> http. + parts = urlparse.urlsplit(uri) + scheme = parts[0] if parts[1] else None + return (scheme or None,) + self.reduce_uri(uri, default_port) + + def _is_suburi_with_scheme(self, base, test): + if (base[0] is not None and test[0] is not None and + base[0] != test[0]): + return False + return self.is_suburi(base[1:], test[1:]) + def is_suburi(self, base, test): """Check if test is below base in a URI tree diff --git a/Lib/xml/etree/ElementPath.py b/Lib/xml/etree/ElementPath.py index 4a626d799c827d..efc05f3748bb7b 100644 --- a/Lib/xml/etree/ElementPath.py +++ b/Lib/xml/etree/ElementPath.py @@ -211,15 +211,22 @@ def select(context, result): index = -1 def select(context, result): parent_map = get_parent_map(context) + cache = {} for elem in result: try: parent = parent_map[elem] + except KeyError: + continue + key = (parent, elem.tag) + if key not in cache: # FIXME: what if the selector is "*" ? elems = list(parent.findall(elem.tag)) - if elems[index] is elem: - yield elem - except (IndexError, KeyError): - pass + try: + cache[key] = elems[index] + except IndexError: + cache[key] = None + if cache[key] is elem: + yield elem return select raise SyntaxError("invalid predicate") diff --git a/Misc/NEWS.d/2.7.18.16.rst b/Misc/NEWS.d/2.7.18.16.rst new file mode 100644 index 00000000000000..58a4d70473933e --- /dev/null +++ b/Misc/NEWS.d/2.7.18.16.rst @@ -0,0 +1,258 @@ +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as0116 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +Fix a hang in tarfile streaming mode (CVE-2026-11972) + +Seeking forward in a :mod:`tarfile` stream opened in ``r|`` mode stopped +only at the target offset, so a member header declaring a huge size made it +loop forever at end of file. It now stops at EOF and raises +:exc:`tarfile.ReadError` ("unexpected end of data"). + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as0216 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +Fix quadratic time in csv.Sniffer on quoted fields (CVE-2026-18503) + +The regular expression :class:`csv.Sniffer` uses to guess the quote character +backtracked quadratically on long quoted fields. Python 2.7's :mod:`re` has +no possessive quantifiers, so the upstream fix is expressed with an +equivalent lookahead and backreference. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as0316 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +ftplib.ftpcp() no longer trusts the host in a PASV reply (CVE-2026-8328) + +:func:`ftplib.ftpcp` connected the target server to the host address given +in the source server's ``PASV`` reply, letting a malicious server direct the +transfer to another host. It now uses the source server's actual peer +address, as :class:`ftplib.FTP` already does for its own transfers. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as0416 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +Reject malformed Content-Length and chunk sizes in httplib (CVE-2026-103995) + +:mod:`httplib` now accepts only decimal digits (with optional surrounding +whitespace) in ``Content-Length``, and only hexadecimal digits for a chunk +size. Values such as ``+5``, ``-0`` or ``0x10``, which other HTTP +implementations parse differently, are rejected or ignored instead of being +used. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as0516 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +Limit what httplib reads around interim responses and trailers (CVE-2021-3737) + +:class:`httplib.HTTPResponse` now reads at most 100 header lines inside a +``100 Continue`` response, skips at most 100 interim (1xx) responses, and +discards at most 100 trailer lines after a chunked body +(GHSA-w4q2-g22w-6fr4). Each limit raises :exc:`httplib.HTTPException` when +exceeded, instead of reading for as long as the server keeps sending. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as0616 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +Fix an open redirect in SimpleHTTPServer (CVE-2021-28861) + +A request path starting with ``//`` could make +:class:`SimpleHTTPServer.SimpleHTTPRequestHandler` redirect to another +host (``Location: //evil.example/...``). Leading slashes are now collapsed +before the path is used. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as0716 +.. release date: 2026-10-28 +.. original section: Windows +.. section: Security + +mimetypes no longer reads files from fixed paths on Windows (CVE-2024-3220) + +On Windows, :mod:`mimetypes` no longer reads the Unix ``knownfiles`` paths +(such as ``C:\etc\mime.types``), which any local user may be able to +create. The registry is still used. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as0816 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +Fix a crash in pyexpat on deeply nested content models (CVE-2026-4224) + +Converting a deeply nested DTD content model for an ``ElementDeclHandler`` +recursed without limit in C and could overflow the stack. It now raises +:exc:`RuntimeError` when the recursion limit is reached. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as0916 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +bz2 objects can no longer be reused after a data error (CVE-2026-9669) + +Calling into libbz2 again after it reported an error could write out of +bounds. :class:`bz2.BZ2Decompressor` and, for reading, :class:`bz2.BZ2File` +now raise :exc:`ValueError` once an error has occurred. Seeking a +:class:`bz2.BZ2File` starts over from a newly opened stream. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as1016 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +Fix quadratic time in ElementPath index predicates (CVE-2026-6879) + +``[1]``, ``[last()]`` and similar index predicates in +:mod:`xml.etree.ElementTree` path expressions re-scanned the parent for every +element, which was quadratic in the number of siblings. Results are now +computed once per parent. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as1116 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +urllib2 password managers no longer send https credentials over http (CVE-2026-15806) + +Credentials added to :class:`urllib2.HTTPPasswordMgr` for an ``https`` URL +were also returned for the same host over ``http``. The URL scheme is now +part of the match; an entry given without a scheme still matches any +scheme. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as1216 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +Fix quadratic time in HTMLParser incremental parsing (CVE-2026-15308) + +:class:`HTMLParser.HTMLParser` copied its whole buffer on every +:meth:`~HTMLParser.HTMLParser.feed` call while a construct was still open, +so feeding a document in small pieces was quadratic. Data is now buffered +until it can be parsed. As upstream, events after such a construct may be +reported later, but none are lost. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as1316 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +Update the bundled Expat to 2.9.0 + +The Expat in ``Modules/expat`` is updated from 2.6.4 to 2.9.0, fixing +CVE-2024-8176, CVE-2025-59375, CVE-2026-24515, CVE-2026-25210, +CVE-2026-32776, CVE-2026-32777, CVE-2026-32778, CVE-2026-41080, +CVE-2026-45186, CVE-2026-50219, CVE-2026-56131, CVE-2026-56132, +CVE-2026-56403 to CVE-2026-56412, CVE-2026-66046, CVE-2026-72522, +CVE-2026-76641, CVE-2026-76956, CVE-2026-76957, CVE-2026-77214, +CVE-2026-93990 and CVE-2026-102633. On Windows, :mod:`pyexpat` links the +external Expat library, which should be 2.9.0 as well. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as1416 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +Give Expat parsers a random 16-byte hash salt (CVE-2026-7210) + +:mod:`pyexpat` and :mod:`xml.etree.cElementTree` now seed every parser's +hash-flooding protection with 16 bytes from the operating system's random +source (``XML_SetHashSalt16Bytes``), instead of a value that is zero unless +hash randomization is enabled, which made Expat fall back to a weak +time-based salt. Requires Expat 2.8.0 or later. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as1516 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +Fix a use-after-free in the ssl SNI callback (CVE-2026-19445) + +A servername callback that switched the connection to another +:class:`ssl.SSLContext` while the original context was released could make +the next callback read freed memory and crash the server. The callback now +finds its context through the connection and keeps it alive while in use. +After a switch, later ClientHello messages go to the new context's +callback. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as1616 +.. release date: 2026-10-28 +.. original section: Library +.. section: Security + +Fix quadratic time in unicodedata.normalize() (CVE-2026-3276) + +Canonical ordering of a long run of combining characters in alternating +order was quadratic. Each run is now sorted separately, with a linear +counting sort for long runs. The results are unchanged. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as1716 +.. release date: 2026-10-28 +.. original section: Windows +.. section: Security + +os.mkdir(path, 0700) restricts access on Windows (CVE-2024-4030) + +On Windows, :func:`os.mkdir` ignored its *mode*, so directories created by +:func:`tempfile.mkdtemp` were readable by other users. A mode of ``0700`` +now creates the directory with an access control list that allows only +SYSTEM, Administrators and the owner. Other modes are still ignored. + +.. bpo: 0 +.. date: 2026-10-28 +.. nonce: as1816 +.. release date: 2026-10-28 +.. original section: Build +.. section: Build + +Fix building _socket without optimization + +Removed an unused function from ``Modules/socketmodule.c`` that referenced +a symbol Python 2.7 does not build, so ``-O0`` and ``--with-pydebug`` builds +produced a ``_socket`` module (and :mod:`ssl`) that failed to import. +``test_ssl`` no longer needs the third-party ``mock`` package. diff --git a/Modules/Setup.dist b/Modules/Setup.dist index bbc9222129641b..6fed5557b5906b 100644 --- a/Modules/Setup.dist +++ b/Modules/Setup.dist @@ -469,7 +469,7 @@ GLHACK=-Dclear=__GLclear # Interface to the Expat XML parser # More information on Expat can be found at www.libexpat.org. # -#pyexpat expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DXML_POOR_ENTROPY=1 -DUSE_PYEXPAT_CAPI +#pyexpat expat/xcs.c expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DXML_POOR_ENTROPY=1 -DUSE_PYEXPAT_CAPI # Hye-Shik Chang's CJKCodecs diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index bbaf9941048592..7cbfa69f81f672 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -2059,6 +2059,9 @@ statichere PyTypeObject TreeBuilder_Type = { #include "pyexpat.h" static struct PyExpat_CAPI* expat_capi; #define EXPAT(func) (expat_capi->func) +/* Hash salt for SetHashSalt16Bytes(), see gh-149018 and pyexpat.c. */ +static unsigned char expat_hash_salt16[16]; +static int expat_hash_salt16_set = 0; #else #define EXPAT(func) (XML_##func) #endif @@ -2592,6 +2595,13 @@ xmlparser(PyObject* self_, PyObject* args, PyObject* kw) PyErr_NoMemory(); return NULL; } +#if defined(USE_PYEXPAT_CAPI) + /* Prefer 16 bytes of entropy, only expat >= 2.8.0. See gh-149018. */ + if (expat_hash_salt16_set && EXPAT(SetHashSalt16Bytes) != NULL) { + EXPAT(SetHashSalt16Bytes)(self->parser, expat_hash_salt16); + } + else +#endif /* expat < 2.1.0 has no XML_SetHashSalt() */ if (EXPAT(SetHashSalt) != NULL) { EXPAT(SetHashSalt)(self->parser, @@ -3209,6 +3219,10 @@ init_elementtree(void) expat_capi->MICRO_VERSION != XML_MICRO_VERSION) expat_capi = NULL; } + if (_PyOS_URandom(expat_hash_salt16, sizeof(expat_hash_salt16)) == 0) + expat_hash_salt16_set = 1; + else + PyErr_Clear(); #endif elementtree_parseerror_obj = PyErr_NewException( diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 0df84021d3814a..232fff61af61e6 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -2413,6 +2413,11 @@ context_dealloc(PySSLContext *self) /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(self); context_clear(self); +#if HAVE_SNI && !defined(OPENSSL_NO_TLSEXT) + /* The SSL_CTX may outlive this object as the session_ctx of sockets that + were switched to another context; leave no Python callback behind. */ + SSL_CTX_set_tlsext_servername_callback(self->ctx, NULL); +#endif SSL_CTX_free(self->ctx); #if HAVE_NPN PyMem_FREE(self->npn_protocols); @@ -3430,31 +3435,46 @@ static int _servername_callback(SSL *s, int *al, void *args) { int ret; - PySSLContext *ssl_ctx = (PySSLContext *) args; + PySSLContext *ssl_ctx; PySSLSocket *ssl; PyObject *servername_o; PyObject *servername_idna; PyObject *result; /* The high-level ssl.SSLSocket object */ PyObject *ssl_socket; + PyObject *sni_cb; const char *servername = SSL_get_servername(s, TLSEXT_NAMETYPE_host_name); #ifdef WITH_THREAD PyGILState_STATE gstate = PyGILState_Ensure(); #endif - if (ssl_ctx->set_hostname == NULL) { - /* remove race condition in this the call back while if removing the - * callback is in progress */ + /* Do not use the SSL_CTX's servername arg to find the context: it is a + borrowed pointer to whichever _SSLContext installed the callback, and + that object may already be gone while OpenSSL still reaches this + callback through the connection's session_ctx (e.g. on the second + ClientHello after a HelloRetryRequest, once the callback has switched + the socket to another context). The socket's current context is + always alive; hold strong references to it and to the callback while + they are used here. */ + (void)args; + ssl = SSL_get_app_data(s); + assert(ssl != NULL && PySSLSocket_Check(ssl)); + ssl_ctx = ssl->ctx; + Py_INCREF(ssl_ctx); + sni_cb = ssl_ctx->set_hostname; + if (sni_cb == NULL) { + Py_DECREF(ssl_ctx); #ifdef WITH_THREAD PyGILState_Release(gstate); #endif return SSL_TLSEXT_ERR_OK; } + Py_INCREF(sni_cb); - ssl = SSL_get_app_data(s); - assert(PySSLSocket_Check(ssl)); if (ssl->ssl_sock == NULL) { + /* The error path below releases ssl_socket. */ ssl_socket = Py_None; + Py_INCREF(ssl_socket); } else { ssl_socket = PyWeakref_GetObject(ssl->ssl_sock); Py_INCREF(ssl_socket); @@ -3464,7 +3484,7 @@ _servername_callback(SSL *s, int *al, void *args) } if (servername == NULL) { - result = PyObject_CallFunctionObjArgs(ssl_ctx->set_hostname, ssl_socket, + result = PyObject_CallFunctionObjArgs(sni_cb, ssl_socket, Py_None, ssl_ctx, NULL); } else { @@ -3480,14 +3500,14 @@ _servername_callback(SSL *s, int *al, void *args) goto error; } Py_DECREF(servername_o); - result = PyObject_CallFunctionObjArgs(ssl_ctx->set_hostname, ssl_socket, + result = PyObject_CallFunctionObjArgs(sni_cb, ssl_socket, servername_idna, ssl_ctx, NULL); Py_DECREF(servername_idna); } Py_DECREF(ssl_socket); if (result == NULL) { - PyErr_WriteUnraisable(ssl_ctx->set_hostname); + PyErr_WriteUnraisable(sni_cb); *al = SSL_AD_HANDSHAKE_FAILURE; ret = SSL_TLSEXT_ERR_ALERT_FATAL; } @@ -3506,6 +3526,8 @@ _servername_callback(SSL *s, int *al, void *args) Py_DECREF(result); } + Py_DECREF(sni_cb); + Py_DECREF(ssl_ctx); #ifdef WITH_THREAD PyGILState_Release(gstate); #endif @@ -3513,6 +3535,8 @@ _servername_callback(SSL *s, int *al, void *args) error: Py_DECREF(ssl_socket); + Py_DECREF(sni_cb); + Py_DECREF(ssl_ctx); *al = SSL_AD_INTERNAL_ERROR; ret = SSL_TLSEXT_ERR_ALERT_FATAL; #ifdef WITH_THREAD @@ -3555,7 +3579,6 @@ set_servername_callback(PySSLContext *self, PyObject *args) Py_INCREF(cb); self->set_hostname = cb; SSL_CTX_set_tlsext_servername_callback(self->ctx, _servername_callback); - SSL_CTX_set_tlsext_servername_arg(self->ctx, self); } Py_RETURN_NONE; #else diff --git a/Modules/bz2module.c b/Modules/bz2module.c index 81f168827b100a..312cd8f5721c96 100644 --- a/Modules/bz2module.c +++ b/Modules/bz2module.c @@ -102,6 +102,8 @@ typedef struct { int mode; Py_off_t pos; Py_off_t size; + int bzerror; /* Sticky decompression error, see + Util_CatchBZ2ReadError() */ #ifdef WITH_THREAD PyThread_type_lock lock; #endif @@ -120,6 +122,7 @@ typedef struct { PyObject_HEAD bz_stream bzs; int running; + int bzerror; PyObject *unused_data; #ifdef WITH_THREAD PyThread_type_lock lock; @@ -203,6 +206,29 @@ Util_CatchBZ2Error(int bzerror) return ret; } +/* BZ2_bzRead() calls BZ2_bzDecompress() on the same stream every time, and + re-entering BZ2_bzDecompress() after an error can write out of bounds + (gh-150599, CVE-2026-9669). A read error therefore makes the file + unusable for reading until a backwards seek() reopens the stream. */ +static int +Util_CatchBZ2ReadError(BZ2FileObject *f, int bzerror) +{ + if (bzerror != BZ_OK && bzerror != BZ_STREAM_END) + f->bzerror = bzerror; + return Util_CatchBZ2Error(bzerror); +} + +static int +Util_CheckBZ2ReadError(BZ2FileObject *f) +{ + if (f->bzerror) { + PyErr_SetString(PyExc_ValueError, + "BZ2File is unusable after a previous error"); + return -1; + } + return 0; +} + #if BUFSIZ < 8192 #define SMALLCHUNK 8192 #else @@ -298,7 +324,7 @@ Util_GetLine(BZ2FileObject *f, int n) f->mode = MODE_READ_EOF; break; } else if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); + Util_CatchBZ2ReadError(f, bzerror); Py_DECREF(v); return NULL; } @@ -437,7 +463,7 @@ Util_ReadAhead(BZ2FileObject *f, int bufsize) f->size = f->pos; f->mode = MODE_READ_EOF; } else if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); + Util_CatchBZ2ReadError(f, bzerror); Util_DropReadAhead(f); return -1; } @@ -535,6 +561,9 @@ BZ2File_read(BZ2FileObject *self, PyObject *args) if (check_iterbuffered(self)) goto cleanup; + if (Util_CheckBZ2ReadError(self)) + goto cleanup; + if (bytesrequested < 0) buffersize = Util_NewBufferSize((size_t)0); else @@ -564,7 +593,7 @@ BZ2File_read(BZ2FileObject *self, PyObject *args) self->mode = MODE_READ_EOF; break; } else if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); + Util_CatchBZ2ReadError(self, bzerror); Py_DECREF(ret); ret = NULL; goto cleanup; @@ -624,6 +653,9 @@ BZ2File_readline(BZ2FileObject *self, PyObject *args) if (check_iterbuffered(self)) goto cleanup; + if (Util_CheckBZ2ReadError(self)) + goto cleanup; + if (sizehint == 0) ret = PyString_FromString(""); else @@ -685,6 +717,9 @@ BZ2File_readlines(BZ2FileObject *self, PyObject *args) if (check_iterbuffered(self)) goto cleanup; + if (Util_CheckBZ2ReadError(self)) + goto cleanup; + if ((list = PyList_New(0)) == NULL) goto cleanup; @@ -704,7 +739,7 @@ BZ2File_readlines(BZ2FileObject *self, PyObject *args) } shortread = 1; } else if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); + Util_CatchBZ2ReadError(self, bzerror); error: Py_DECREF(list); list = NULL; @@ -995,6 +1030,41 @@ BZ2File_writelines(BZ2FileObject *self, PyObject *seq) #undef CHUNKSIZE } +/* Close the read stream and reopen it at the start of the file. */ +static int +Util_BZ2FileRewind(BZ2FileObject *self) +{ + PyObject *ret; + int bzerror; + + BZ2_bzReadClose(&bzerror, self->fp); + if (self->fp) { + PyFile_DecUseCount((PyFileObject *)self->file); + self->fp = NULL; + } + if (bzerror != BZ_OK) { + Util_CatchBZ2Error(bzerror); + return -1; + } + ret = PyObject_CallMethod(self->file, "seek", "(i)", 0); + if (!ret) + return -1; + Py_DECREF(ret); + self->pos = 0; + self->fp = BZ2_bzReadOpen(&bzerror, PyFile_AsFile(self->file), + 0, 0, NULL, 0); + if (self->fp) + PyFile_IncUseCount((PyFileObject *)self->file); + if (bzerror != BZ_OK) { + Util_CatchBZ2Error(bzerror); + return -1; + } + self->mode = MODE_READ; + /* A fresh stream: earlier read errors no longer apply. */ + self->bzerror = 0; + return 0; +} + PyDoc_STRVAR(BZ2File_seek__doc__, "seek(offset [, whence]) -> None\n\ \n\ @@ -1052,6 +1122,11 @@ BZ2File_seek(BZ2FileObject *self, PyObject *args) goto cleanup; } + /* After a read error the stream must not be decompressed again + (gh-150599): start over from a freshly opened one. */ + if (self->bzerror && Util_BZ2FileRewind(self) < 0) + goto cleanup; + if (where == 2) { if (self->size == -1) { assert(self->mode != MODE_READ_EOF); @@ -1068,7 +1143,7 @@ BZ2File_seek(BZ2FileObject *self, PyObject *args) if (bzerror == BZ_STREAM_END) { break; } else if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); + Util_CatchBZ2ReadError(self, bzerror); goto cleanup; } } @@ -1089,30 +1164,8 @@ BZ2File_seek(BZ2FileObject *self, PyObject *args) offset -= self->pos; } else { /* we cannot move back, so rewind the stream */ - BZ2_bzReadClose(&bzerror, self->fp); - if (self->fp) { - PyFile_DecUseCount((PyFileObject *)self->file); - self->fp = NULL; - } - if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); + if (Util_BZ2FileRewind(self) < 0) goto cleanup; - } - ret = PyObject_CallMethod(self->file, "seek", "(i)", 0); - if (!ret) - goto cleanup; - Py_DECREF(ret); - ret = NULL; - self->pos = 0; - self->fp = BZ2_bzReadOpen(&bzerror, PyFile_AsFile(self->file), - 0, 0, NULL, 0); - if (self->fp) - PyFile_IncUseCount((PyFileObject *)self->file); - if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - goto cleanup; - } - self->mode = MODE_READ; } if (offset <= 0 || self->mode == MODE_READ_EOF) @@ -1139,7 +1192,7 @@ BZ2File_seek(BZ2FileObject *self, PyObject *args) self->mode = MODE_READ_EOF; break; } else if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); + Util_CatchBZ2ReadError(self, bzerror); goto cleanup; } if (bytesread == offset) @@ -1391,6 +1444,7 @@ BZ2File_init(BZ2FileObject *self, PyObject *args, PyObject *kwargs) int mode_char = 0; self->size = -1; + self->bzerror = 0; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|sii:BZ2File", kwlist, &name, &mode, &buffering, @@ -1532,6 +1586,10 @@ BZ2File_iternext(BZ2FileObject *self) "I/O operation on closed file"); return NULL; } + if (self->mode == MODE_READ && Util_CheckBZ2ReadError(self)) { + RELEASE_LOCK(self); + return NULL; + } ret = Util_ReadAheadGetLineSkip(self, 0, READAHEAD_BUFSIZE); RELEASE_LOCK(self); if (ret == NULL || PyString_GET_SIZE(ret) == 0) { @@ -1942,6 +2000,13 @@ BZ2Decomp_decompress(BZ2DecompObject *self, PyObject *args) "already found"); goto error; } + if (self->bzerror) { + /* Re-entering BZ2_bzDecompress() after an error can write out of + bounds. */ + PyErr_SetString(PyExc_ValueError, + "Decompressor is unusable after a previous error"); + goto error; + } ret = PyString_FromStringAndSize(NULL, SMALLCHUNK); if (!ret) @@ -1975,6 +2040,7 @@ BZ2Decomp_decompress(BZ2DecompObject *self, PyObject *args) break; } if (bzerror != BZ_OK) { + self->bzerror = bzerror; Util_CatchBZ2Error(bzerror); goto error; } @@ -1989,6 +2055,8 @@ BZ2Decomp_decompress(BZ2DecompObject *self, PyObject *args) if (buffer_left == 0) { if (Util_GrowBuffer(&ret) < 0) { BZ2_bzDecompressEnd(bzs); + /* The stream state is freed: don't reuse it. */ + self->bzerror = BZ_MEM_ERROR; goto error; } bzs->next_out = BUF(ret) + output_size; @@ -2050,6 +2118,7 @@ BZ2Decomp_init(BZ2DecompObject *self, PyObject *args, PyObject *kwargs) } self->running = 1; + self->bzerror = 0; return 0; diff --git a/Modules/expat/COPYING b/Modules/expat/COPYING index ce9e5939291e45..c6d184a8aae845 100644 --- a/Modules/expat/COPYING +++ b/Modules/expat/COPYING @@ -1,5 +1,5 @@ Copyright (c) 1998-2000 Thai Open Source Software Center Ltd and Clark Cooper -Copyright (c) 2001-2022 Expat maintainers +Copyright (c) 2001-2025 Expat maintainers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/Modules/expat/ascii.h b/Modules/expat/ascii.h index 1f594d2e54b4d2..1d9cf70bf695da 100644 --- a/Modules/expat/ascii.h +++ b/Modules/expat/ascii.h @@ -31,6 +31,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #define ASCII_A 0x41 diff --git a/Modules/expat/asciitab.h b/Modules/expat/asciitab.h index af766fb24785ea..43af0876ccd3f8 100644 --- a/Modules/expat/asciitab.h +++ b/Modules/expat/asciitab.h @@ -30,6 +30,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ /* 0x00 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML, diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 523b37d8d5787d..1b2852dd76cb48 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -11,7 +11,7 @@ Copyright (c) 2000-2005 Fred L. Drake, Jr. Copyright (c) 2001-2002 Greg Stein Copyright (c) 2002-2016 Karl Waclawek - Copyright (c) 2016-2024 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2016 Cristian Rodríguez Copyright (c) 2016 Thomas Beutlich Copyright (c) 2017 Rhodri James @@ -19,6 +19,8 @@ Copyright (c) 2023 Hanno Böck Copyright (c) 2023 Sony Corporation / Snild Dolkow Copyright (c) 2024 Taichi Haradaguchi <20001722@ymail.ne.jp> + Copyright (c) 2025 Matthew Fernandez + Copyright (c) 2026 Braian Plaku Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -39,24 +41,27 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifndef Expat_INCLUDED -#define Expat_INCLUDED 1 +# define Expat_INCLUDED 1 -#include -#include "expat_external.h" +# include // for int64_t, uint8_t, uint64_t +# include +# include "expat_external.h" -#ifdef __cplusplus +# ifdef __cplusplus extern "C" { -#endif +# endif struct XML_ParserStruct; typedef struct XML_ParserStruct *XML_Parser; typedef unsigned char XML_Bool; -#define XML_TRUE ((XML_Bool)1) -#define XML_FALSE ((XML_Bool)0) +# define XML_TRUE ((XML_Bool)1) +# define XML_FALSE ((XML_Bool)0) /* The XML_Status enum gives the possible return values for several API functions. The preprocessor #defines are included so this @@ -73,11 +78,11 @@ typedef unsigned char XML_Bool; */ enum XML_Status { XML_STATUS_ERROR = 0, -#define XML_STATUS_ERROR XML_STATUS_ERROR +# define XML_STATUS_ERROR XML_STATUS_ERROR XML_STATUS_OK = 1, -#define XML_STATUS_OK XML_STATUS_OK +# define XML_STATUS_OK XML_STATUS_OK XML_STATUS_SUSPENDED = 2 -#define XML_STATUS_SUSPENDED XML_STATUS_SUSPENDED +# define XML_STATUS_SUSPENDED XML_STATUS_SUSPENDED }; enum XML_Error { @@ -135,6 +140,26 @@ enum XML_Error { XML_ERROR_NOT_STARTED, }; +/* Added in 2.9.0. */ +enum XML_Prop_Error { + XML_PROP_ERROR_NONE = 0, // i.e. success + + XML_PROP_ERROR_INVALID_KEY = 1, // i.e. the property is not known at all + XML_PROP_ERROR_INVALID_TYPE = 2, // i.e. the property is known but expects a + // different type (so a different + // getter/setter would need to be called) + XML_PROP_ERROR_INVALID_VALUE = 3, // i.e. the value is invalid (with a setter + // function) or a NULL-pointer (with a + // getter function) + + XML_PROP_ERROR_PARSER_NULL = 4, // i.e. the parser argument is NULL + XML_PROP_ERROR_PARSER_NOT_ROOT = 5, // i.e. the parser is not a root parser + // but a subparser (and a root parser is + // needed) + + /* potentially more error codes upcoming here */ +}; + enum XML_Content_Type { XML_CTYPE_EMPTY = 1, XML_CTYPE_ANY, @@ -276,7 +301,7 @@ XML_ParserCreate_MM(const XML_Char *encoding, /* Prepare a parser object to be reused. This is particularly valuable when memory allocation overhead is disproportionately high, - such as when a large number of small documnents need to be parsed. + such as when a large number of small documents need to be parsed. All handlers are cleared from the parser, except for the unknownEncodingHandler. The parser's external state is re-initialized except for the values of ns and ns_triplets. @@ -680,7 +705,7 @@ XMLPARSEAPI(void) XML_SetUserData(XML_Parser parser, void *userData); /* Returns the last value set by XML_SetUserData or NULL. */ -#define XML_GetUserData(parser) (*(void **)(parser)) +# define XML_GetUserData(parser) (*(void **)(parser)) /* This is equivalent to supplying an encoding argument to XML_ParserCreate. On success XML_SetEncoding returns non-zero, @@ -752,29 +777,6 @@ XML_GetSpecifiedAttributeCount(XML_Parser parser); XMLPARSEAPI(int) XML_GetIdAttributeIndex(XML_Parser parser); -#ifdef XML_ATTR_INFO -/* Source file byte offsets for the start and end of attribute names and values. - The value indices are exclusive of surrounding quotes; thus in a UTF-8 source - file an attribute value of "blah" will yield: - info->valueEnd - info->valueStart = 4 bytes. -*/ -typedef struct { - XML_Index nameStart; /* Offset to beginning of the attribute name. */ - XML_Index nameEnd; /* Offset after the attribute name's last byte. */ - XML_Index valueStart; /* Offset to beginning of the attribute value. */ - XML_Index valueEnd; /* Offset after the attribute value's last byte. */ -} XML_AttrInfo; - -/* Returns an array of XML_AttrInfo structures for the attribute/value pairs - passed in last call to the XML_StartElementHandler that were specified - in the start-tag rather than defaulted. Each attribute/value pair counts - as 1; thus the number of entries in the array is - XML_GetSpecifiedAttributeCount(parser) / 2. -*/ -XMLPARSEAPI(const XML_AttrInfo *) -XML_GetAttributeInfo(XML_Parser parser); -#endif - /* Parses some input. Returns XML_STATUS_ERROR if a fatal error is detected. The last call to XML_Parse must have isFinal true; len may be zero for this call (or any other). @@ -916,10 +918,23 @@ XML_SetParamEntityParsing(XML_Parser parser, function behavior. This must be called before parsing is started. Returns 1 if successful, 0 when called after parsing has started. Note: If parser == NULL, the function will do nothing and return 0. + DEPRECATED since Expat 2.8.0. + Please use XML_SetHashSalt16Bytes instead. */ +XML_ATTR_DEPRECATED("please use XML_SetHashSalt16Bytes instead") XMLPARSEAPI(int) XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt); +/* Sets the hash salt to use for internal hash calculations. + Helps in preventing DoS attacks based on predicting hash function behavior. + This must be called before parsing is started. + Returns XML_TRUE if successful, XML_FALSE when called after parsing has + started or when parser is NULL. + Added in Expat 2.8.0. +*/ +XMLPARSEAPI(XML_Bool) +XML_SetHashSalt16Bytes(XML_Parser parser, const uint8_t entropy[16]); + /* If XML_Parse or XML_ParseBuffer have returned XML_STATUS_ERROR, then XML_GetErrorCode returns information about the error. */ @@ -946,15 +961,24 @@ XML_GetErrorCode(XML_Parser parser); return 0 to indicate an error. Note: XML_GetCurrentByteIndex returns -1 to indicate an error. */ +XML_ATTR_DEPRECATED("please use XML_GetCurrentLineNumber64 instead") XMLPARSEAPI(XML_Size) XML_GetCurrentLineNumber(XML_Parser parser); +XMLPARSEAPI(uint64_t) XML_GetCurrentLineNumber64(XML_Parser parser); +XML_ATTR_DEPRECATED("please use XML_GetCurrentColumnNumber64 instead") XMLPARSEAPI(XML_Size) XML_GetCurrentColumnNumber(XML_Parser parser); +XMLPARSEAPI(uint64_t) XML_GetCurrentColumnNumber64(XML_Parser parser); +XML_ATTR_DEPRECATED("please use XML_GetCurrentByteIndex64 instead") XMLPARSEAPI(XML_Index) XML_GetCurrentByteIndex(XML_Parser parser); +XMLPARSEAPI(int64_t) XML_GetCurrentByteIndex64(XML_Parser parser); /* Return the number of bytes in the current event. Returns 0 if the event is in an internal entity. */ +XML_ATTR_DEPRECATED("please use XML_GetCurrentByteCount64 instead") XMLPARSEAPI(int) XML_GetCurrentByteCount(XML_Parser parser); +XMLPARSEAPI(uint64_t) +XML_GetCurrentByteCount64(XML_Parser parser); /* If XML_CONTEXT_BYTES is >=1, returns the input buffer, sets the integer pointed to by offset to the offset within this buffer @@ -966,13 +990,16 @@ XML_GetCurrentByteCount(XML_Parser parser); NOTE: The character pointer returned should not be used outside the handler that makes the call. */ +XML_ATTR_DEPRECATED("please use XML_GetInputContext64 instead") XMLPARSEAPI(const char *) XML_GetInputContext(XML_Parser parser, int *offset, int *size); +XMLPARSEAPI(const char *) +XML_GetInputContext64(XML_Parser parser, int64_t *offset, uint64_t *size); /* For backwards compatibility with previous versions. */ -#define XML_GetErrorLineNumber XML_GetCurrentLineNumber -#define XML_GetErrorColumnNumber XML_GetCurrentColumnNumber -#define XML_GetErrorByteIndex XML_GetCurrentByteIndex +# define XML_GetErrorLineNumber XML_GetCurrentLineNumber +# define XML_GetErrorColumnNumber XML_GetCurrentColumnNumber +# define XML_GetErrorByteIndex XML_GetCurrentByteIndex /* Frees the content model passed to the element declaration handler */ XMLPARSEAPI(void) @@ -1022,17 +1049,24 @@ enum XML_FeatureEnum { XML_FEATURE_UNICODE_WCHAR_T, XML_FEATURE_DTD, XML_FEATURE_CONTEXT_BYTES, + /* Added in Expat 1.95.5, removed in Expat 2.9.0. */ XML_FEATURE_MIN_SIZE, XML_FEATURE_SIZEOF_XML_CHAR, XML_FEATURE_SIZEOF_XML_LCHAR, + /* Added in Expat 2.0.0. */ XML_FEATURE_NS, + /* Added in Expat 2.0.1. */ XML_FEATURE_LARGE_SIZE, + /* Added in Expat 2.1.0, removed in Expat 2.9.0. */ XML_FEATURE_ATTR_INFO, /* Added in Expat 2.4.0. */ XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT, XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT, /* Added in Expat 2.6.0. */ - XML_FEATURE_GE + XML_FEATURE_GE, + /* Added in Expat 2.7.2. */ + XML_FEATURE_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT, + XML_FEATURE_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT, /* Additional features must be added to the end of this enum. */ }; @@ -1045,7 +1079,42 @@ typedef struct { XMLPARSEAPI(const XML_Feature *) XML_GetFeatureList(void); -#if defined(XML_DTD) || (defined(XML_GE) && XML_GE == 1) +/* Added in Expat 2.9.0. */ +enum XML_Parser_Property { +# if defined(XML_TESTING) + XML_PROP_INVALID = 0, +# endif +# if XML_GE == 1 + XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD = 1, // of type `uint64_t` + XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION = 2, // of type `double` + XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD = 3, // of type `uint64_t` + XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION = 4, // of type `double` +# endif + XML_PROP_REPARSE_DEFERRAL_ENABLED = 5, // of type `XML_Bool` + /* more properties upcoming here */ +}; + +/* Added in Expat 2.9.0. */ +XMLPARSEAPI(enum XML_Prop_Error) +XML_GetPropertyBool(XML_Parser parser, enum XML_Parser_Property property, + XML_Bool *value); +XMLPARSEAPI(enum XML_Prop_Error) +XML_GetPropertyDouble(XML_Parser parser, enum XML_Parser_Property property, + double *value); +XMLPARSEAPI(enum XML_Prop_Error) +XML_GetPropertyUInt64(XML_Parser parser, enum XML_Parser_Property property, + uint64_t *value); +XMLPARSEAPI(enum XML_Prop_Error) +XML_SetPropertyBool(XML_Parser parser, enum XML_Parser_Property property, + XML_Bool value); +XMLPARSEAPI(enum XML_Prop_Error) +XML_SetPropertyDouble(XML_Parser parser, enum XML_Parser_Property property, + double value); +XMLPARSEAPI(enum XML_Prop_Error) +XML_SetPropertyUInt64(XML_Parser parser, enum XML_Parser_Property property, + uint64_t value); + +# if defined(XML_DTD) || (defined(XML_GE) && XML_GE == 1) /* Added in Expat 2.4.0 for XML_DTD defined and * added in Expat 2.6.0 for XML_GE == 1. */ XMLPARSEAPI(XML_Bool) @@ -1057,7 +1126,17 @@ XML_SetBillionLaughsAttackProtectionMaximumAmplification( XMLPARSEAPI(XML_Bool) XML_SetBillionLaughsAttackProtectionActivationThreshold( XML_Parser parser, unsigned long long activationThresholdBytes); -#endif + +/* Added in Expat 2.7.2. */ +XMLPARSEAPI(XML_Bool) +XML_SetAllocTrackerMaximumAmplification(XML_Parser parser, + float maximumAmplificationFactor); + +/* Added in Expat 2.7.2. */ +XMLPARSEAPI(XML_Bool) +XML_SetAllocTrackerActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes); +# endif /* Added in Expat 2.6.0. */ XMLPARSEAPI(XML_Bool) @@ -1066,12 +1145,12 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); /* Expat follows the semantic versioning convention. See https://semver.org */ -#define XML_MAJOR_VERSION 2 -#define XML_MINOR_VERSION 6 -#define XML_MICRO_VERSION 4 +# define XML_MAJOR_VERSION 2 +# define XML_MINOR_VERSION 9 +# define XML_MICRO_VERSION 0 -#ifdef __cplusplus +# ifdef __cplusplus } -#endif +# endif #endif /* not Expat_INCLUDED */ diff --git a/Modules/expat/expat_config.h b/Modules/expat/expat_config.h index c3967f03cec52e..f8448f5bb43f57 100644 --- a/Modules/expat/expat_config.h +++ b/Modules/expat/expat_config.h @@ -3,7 +3,7 @@ * distribution. */ #ifndef EXPAT_CONFIG_H -#define EXPAT_CONFIG_H +#define EXPAT_CONFIG_H 1 #include #ifdef WORDS_BIGENDIAN @@ -12,9 +12,17 @@ #define BYTEORDER 1234 #endif +#define HAVE_MEMMOVE 1 + #define XML_NS 1 #define XML_DTD 1 #define XML_GE 1 #define XML_CONTEXT_BYTES 1024 +#undef HAVE_ARC4RANDOM +#undef HAVE_ARC4RANDOM_BUF +#undef HAVE_GETENTROPY +#undef HAVE_GETRANDOM +#undef HAVE_SYSCALL_GETRANDOM + #endif /* EXPAT_CONFIG_H */ diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h index 8829f77091047a..0620fe47ed3cd2 100644 --- a/Modules/expat/expat_external.h +++ b/Modules/expat/expat_external.h @@ -12,9 +12,11 @@ Copyright (c) 2001-2002 Greg Stein Copyright (c) 2002-2006 Karl Waclawek Copyright (c) 2016 Cristian Rodríguez - Copyright (c) 2016-2019 Sebastian Pipping + Copyright (c) 2016-2025 Sebastian Pipping Copyright (c) 2017 Rhodri James Copyright (c) 2018 Yury Gribov + Copyright (c) 2026 Matthew Fernandez + Copyright (c) 2026 Braian Plaku Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -35,17 +37,19 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifndef Expat_External_INCLUDED -#define Expat_External_INCLUDED 1 +# define Expat_External_INCLUDED 1 /* External API definitions */ /* Expat tries very hard to make the API boundary very specifically defined. There are two macros defined to control this boundary; each of these can be defined before including this header to - achieve some different behavior, but doing so it not recommended or + achieve some different behavior, but doing so is not recommended or tested frequently. XMLCALL - The calling convention to use for all calls across the @@ -64,12 +68,12 @@ compiled with the cdecl calling convention as the default since system headers may assume the cdecl convention. */ -#ifndef XMLCALL -# if defined(_MSC_VER) -# define XMLCALL __cdecl -# elif defined(__GNUC__) && defined(__i386) && ! defined(__INTEL_COMPILER) -# define XMLCALL __attribute__((cdecl)) -# else +# ifndef XMLCALL +# if defined(_MSC_VER) +# define XMLCALL __cdecl +# elif defined(__GNUC__) && defined(__i386) && ! defined(__INTEL_COMPILER) +# define XMLCALL __attribute__((cdecl)) +# else /* For any platform which uses this definition and supports more than one calling convention, we need to extend this definition to declare the convention used on that platform, if it's possible to @@ -80,86 +84,100 @@ pre-processor and how to specify the same calling convention as the platform's malloc() implementation. */ -# define XMLCALL -# endif -#endif /* not defined XMLCALL */ +# define XMLCALL +# endif +# endif /* not defined XMLCALL */ -#if ! defined(XML_STATIC) && ! defined(XMLIMPORT) -# ifndef XML_BUILDING_EXPAT +# if ! defined(XML_STATIC) && ! defined(XMLIMPORT) +# ifndef XML_BUILDING_EXPAT /* using Expat from an application */ -# if defined(_MSC_EXTENSIONS) && ! defined(__BEOS__) && ! defined(__CYGWIN__) -# define XMLIMPORT __declspec(dllimport) +# if defined(_MSC_VER) && ! defined(__BEOS__) && ! defined(__CYGWIN__) +# define XMLIMPORT __declspec(dllimport) +# endif + # endif +# endif /* not defined XML_STATIC */ +# ifndef XML_ENABLE_VISIBILITY +# define XML_ENABLE_VISIBILITY 0 # endif -#endif /* not defined XML_STATIC */ - -#ifndef XML_ENABLE_VISIBILITY -# define XML_ENABLE_VISIBILITY 0 -#endif -#if ! defined(XMLIMPORT) && XML_ENABLE_VISIBILITY -# define XMLIMPORT __attribute__((visibility("default"))) -#endif +# if ! defined(XMLIMPORT) && XML_ENABLE_VISIBILITY +# define XMLIMPORT __attribute__((visibility("default"))) +# endif /* If we didn't define it above, define it away: */ -#ifndef XMLIMPORT -# define XMLIMPORT -#endif - -#if defined(__GNUC__) \ - && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96)) -# define XML_ATTR_MALLOC __attribute__((__malloc__)) -#else -# define XML_ATTR_MALLOC -#endif - -#if defined(__GNUC__) \ - && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) -# define XML_ATTR_ALLOC_SIZE(x) __attribute__((__alloc_size__(x))) -#else -# define XML_ATTR_ALLOC_SIZE(x) -#endif - -#define XMLPARSEAPI(type) XMLIMPORT type XMLCALL - -#ifdef __cplusplus -extern "C" { -#endif +# ifndef XMLIMPORT +# define XMLIMPORT +# endif -#ifdef XML_UNICODE_WCHAR_T -# ifndef XML_UNICODE -# define XML_UNICODE +# if defined(__GNUC__) \ + && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96)) +# define XML_ATTR_MALLOC __attribute__((__malloc__)) +# else +# define XML_ATTR_MALLOC # endif -# if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2) -# error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc" + +# if defined(__GNUC__) \ + && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) +# define XML_ATTR_ALLOC_SIZE(x) __attribute__((__alloc_size__(x))) +# else +# define XML_ATTR_ALLOC_SIZE(x) +# endif + +/* Marks a function that Expat still provides but that callers should move off + of. */ +# if defined(__clang__) || defined(__GNUC__) +# define XML_ATTR_DEPRECATED(message) \ + __attribute__((__deprecated__(message))) +# elif defined(_MSC_VER) +# define XML_ATTR_DEPRECATED(message) __declspec(deprecated(message)) +# else +# define XML_ATTR_DEPRECATED(message) // empty i.e. no deprecation +# endif + +# define XMLPARSEAPI(type) XMLIMPORT type XMLCALL + +# ifdef __cplusplus +extern "C" { # endif -#endif -#ifdef XML_UNICODE /* Information is UTF-16 encoded. */ # ifdef XML_UNICODE_WCHAR_T +# ifndef XML_UNICODE +# define XML_UNICODE +# endif +# if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2) +# error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc" +# endif +# endif + +# ifdef XML_UNICODE /* Information is UTF-16 encoded. */ +# ifdef XML_UNICODE_WCHAR_T typedef wchar_t XML_Char; typedef wchar_t XML_LChar; -# else +# else typedef unsigned short XML_Char; typedef char XML_LChar; -# endif /* XML_UNICODE_WCHAR_T */ -#else /* Information is UTF-8 encoded. */ +# endif /* XML_UNICODE_WCHAR_T */ +# else /* Information is UTF-8 encoded. */ typedef char XML_Char; typedef char XML_LChar; -#endif /* XML_UNICODE */ +# endif /* XML_UNICODE */ -#ifdef XML_LARGE_SIZE /* Use large integers for file/stream positions. */ +# ifdef XML_LARGE_SIZE /* Use large integers for file/stream positions. */ +# if defined(__clang__) || defined(__GNUC__) +# warning Macro XML_LARGE_SIZE is deprecated, please use the 64bit location API functions instead. +# endif typedef long long XML_Index; typedef unsigned long long XML_Size; -#else +# else typedef long XML_Index; typedef unsigned long XML_Size; -#endif /* XML_LARGE_SIZE */ +# endif /* XML_LARGE_SIZE */ -#ifdef __cplusplus +# ifdef __cplusplus } -#endif +# endif #endif /* not Expat_External_INCLUDED */ diff --git a/Modules/expat/fallthrough.h b/Modules/expat/fallthrough.h new file mode 100644 index 00000000000000..0152d1bdc5de8f --- /dev/null +++ b/Modules/expat/fallthrough.h @@ -0,0 +1,51 @@ +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2026 Nick Begg + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT +*/ + +#ifndef FALLTHROUGH_H +# define FALLTHROUGH_H 1 + +// Explicit fallthrough in switch case to avoid warnings +// with compiler flag -Wimplicit-fallthrough. + +# define EXPAT_FALLTHROUGH \ + do { \ + } while (0) + +# if defined(__has_attribute) +# if __has_attribute(fallthrough) +# undef EXPAT_FALLTHROUGH +# define EXPAT_FALLTHROUGH __attribute__((fallthrough)) +# endif +# endif + +#endif // FALLTHROUGH_H diff --git a/Modules/expat/hash_table.h b/Modules/expat/hash_table.h new file mode 100644 index 00000000000000..f290aad18693fe --- /dev/null +++ b/Modules/expat/hash_table.h @@ -0,0 +1,78 @@ +/* Hash table related internal API + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2026 Sebastian Pipping + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT +*/ + +#if ! defined(HASH_TABLE_H) +# define HASH_TABLE_H 1 + +# include "expat.h" // for XML_Bool, XML_Parser +# include "internal.h" // for XML_NONTESTING_STATIC + +# include // for size_t + +typedef const XML_Char *KEY; + +typedef struct { + KEY name; +} NAMED; + +typedef struct { + NAMED **v; + unsigned char power; + size_t size; + size_t used; + XML_Parser parser; +} HASH_TABLE; + +typedef struct { + NAMED **p; + NAMED **end; +} HASH_TABLE_ITER; + +XML_NONTESTING_STATIC NAMED *lookupWithLength(XML_Parser parser, + HASH_TABLE *table, KEY name, + size_t nameLen, + size_t createSize); +XML_NONTESTING_STATIC NAMED *lookup(XML_Parser parser, HASH_TABLE *table, + KEY name, size_t createSize); + +XML_NONTESTING_STATIC void hashTableInit(HASH_TABLE *table, XML_Parser parser); +XML_NONTESTING_STATIC void hashTableClear(HASH_TABLE *table); +XML_NONTESTING_STATIC void hashTableDestroy(HASH_TABLE *table); +XML_NONTESTING_STATIC void hashTableIterInit(HASH_TABLE_ITER *iter, + const HASH_TABLE *table); +XML_NONTESTING_STATIC NAMED *hashTableIterNext(HASH_TABLE_ITER *iter); + +XML_NONTESTING_STATIC XML_Bool keyeq(KEY s1, size_t s1len, KEY s2); +XML_NONTESTING_STATIC size_t keylen(KEY s); + +#endif // ! defined(HASH_TABLE_H) diff --git a/Modules/expat/iasciitab.h b/Modules/expat/iasciitab.h index 5d8646f2a318b8..1de8d519c8b7ba 100644 --- a/Modules/expat/iasciitab.h +++ b/Modules/expat/iasciitab.h @@ -30,6 +30,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ /* Like asciitab.h, except that 0xD has code BT_S rather than BT_CR */ diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h index 167ec36804a43b..de34cd7dfc55c5 100644 --- a/Modules/expat/internal.h +++ b/Modules/expat/internal.h @@ -6,13 +6,6 @@ The following calling convention macros are defined for frequently called functions: - FASTCALL - Used for those internal functions that have a simple - body and a low number of arguments and local variables. - - PTRCALL - Used for functions called though function pointers. - - PTRFASTCALL - Like PTRCALL, but for low number of arguments. - inline - Used for selected internal functions for which inlining may improve performance on some platforms. @@ -28,11 +21,14 @@ Copyright (c) 2002-2003 Fred L. Drake, Jr. Copyright (c) 2002-2006 Karl Waclawek Copyright (c) 2003 Greg Stein - Copyright (c) 2016-2024 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2018 Yury Gribov Copyright (c) 2019 David Loffredo Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow Copyright (c) 2024 Taichi Haradaguchi <20001722@ymail.ne.jp> + Copyright (c) 2026 Matthew Wozniczka + Copyright (c) 2026 Braian Plaku + Copyright (c) 2026 Florian Schmaus Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -53,124 +49,106 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ - -#if defined(__GNUC__) && defined(__i386__) && ! defined(__MINGW32__) -/* We'll use this version by default only where we know it helps. - - regparm() generates warnings on Solaris boxes. See SF bug #692878. - Instability reported with egcs on a RedHat Linux 7.3. - Let's comment out: - #define FASTCALL __attribute__((stdcall, regparm(3))) - and let's try this: + SPDX-License-Identifier: MIT */ -# define FASTCALL __attribute__((regparm(3))) -# define PTRFASTCALL __attribute__((regparm(3))) -#endif - -/* Using __fastcall seems to have an unexpected negative effect under - MS VC++, especially for function pointers, so we won't use it for - now on that platform. It may be reconsidered for a future release - if it can be made more effective. - Likely reason: __fastcall on Windows is like stdcall, therefore - the compiler cannot perform stack optimizations for call clusters. -*/ - -/* Make sure all of these are defined if they aren't already. */ - -#ifndef FASTCALL -# define FASTCALL -#endif -#ifndef PTRCALL -# define PTRCALL -#endif +#if ! defined(INTERNAL_H) +# define INTERNAL_H 1 -#ifndef PTRFASTCALL -# define PTRFASTCALL -#endif - -#ifndef XML_MIN_SIZE -# if ! defined(__cplusplus) && ! defined(inline) +# if ! defined(inline) # ifdef __GNUC__ # define inline __inline # endif /* __GNUC__ */ # endif -#endif /* XML_MIN_SIZE */ -#ifdef __cplusplus -# define inline inline -#else # ifndef inline # define inline # endif -#endif - -#include // ULONG_MAX -#if defined(_WIN32) \ - && (! defined(__USE_MINGW_ANSI_STDIO) \ - || (1 - __USE_MINGW_ANSI_STDIO - 1 == 0)) -# define EXPAT_FMT_ULL(midpart) "%" midpart "I64u" -# if defined(_WIN64) // Note: modifiers "td" and "zu" do not work for MinGW -# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "I64d" -# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "I64u" -# else -# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "d" -# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u" +# if ! defined(XML_NONTESTING_STATIC) +# if defined(XML_TESTING) +# define XML_NONTESTING_STATIC // empty i.e. not static +# else +# define XML_NONTESTING_STATIC static +# endif # endif -#else -# define EXPAT_FMT_ULL(midpart) "%" midpart "llu" -# if ! defined(ULONG_MAX) -# error Compiler did not define ULONG_MAX for us -# elif ULONG_MAX == 18446744073709551615u // 2^64-1 -# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "ld" -# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "lu" + +# include // ULONG_MAX +# include // size_t + +# if defined(_WIN32) \ + && (! defined(__USE_MINGW_ANSI_STDIO) \ + || (1 - __USE_MINGW_ANSI_STDIO - 1 == 0)) +# define EXPAT_FMT_LLX(midpart) "%" midpart "I64x" +# define EXPAT_FMT_ULL(midpart) "%" midpart "I64u" +# if defined(_WIN64) // Note: modifiers "td" and "zu" do not work for MinGW +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "I64d" +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "I64u" +# else +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "d" +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u" +# endif # else -# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "d" -# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u" +# include // PRIdPTR, PRIuPTR +# define EXPAT_FMT_LLX(midpart) "%" midpart "llx" +# define EXPAT_FMT_ULL(midpart) "%" midpart "llu" +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart PRIdPTR +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart PRIuPTR # endif -#endif -#ifndef UNUSED_P -# define UNUSED_P(p) (void)p -#endif +# ifndef UNUSED_P +# define UNUSED_P(p) (void)p +# endif /* NOTE BEGIN If you ever patch these defaults to greater values for non-attack XML payload in your environment, please file a bug report with libexpat. Thank you! */ -#define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT \ - 100.0f -#define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT \ - 8388608 // 8 MiB, 2^23 -/* NOTE END */ +# define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT \ + 100.0f +# define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT \ + 8388608 // 8 MiB, 2^23 + +# define EXPAT_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT 100.0f +# define EXPAT_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT \ + 67108864 // 64 MiB, 2^26 + +// NOTE: If function expat_alloc was user facing, EXPAT_MALLOC_ALIGNMENT would +// have to take sizeof(long double) into account +union expat_align { + long long l; + void *p; +}; +# define EXPAT_MALLOC_ALIGNMENT sizeof(union expat_align) +# define EXPAT_MALLOC_PADDING ((EXPAT_MALLOC_ALIGNMENT) - sizeof(size_t)) -#include "expat.h" // so we can use type XML_Parser below +/* NOTE END */ -#ifdef __cplusplus -extern "C" { -#endif +# include "expat.h" // so we can use type XML_Parser below void _INTERNAL_trim_to_complete_utf8_characters(const char *from, const char **fromLimRef); -#if defined(XML_GE) && XML_GE == 1 +# if defined(XML_GE) && XML_GE == 1 unsigned long long testingAccountingGetCountBytesDirect(XML_Parser parser); unsigned long long testingAccountingGetCountBytesIndirect(XML_Parser parser); const char *unsignedCharToPrintable(unsigned char c); -#endif +# endif extern -#if ! defined(XML_TESTING) +# if ! defined(XML_TESTING) const -#endif +# endif XML_Bool g_reparseDeferralEnabledDefault; // written ONLY in runtests.c -#if defined(XML_TESTING) +# if defined(XML_TESTING) + +int xmlSetHashSalt(XML_Parser parser, unsigned long hash_salt); + +void *expat_malloc(XML_Parser parser, size_t size, int sourceLine); +void expat_free(XML_Parser parser, void *ptr, int sourceLine); +void *expat_realloc(XML_Parser parser, void *ptr, size_t size, int sourceLine); extern unsigned int g_bytesScanned; // used for testing only -#endif +# endif -#ifdef __cplusplus -} -#endif +#endif // not defined INTERNAL_H diff --git a/Modules/expat/latin1tab.h b/Modules/expat/latin1tab.h index b681d278af6569..3793f4f3cc7841 100644 --- a/Modules/expat/latin1tab.h +++ b/Modules/expat/latin1tab.h @@ -30,6 +30,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ /* 0x80 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER, diff --git a/Modules/expat/memory_sanitizer.h b/Modules/expat/memory_sanitizer.h new file mode 100644 index 00000000000000..f739b88f304871 --- /dev/null +++ b/Modules/expat/memory_sanitizer.h @@ -0,0 +1,53 @@ +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2026 Matthew Fernandez + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT +*/ + +#if ! defined(MEMORY_SANITIZER_H) +# define MEMORY_SANITIZER_H 1 + +# if defined(__has_feature) +# if __has_feature(memory_sanitizer) +# include + +// inform Memory Sanitizer that [base, base + extent) is now initialized +# define MSAN_UNPOISON(base, extent) __msan_unpoison((base), (extent)) + +# endif +# endif + +# if ! defined(MSAN_UNPOISON) +# define MSAN_UNPOISON(base, extent) \ + do { \ + } while (0) +# endif + +#endif // ! defined(MEMORY_SANITIZER_H) diff --git a/Modules/expat/nametab.h b/Modules/expat/nametab.h index 63485446b96727..2385851562e7dc 100644 --- a/Modules/expat/nametab.h +++ b/Modules/expat/nametab.h @@ -28,6 +28,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ static const unsigned namingBitmap[] = { diff --git a/Modules/expat/pyexpatns.h b/Modules/expat/pyexpatns.h index 999c5c730abd7a..4c550b7d013181 100644 --- a/Modules/expat/pyexpatns.h +++ b/Modules/expat/pyexpatns.h @@ -26,7 +26,7 @@ * http://lxr.mozilla.org/seamonkey/source/modules/libimg/png/mozpngconf.h#115 * * The list of relevant exported symbols can be had with this command: - * + * nm pyexpat.so \ | grep -v " [a-zBUA] " \ | grep -v "_fini\|_init\|initpyexpat" @@ -38,6 +38,9 @@ #ifndef PYEXPATNS_H #define PYEXPATNS_H +#define testingAccountingGetCountBytesDirect PyExpat_testingAccountingGetCountBytesDirect +#define testingAccountingGetCountBytesIndirect PyExpat_testingAccountingGetCountBytesIndirect +#define unsignedCharToPrintable PyExpat_unsignedCharToPrintable #define XML_DefaultCurrent PyExpat_XML_DefaultCurrent #define XML_ErrorString PyExpat_XML_ErrorString #define XML_ExpatVersion PyExpat_XML_ExpatVersion @@ -47,14 +50,22 @@ #define XML_GetBase PyExpat_XML_GetBase #define XML_GetBuffer PyExpat_XML_GetBuffer #define XML_GetCurrentByteCount PyExpat_XML_GetCurrentByteCount +#define XML_GetCurrentByteCount64 PyExpat_XML_GetCurrentByteCount64 #define XML_GetCurrentByteIndex PyExpat_XML_GetCurrentByteIndex +#define XML_GetCurrentByteIndex64 PyExpat_XML_GetCurrentByteIndex64 #define XML_GetCurrentColumnNumber PyExpat_XML_GetCurrentColumnNumber +#define XML_GetCurrentColumnNumber64 PyExpat_XML_GetCurrentColumnNumber64 #define XML_GetCurrentLineNumber PyExpat_XML_GetCurrentLineNumber +#define XML_GetCurrentLineNumber64 PyExpat_XML_GetCurrentLineNumber64 #define XML_GetErrorCode PyExpat_XML_GetErrorCode #define XML_GetFeatureList PyExpat_XML_GetFeatureList #define XML_GetIdAttributeIndex PyExpat_XML_GetIdAttributeIndex #define XML_GetInputContext PyExpat_XML_GetInputContext +#define XML_GetInputContext64 PyExpat_XML_GetInputContext64 #define XML_GetParsingStatus PyExpat_XML_GetParsingStatus +#define XML_GetPropertyBool PyExpat_XML_GetPropertyBool +#define XML_GetPropertyDouble PyExpat_XML_GetPropertyDouble +#define XML_GetPropertyUInt64 PyExpat_XML_GetPropertyUInt64 #define XML_GetSpecifiedAttributeCount PyExpat_XML_GetSpecifiedAttributeCount #define XmlGetUtf16InternalEncoding PyExpat_XmlGetUtf16InternalEncoding #define XmlGetUtf16InternalEncodingNS PyExpat_XmlGetUtf16InternalEncodingNS @@ -79,8 +90,12 @@ #define XmlPrologStateInit PyExpat_XmlPrologStateInit #define XmlPrologStateInitExternalEntity PyExpat_XmlPrologStateInitExternalEntity #define XML_ResumeParser PyExpat_XML_ResumeParser +#define XML_SetAllocTrackerActivationThreshold PyExpat_XML_SetAllocTrackerActivationThreshold +#define XML_SetAllocTrackerMaximumAmplification PyExpat_XML_SetAllocTrackerMaximumAmplification #define XML_SetAttlistDeclHandler PyExpat_XML_SetAttlistDeclHandler #define XML_SetBase PyExpat_XML_SetBase +#define XML_SetBillionLaughsAttackProtectionActivationThreshold PyExpat_XML_SetBillionLaughsAttackProtectionActivationThreshold +#define XML_SetBillionLaughsAttackProtectionMaximumAmplification PyExpat_XML_SetBillionLaughsAttackProtectionMaximumAmplification #define XML_SetCdataSectionHandler PyExpat_XML_SetCdataSectionHandler #define XML_SetCharacterDataHandler PyExpat_XML_SetCharacterDataHandler #define XML_SetCommentHandler PyExpat_XML_SetCommentHandler @@ -103,6 +118,10 @@ #define XML_SetNotStandaloneHandler PyExpat_XML_SetNotStandaloneHandler #define XML_SetParamEntityParsing PyExpat_XML_SetParamEntityParsing #define XML_SetProcessingInstructionHandler PyExpat_XML_SetProcessingInstructionHandler +#define XML_SetPropertyBool PyExpat_XML_SetPropertyBool +#define XML_SetPropertyDouble PyExpat_XML_SetPropertyDouble +#define XML_SetPropertyUInt64 PyExpat_XML_SetPropertyUInt64 +#define XML_SetReparseDeferralEnabled PyExpat_XML_SetReparseDeferralEnabled #define XML_SetReturnNSTriplet PyExpat_XML_SetReturnNSTriplet #define XML_SetSkippedEntityHandler PyExpat_XML_SetSkippedEntityHandler #define XML_SetStartCdataSectionHandler PyExpat_XML_SetStartCdataSectionHandler diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h index 04f6f74585b5a2..ac1fbfaf0bde81 100644 --- a/Modules/expat/siphash.h +++ b/Modules/expat/siphash.h @@ -8,6 +8,8 @@ * * 1. https://www.131002.net/siphash/siphash24.c * 2. https://www.131002.net/siphash/ + * + * SPDX-License-Identifier: CC0-1.0 * -------------------------------------------------------------------------- * HISTORY: * @@ -100,6 +102,7 @@ #include /* size_t */ #include /* uint64_t uint32_t uint8_t */ +#include "fallthrough.h" /* * Workaround to not require a C++11 compiler for using ULL suffix @@ -234,25 +237,25 @@ sip24_final(struct siphash *H) { switch (left) { case 7: b |= (uint64_t)H->buf[6] << 48; - /* fall through */ + EXPAT_FALLTHROUGH; case 6: b |= (uint64_t)H->buf[5] << 40; - /* fall through */ + EXPAT_FALLTHROUGH; case 5: b |= (uint64_t)H->buf[4] << 32; - /* fall through */ + EXPAT_FALLTHROUGH; case 4: b |= (uint64_t)H->buf[3] << 24; - /* fall through */ + EXPAT_FALLTHROUGH; case 3: b |= (uint64_t)H->buf[2] << 16; - /* fall through */ + EXPAT_FALLTHROUGH; case 2: b |= (uint64_t)H->buf[1] << 8; - /* fall through */ + EXPAT_FALLTHROUGH; case 1: b |= (uint64_t)H->buf[0] << 0; - /* fall through */ + EXPAT_FALLTHROUGH; case 0: break; } diff --git a/Modules/expat/utf8tab.h b/Modules/expat/utf8tab.h index 88efcf91cc16a6..73732d1527f142 100644 --- a/Modules/expat/utf8tab.h +++ b/Modules/expat/utf8tab.h @@ -30,6 +30,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ /* 0x80 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL, diff --git a/Modules/expat/winconfig.h b/Modules/expat/winconfig.h index 05805514ec7fa2..9e7e8bdc22ca36 100644 --- a/Modules/expat/winconfig.h +++ b/Modules/expat/winconfig.h @@ -31,6 +31,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifndef WINCONFIG_H diff --git a/Modules/expat/xcs.c b/Modules/expat/xcs.c new file mode 100644 index 00000000000000..ae5e042d778fff --- /dev/null +++ b/Modules/expat/xcs.c @@ -0,0 +1,90 @@ +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2022-2026 Sebastian Pipping + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT +*/ + +#include "xcs.h" + +#if defined(XML_UNICODE) +# if defined(XML_UNICODE_WCHAR_T) +# include // for wcscmp, wcslen, wcsncmp +# endif +#else +# include // for strcmp, strlen, strncmp +#endif + +size_t +xcslen(const XML_Char *s) { +#ifdef XML_UNICODE +# ifdef XML_UNICODE_WCHAR_T + return wcslen(s); +# else + // XML_Char is unsigned short + size_t len = 0; + while (s[len]) { + len++; + } + return len; +# endif +#else + return strlen(s); +#endif +} + +int +xcscmp(const XML_Char *a, const XML_Char *b) { +#if defined(XML_UNICODE) +# if defined(XML_UNICODE_WCHAR_T) + return wcscmp(a, b); +# else + for (; a[0] && b[0] && a[0] == b[0]; a++, b++) + ; + return a[0] - b[0]; +# endif +#else + return strcmp(a, b); +#endif +} + +int +xcsncmp(const XML_Char *a, const XML_Char *b, size_t len) { +#if defined(XML_UNICODE) +# if defined(XML_UNICODE_WCHAR_T) + return wcsncmp(a, b, len); +# else + for (; len > 0 && a[0] && b[0] && a[0] == b[0]; len--, a++, b++) { + } + return (len == 0) ? 0 : (a[0] - b[0]); +# endif +#else + return strncmp(a, b, len); +#endif +} diff --git a/Modules/expat/xcs.h b/Modules/expat/xcs.h new file mode 100644 index 00000000000000..1b942647d33309 --- /dev/null +++ b/Modules/expat/xcs.h @@ -0,0 +1,44 @@ +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2026 Sebastian Pipping + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT +*/ + +#if ! defined(EXPAT_XCS_H) + +# include // size_t +# include "expat_external.h" // for XML_Char + +extern size_t xcslen(const XML_Char *s); + +extern int xcscmp(const XML_Char *a, const XML_Char *b); +extern int xcsncmp(const XML_Char *a, const XML_Char *b, size_t len); + +#endif // ! defined(EXPAT_XCS_H) diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index 836db6c0b97f14..f1dd54cb605900 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* c5625880f4bf417c1463deee4eb92d86ff413f802048621c57e25fe483eb59e4 (2.6.4+) +/* e3ca845466942140c9af531ac57808c4650116df5d6dbeda3b6546d2548a586e (2.9.0+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -13,7 +13,7 @@ Copyright (c) 2002-2016 Karl Waclawek Copyright (c) 2005-2009 Steven Solie Copyright (c) 2016 Eric Rahm - Copyright (c) 2016-2024 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2016 Gaurav Copyright (c) 2016 Thomas Beutlich Copyright (c) 2016 Gustavo Grieco @@ -39,8 +39,25 @@ Copyright (c) 2022 Sean McBride Copyright (c) 2023 Owain Davies Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow - Copyright (c) 2024 Berkay Eren Ürün + Copyright (c) 2024-2025 Berkay Eren Ürün Copyright (c) 2024 Hanno Böck + Copyright (c) 2025-2026 Matthew Fernandez + Copyright (c) 2025 Atrem Borovik + Copyright (c) 2025 Alfonso Gregory + Copyright (c) 2026 Rosen Penev + Copyright (c) 2026 Francesco Bertolaccini + Copyright (c) 2026 Christian Ng + Copyright (c) 2026 Nick Begg + Copyright (c) 2026 Kartik Kenchi + Copyright (c) 2026 Haris Hussain + Copyright (c) 2026 Evgeny Kotkov + Copyright (c) 2026 Darren Carreras + Copyright (c) 2026 Alberto Maschietto + Copyright (c) 2026 Zeyou Liu + Copyright (c) 2026 Stan Ulbrych + Copyright (c) 2026 Braian Plaku + Copyright (c) 2026 Filippo Tedeschi + Copyright (c) 2026 Junki Lee Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -61,6 +78,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #define XML_BUILDING_EXPAT 1 @@ -80,28 +99,17 @@ # error XML_CONTEXT_BYTES must be defined, non-empty and >=0 (0 to disable, >=1 to enable; 1024 is a common default) #endif -#if defined(HAVE_SYSCALL_GETRANDOM) -# if ! defined(_GNU_SOURCE) -# define _GNU_SOURCE 1 /* syscall prototype */ -# endif -#endif - -#ifdef _WIN32 -/* force stdlib to define rand_s() */ -# if ! defined(_CRT_RAND_S) -# define _CRT_RAND_S -# endif -#endif - #include #include #include /* memset(), memcpy() */ #include -#include /* UINT_MAX */ +#include /* INT_MAX, UINT_MAX */ #include /* fprintf */ -#include /* getenv, rand_s */ -#include /* uintptr_t */ +#include /* getenv */ +#include /* SIZE_MAX, UINT64_MAX, uint64_t, uintptr_t */ #include /* isnan */ +#include +#include /* wcsncmp */ #ifdef _WIN32 # define getpid GetCurrentProcessId @@ -120,32 +128,36 @@ #include "ascii.h" #include "expat.h" #include "siphash.h" +#include "xcs.h" + +#if defined(HAVE_ARC4RANDOM) +# include "random_arc4random.h" +#endif /* defined(HAVE_ARC4RANDOM) */ + +#if defined(HAVE_ARC4RANDOM_BUF) +# include "random_arc4random_buf.h" +#endif // defined(HAVE_ARC4RANDOM_BUF) + +#if defined(XML_DEV_URANDOM) +# include "random_dev_urandom.h" +#endif /* defined(XML_DEV_URANDOM) */ + +#if defined(HAVE_GETENTROPY) +# include "random_getentropy.h" +#endif // defined(HAVE_GETENTROPY) #if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) -# if defined(HAVE_GETRANDOM) -# include /* getrandom */ -# else -# include /* syscall */ -# include /* SYS_getrandom */ -# endif -# if ! defined(GRND_NONBLOCK) -# define GRND_NONBLOCK 0x0001 -# endif /* defined(GRND_NONBLOCK) */ -#endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ - -#if defined(HAVE_LIBBSD) \ - && (defined(HAVE_ARC4RANDOM_BUF) || defined(HAVE_ARC4RANDOM)) -# include -#endif +# include "random_getrandom.h" +#endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ -#if defined(_WIN32) && ! defined(LOAD_LIBRARY_SEARCH_SYSTEM32) -# define LOAD_LIBRARY_SEARCH_SYSTEM32 0x00000800 -#endif +#if defined(_WIN32) && ! defined(XML_POOR_ENTROPY) +# include "random_rand_s.h" +#endif /* defined(_WIN32) && ! defined(XML_POOR_ENTROPY) */ #if ! defined(HAVE_GETRANDOM) && ! defined(HAVE_SYSCALL_GETRANDOM) \ && ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) \ - && ! defined(XML_DEV_URANDOM) && ! defined(_WIN32) \ - && ! defined(XML_POOR_ENTROPY) + && ! defined(HAVE_GETENTROPY) && ! defined(XML_DEV_URANDOM) \ + && ! defined(_WIN32) && ! defined(XML_POOR_ENTROPY) # error You do not have support for any sources of high quality entropy \ enabled. For end user security, that is probably not what you want. \ \ @@ -154,13 +166,12 @@ * Linux >=3.17 + glibc (including <2.25) (syscall SYS_getrandom): HAVE_SYSCALL_GETRANDOM, \ * BSD / macOS >=10.7 / glibc >=2.36 (arc4random_buf): HAVE_ARC4RANDOM_BUF, \ * BSD / macOS (including <10.7) / glibc >=2.36 (arc4random): HAVE_ARC4RANDOM, \ - * libbsd (arc4random_buf): HAVE_ARC4RANDOM_BUF + HAVE_LIBBSD, \ - * libbsd (arc4random): HAVE_ARC4RANDOM + HAVE_LIBBSD, \ + * BSD / macOS >=10.12 / glibc >=2.25 (getentropy): HAVE_GETENTROPY, \ * Linux (including <3.17) / BSD / macOS (including <10.7) / Solaris >=8 (/dev/urandom): XML_DEV_URANDOM, \ * Windows >=Vista (rand_s): _WIN32. \ \ - If insist on not using any of these, bypass this error by defining \ - XML_POOR_ENTROPY; you have been warned. \ + If you insist on not using any of these, bypass this error by defining \ + XML_POOR_ENTROPY and be vulnerable to hash flooding; you have been warned. \ \ If you have reasons to patch this detection code away or need changes \ to the build system, please open a bug. Thank you! @@ -220,25 +231,10 @@ typedef char ICHAR; #define EXPAT_MIN(a, b) (((a) < (b)) ? (a) : (b)) #include "internal.h" +#include "hash_table.h" #include "xmltok.h" #include "xmlrole.h" -typedef const XML_Char *KEY; - -typedef struct { - KEY name; -} NAMED; - -typedef struct { - NAMED **v; - unsigned char power; - size_t size; - size_t used; - const XML_Memory_Handling_Suite *mem; -} HASH_TABLE; - -static size_t keylen(KEY s); - static void copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key); /* For probing (after a collision) we need a step size relative prime @@ -254,11 +250,6 @@ static void copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key); #define PROBE_STEP(hash, mask, power) \ ((unsigned char)((SECOND_HASH(hash, mask, power)) | 1)) -typedef struct { - NAMED **p; - NAMED **end; -} HASH_TABLE_ITER; - #define INIT_TAG_BUF_SIZE 32 /* must be a multiple of sizeof(XML_Char) */ #define INIT_DATA_BUF_SIZE 1024 #define INIT_ATTS_SIZE 16 @@ -274,8 +265,8 @@ typedef struct binding { struct binding *prevPrefixBinding; const struct attribute_id *attId; XML_Char *uri; - int uriLen; - int uriAlloc; + size_t uriLen; + size_t uriAlloc; } BINDING; typedef struct prefix { @@ -287,9 +278,9 @@ typedef struct { const XML_Char *str; const XML_Char *localPart; const XML_Char *prefix; - int strLen; - int uriLen; - int prefixLen; + size_t strLen; + size_t uriLen; + size_t prefixLen; } TAG_NAME; /* TAG represents an open element. @@ -310,8 +301,11 @@ typedef struct tag { const char *rawName; /* tagName in the original encoding */ int rawNameLength; TAG_NAME name; /* tagName in the API encoding */ - char *buf; /* buffer for name components */ - char *bufEnd; /* end of the buffer */ + union { + char *raw; /* for byte-level access (rawName storage) */ + XML_Char *str; /* for character-level access (converted name) */ + } buf; /* buffer for name components */ + char *bufEnd; /* end of the buffer */ BINDING *bindings; } TAG; @@ -324,9 +318,13 @@ typedef struct { const XML_Char *base; const XML_Char *publicId; const XML_Char *notation; - XML_Bool open; - XML_Bool is_param; - XML_Bool is_internal; /* true if declared in internal subset outside PE */ + bool open; + bool hasMore; /* true if entity has not been completely processed */ + /* An entity can be open while being already completely processed (!hasMore). + The reason is the delayed closing of entities until their inner + entities are processed and closed */ + bool is_param; + bool is_internal; /* true if declared in internal subset outside PE */ } ENTITY; typedef struct { @@ -344,7 +342,7 @@ typedef struct { typedef struct block { struct block *next; int size; - XML_Char s[1]; + XML_Char s[]; } BLOCK; typedef struct { @@ -353,7 +351,7 @@ typedef struct { const XML_Char *end; XML_Char *ptr; XML_Char *start; - const XML_Memory_Handling_Suite *mem; + XML_Parser parser; } STRING_POOL; /* The XML_Char before the name is used to determine whether @@ -361,8 +359,8 @@ typedef struct { typedef struct attribute_id { XML_Char *name; PPREFIX *prefix; - XML_Bool maybeTokenized; - XML_Bool xmlns; + bool maybeTokenized; + bool xmlns; } ATTRIBUTE_ID; typedef struct { @@ -371,6 +369,22 @@ typedef struct { const XML_Char *value; } DEFAULT_ATTRIBUTE; +// This structure allows mapping attribute names to instances of +// `DEFAULT_ATTRIBUTE`. +typedef struct { + // Member `name` goes first to make this structure compatible with structure + // `NAMED` (further up), which is needed to support use of structure + // `NAME_AND_DEFAULT_ATTRIBUTE` in a hash table as implemented by function + // `lookup` (further down). + const XML_Char *name; + // We would store a `DEFAULT_ATTRIBUTE *` here but the backing array + // can be reallocated which would invalidate the pointer. Using an index + // into the array instead, avoids that problem. + size_t attIndex; + // This is set to `false` by function `lookup`. + bool initialized; +} NAME_AND_DEFAULT_ATTRIBUTE; + typedef struct { unsigned long version; unsigned long hash; @@ -381,9 +395,10 @@ typedef struct { const XML_Char *name; PPREFIX *prefix; const ATTRIBUTE_ID *idAtt; - int nDefaultAtts; - int allocDefaultAtts; + size_t nDefaultAtts; + size_t allocDefaultAtts; DEFAULT_ATTRIBUTE *defaultAtts; + HASH_TABLE defaultAttForName; } ELEMENT_TYPE; typedef struct { @@ -401,27 +416,35 @@ typedef struct { XML_Bool standalone; #ifdef XML_DTD /* indicates if external PE has been read */ - XML_Bool paramEntityRead; + bool paramEntityRead; HASH_TABLE paramEntities; #endif /* XML_DTD */ PPREFIX defaultPrefix; /* === scaffolding for building content model === */ - XML_Bool in_eldecl; + bool in_eldecl; CONTENT_SCAFFOLD *scaffold; unsigned contentStringLen; unsigned scaffSize; unsigned scaffCount; int scaffLevel; int *scaffIndex; + size_t scaffIndexSize; } DTD; +enum EntityType { + ENTITY_INTERNAL, + ENTITY_ATTRIBUTE, + ENTITY_VALUE, +}; + typedef struct open_internal_entity { const char *internalEventPtr; const char *internalEventEndPtr; struct open_internal_entity *next; ENTITY *entity; int startTagLevel; - XML_Bool betweenDecl; /* WFC: PE Between Declarations */ + bool betweenDecl; /* WFC: PE Between Declarations */ + enum EntityType type; } OPEN_INTERNAL_ENTITY; enum XML_Account { @@ -441,6 +464,14 @@ typedef struct accounting { unsigned long long activationThresholdBytes; } ACCOUNTING; +typedef struct MALLOC_TRACKER { + size_t bytesAllocated; + size_t peakBytesAllocated; // updated live only for debug level >=2 + unsigned long debugLevel; + float maximumAmplificationFactor; // >=1.0 + XmlBigCount activationThresholdBytes; +} MALLOC_TRACKER; + typedef struct entity_stats { unsigned int countEverOpened; unsigned int currentDepth; @@ -449,8 +480,8 @@ typedef struct entity_stats { } ENTITY_STATS; #endif /* XML_GE == 1 */ -typedef enum XML_Error PTRCALL Processor(XML_Parser parser, const char *start, - const char *end, const char **endPtr); +typedef enum XML_Error Processor(XML_Parser parser, const char *start, + const char *end, const char **endPtr); static Processor prologProcessor; static Processor prologInitProcessor; @@ -481,8 +512,8 @@ static enum XML_Error doProlog(XML_Parser parser, const ENCODING *enc, const char *next, const char **nextPtr, XML_Bool haveMore, XML_Bool allowClosingDoctype, enum XML_Account account); -static enum XML_Error processInternalEntity(XML_Parser parser, ENTITY *entity, - XML_Bool betweenDecl); +static enum XML_Error processEntity(XML_Parser parser, ENTITY *entity, + bool betweenDecl, enum EntityType type); static enum XML_Error doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, const char *start, const char *end, const char **endPtr, @@ -513,18 +544,22 @@ static enum XML_Error storeAttributeValue(XML_Parser parser, const char *ptr, const char *end, STRING_POOL *pool, enum XML_Account account); -static enum XML_Error appendAttributeValue(XML_Parser parser, - const ENCODING *enc, - XML_Bool isCdata, const char *ptr, - const char *end, STRING_POOL *pool, - enum XML_Account account); +static enum XML_Error +appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, + const char *ptr, const char *end, STRING_POOL *pool, + enum XML_Account account, const char **nextPtr); static ATTRIBUTE_ID *getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start, const char *end); static int setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *elementType); #if XML_GE == 1 static enum XML_Error storeEntityValue(XML_Parser parser, const ENCODING *enc, const char *start, const char *end, - enum XML_Account account); + enum XML_Account account, + const char **nextPtr); +static enum XML_Error callStoreEntityValue(XML_Parser parser, + const ENCODING *enc, + const char *start, const char *end, + enum XML_Account account); #else static enum XML_Error storeSelfEntityValue(XML_Parser parser, ENTITY *entity); #endif @@ -538,57 +573,47 @@ static void reportDefault(XML_Parser parser, const ENCODING *enc, static const XML_Char *getContext(XML_Parser parser); static XML_Bool setContext(XML_Parser parser, const XML_Char *context); -static void FASTCALL normalizePublicId(XML_Char *s); +static void normalizePublicId(XML_Char *s); -static DTD *dtdCreate(const XML_Memory_Handling_Suite *ms); +static DTD *dtdCreate(XML_Parser parser); /* do not call if m_parentParser != NULL */ -static void dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms); -static void dtdDestroy(DTD *p, XML_Bool isDocEntity, - const XML_Memory_Handling_Suite *ms); +static void dtdReset(DTD *p, XML_Parser parser); +static void dtdDestroy(DTD *p, XML_Bool isDocEntity, XML_Parser parser); static int dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, - const XML_Memory_Handling_Suite *ms); + XML_Parser parser); static int copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, STRING_POOL *newPool, const HASH_TABLE *oldTable); -static NAMED *lookup(XML_Parser parser, HASH_TABLE *table, KEY name, - size_t createSize); -static void FASTCALL hashTableInit(HASH_TABLE *table, - const XML_Memory_Handling_Suite *ms); -static void FASTCALL hashTableClear(HASH_TABLE *table); -static void FASTCALL hashTableDestroy(HASH_TABLE *table); -static void FASTCALL hashTableIterInit(HASH_TABLE_ITER *iter, - const HASH_TABLE *table); -static NAMED *FASTCALL hashTableIterNext(HASH_TABLE_ITER *iter); - -static void FASTCALL poolInit(STRING_POOL *pool, - const XML_Memory_Handling_Suite *ms); -static void FASTCALL poolClear(STRING_POOL *pool); -static void FASTCALL poolDestroy(STRING_POOL *pool); + +static void poolInit(STRING_POOL *pool, XML_Parser parser); +static void poolClear(STRING_POOL *pool); +static void poolDestroy(STRING_POOL *pool); static XML_Char *poolAppend(STRING_POOL *pool, const ENCODING *enc, const char *ptr, const char *end); static XML_Char *poolStoreString(STRING_POOL *pool, const ENCODING *enc, const char *ptr, const char *end); -static XML_Bool FASTCALL poolGrow(STRING_POOL *pool); -static const XML_Char *FASTCALL poolCopyString(STRING_POOL *pool, - const XML_Char *s); +static XML_Bool poolGrow(STRING_POOL *pool); +static bool poolGrowUntil(STRING_POOL *pool, size_t needed); +static const XML_Char *poolCopyString(STRING_POOL *pool, const XML_Char *s); +static const XML_Char *poolCopyStringNoFinish(STRING_POOL *pool, + const XML_Char *s); static const XML_Char *poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n); -static const XML_Char *FASTCALL poolAppendString(STRING_POOL *pool, - const XML_Char *s); +static const XML_Char *poolAppendString(STRING_POOL *pool, const XML_Char *s); -static int FASTCALL nextScaffoldPart(XML_Parser parser); +static int nextScaffoldPart(XML_Parser parser); static XML_Content *build_model(XML_Parser parser); static ELEMENT_TYPE *getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, const char *end); -static XML_Char *copyString(const XML_Char *s, - const XML_Memory_Handling_Suite *memsuite); +static XML_Char *copyString(const XML_Char *s, XML_Parser parser); -static unsigned long generate_hash_secret_salt(XML_Parser parser); +static struct sipkey generate_hash_secret_salt(void); static XML_Bool startParsing(XML_Parser parser); static XML_Parser parserCreate(const XML_Char *encodingName, const XML_Memory_Handling_Suite *memsuite, - const XML_Char *nameSep, DTD *dtd); + const XML_Char *nameSep, DTD *dtd, + XML_Parser parentParser); static void parserInit(XML_Parser parser, const XML_Char *encodingName); @@ -612,24 +637,67 @@ static void entityTrackingOnOpen(XML_Parser parser, ENTITY *entity, int sourceLine); static void entityTrackingOnClose(XML_Parser parser, ENTITY *entity, int sourceLine); +#endif /* XML_GE == 1 */ static XML_Parser getRootParserOf(XML_Parser parser, unsigned int *outLevelDiff); -#endif /* XML_GE == 1 */ static unsigned long getDebugLevel(const char *variableName, unsigned long defaultDebugLevel); +static bool poolAppendChar(STRING_POOL *pool, XML_Char c); + +static bool poolAppendChars(STRING_POOL *pool, const XML_Char *s, size_t len); + +#if XML_GE == 1 +static enum XML_Prop_Error setBillionLaughsAttackProtectionMaximumAmplification( + XML_Parser parser, float maximumAmplificationFactor); + +static enum XML_Prop_Error setBillionLaughsAttackProtectionActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes); + +static enum XML_Prop_Error +setAllocTrackerMaximumAmplification(XML_Parser parser, + float maximumAmplificationFactor); + +static enum XML_Prop_Error +setAllocTrackerActivationThreshold(XML_Parser parser, + unsigned long long activationThresholdBytes); +#endif /* XML_GE == 1 */ + +static enum XML_Prop_Error setReparseDeferralEnabled(XML_Parser parser, + XML_Bool enabled); + #define poolStart(pool) ((pool)->start) #define poolLength(pool) ((pool)->ptr - (pool)->start) #define poolChop(pool) ((void)--(pool->ptr)) #define poolLastChar(pool) (((pool)->ptr)[-1]) #define poolDiscard(pool) ((pool)->ptr = (pool)->start) #define poolFinish(pool) ((pool)->start = (pool)->ptr) -#define poolAppendChar(pool, c) \ - (((pool)->ptr == (pool)->end && ! poolGrow(pool)) \ - ? 0 \ - : ((*((pool)->ptr)++ = c), 1)) + +bool +poolAppendChar(STRING_POOL *pool, XML_Char c) { + if (pool->ptr == pool->end && ! poolGrow(pool)) + return false; + + *(pool->ptr)++ = c; + return true; +} + +bool +poolAppendChars(STRING_POOL *pool, const XML_Char *s, size_t len) { + // Detect and prevent integer overflow + if (len > SIZE_MAX / sizeof(XML_Char)) + return false; + + if (! poolGrowUntil(pool, len)) + return false; + + memcpy(pool->ptr, s, len * sizeof(XML_Char)); + pool->ptr += len; + + return true; +} #if ! defined(XML_TESTING) const @@ -661,7 +729,7 @@ struct XML_ParserStruct { char *m_bufferEnd; // past last character to be parsed const char *m_bufferLim; // allocated end of m_buffer - XML_Index m_parseEndByteIndex; + uint64_t m_parseEndByteIndex; const char *m_parseEndPtr; size_t m_partialTokenBytesBefore; /* used in heuristic to avoid O(n^2) */ XML_Bool m_reparseDeferralEnabled; @@ -700,6 +768,8 @@ struct XML_ParserStruct { void *m_unknownEncodingMem; void *m_unknownEncodingData; void *m_unknownEncodingHandlerData; + // Application callback invoked by callUnknownEncodingConvert. + int(XMLCALL *m_unknownEncodingConvert)(void *, const char *); void(XMLCALL *m_unknownEncodingRelease)(void *); PROLOG_STATE m_prologState; Processor *m_processor; @@ -708,7 +778,9 @@ struct XML_ParserStruct { const char *m_eventEndPtr; const char *m_positionPtr; OPEN_INTERNAL_ENTITY *m_openInternalEntities; - OPEN_INTERNAL_ENTITY *m_freeInternalEntities; + OPEN_INTERNAL_ENTITY *m_openAttributeEntities; + OPEN_INTERNAL_ENTITY *m_openValueEntities; + OPEN_INTERNAL_ENTITY *m_freeEntities; XML_Bool m_defaultExpandInternalEntities; int m_tagLevel; ENTITY *m_declEntity; @@ -728,21 +800,18 @@ struct XML_ParserStruct { TAG *m_freeTagList; BINDING *m_inheritedBindings; BINDING *m_freeBindingList; - int m_attsSize; + size_t m_attsSize; int m_nSpecifiedAtts; int m_idAttIndex; ATTRIBUTE *m_atts; NS_ATT *m_nsAtts; unsigned long m_nsAttsVersion; unsigned char m_nsAttsPower; -#ifdef XML_ATTR_INFO - XML_AttrInfo *m_attInfo; -#endif POSITION m_position; STRING_POOL m_tempPool; STRING_POOL m_temp2Pool; char *m_groupConnector; - unsigned int m_groupSize; + size_t m_groupSize; XML_Char m_namespaceSeparator; XML_Parser m_parentParser; XML_ParsingStatus m_parsingStatus; @@ -751,165 +820,265 @@ struct XML_ParserStruct { XML_Bool m_useForeignDTD; enum XML_ParamEntityParsing m_paramEntityParsing; #endif - unsigned long m_hash_secret_salt; + struct sipkey m_hash_secret_salt_128; + XML_Bool m_hash_secret_salt_set; #if XML_GE == 1 ACCOUNTING m_accounting; + MALLOC_TRACKER m_alloc_tracker; ENTITY_STATS m_entity_stats; #endif + XML_Bool m_reenter; + unsigned m_handlerCallDepth; }; -#define MALLOC(parser, s) (parser->m_mem.malloc_fcn((s))) -#define REALLOC(parser, p, s) (parser->m_mem.realloc_fcn((p), (s))) -#define FREE(parser, p) (parser->m_mem.free_fcn((p))) - -XML_Parser XMLCALL -XML_ParserCreate(const XML_Char *encodingName) { - return XML_ParserCreate_MM(encodingName, NULL, NULL); -} +#if XML_GE == 1 +# define MALLOC(parser, s) (expat_malloc((parser), (s), __LINE__)) +# define REALLOC(parser, p, s) (expat_realloc((parser), (p), (s), __LINE__)) +# define FREE(parser, p) (expat_free((parser), (p), __LINE__)) +#else +# define MALLOC(parser, s) (parser->m_mem.malloc_fcn((s))) +# define REALLOC(parser, p, s) (parser->m_mem.realloc_fcn((p), (s))) +# define FREE(parser, p) (parser->m_mem.free_fcn((p))) +#endif -XML_Parser XMLCALL -XML_ParserCreateNS(const XML_Char *encodingName, XML_Char nsSep) { - XML_Char tmp[2] = {nsSep, 0}; - return XML_ParserCreate_MM(encodingName, NULL, tmp); +#if XML_GE == 1 +static void +expat_heap_stat(XML_Parser rootParser, char operator, size_t absDiff, + size_t newTotal, size_t peakTotal, int sourceLine) { + // NOTE: This can be +infinity or -nan + const float amplification + = (float)newTotal / (float)rootParser->m_accounting.countBytesDirect; + fprintf( + stderr, + "expat: Allocations(%p): Direct " EXPAT_FMT_ULL("10") ", allocated %c" EXPAT_FMT_SIZE_T( + "10") " to " EXPAT_FMT_SIZE_T("10") " (" EXPAT_FMT_SIZE_T("10") " peak), amplification %8.2f (xmlparse.c:%d)\n", + (void *)rootParser, rootParser->m_accounting.countBytesDirect, operator, + absDiff, newTotal, peakTotal, (double)amplification, sourceLine); } -// "xml=http://www.w3.org/XML/1998/namespace" -static const XML_Char implicitContext[] - = {ASCII_x, ASCII_m, ASCII_l, ASCII_EQUALS, ASCII_h, - ASCII_t, ASCII_t, ASCII_p, ASCII_COLON, ASCII_SLASH, - ASCII_SLASH, ASCII_w, ASCII_w, ASCII_w, ASCII_PERIOD, - ASCII_w, ASCII_3, ASCII_PERIOD, ASCII_o, ASCII_r, - ASCII_g, ASCII_SLASH, ASCII_X, ASCII_M, ASCII_L, - ASCII_SLASH, ASCII_1, ASCII_9, ASCII_9, ASCII_8, - ASCII_SLASH, ASCII_n, ASCII_a, ASCII_m, ASCII_e, - ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e, - '\0'}; +static bool +expat_heap_increase_tolerable(XML_Parser rootParser, size_t increase, + int sourceLine) { + assert(rootParser != NULL); + assert(increase > 0); -/* To avoid warnings about unused functions: */ -#if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) + size_t newTotal = 0; + bool tolerable = true; -# if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) + // Detect integer overflow + if (SIZE_MAX - rootParser->m_alloc_tracker.bytesAllocated < increase) { + tolerable = false; + } else { + newTotal = rootParser->m_alloc_tracker.bytesAllocated + increase; + + if (newTotal >= rootParser->m_alloc_tracker.activationThresholdBytes) { + assert(newTotal > 0); + // NOTE: This can be +infinity when dividing by zero but not -nan + const float amplification + = (float)newTotal / (float)rootParser->m_accounting.countBytesDirect; + if (amplification + > rootParser->m_alloc_tracker.maximumAmplificationFactor) { + tolerable = false; + } + } + } -/* Obtain entropy on Linux 3.17+ */ -static int -writeRandomBytes_getrandom_nonblock(void *target, size_t count) { - int success = 0; /* full count bytes written? */ - size_t bytesWrittenTotal = 0; - const unsigned int getrandomFlags = GRND_NONBLOCK; + if (! tolerable && (rootParser->m_alloc_tracker.debugLevel >= 1)) { + expat_heap_stat(rootParser, '+', increase, newTotal, newTotal, sourceLine); + } - do { - void *const currentTarget = (void *)((char *)target + bytesWrittenTotal); - const size_t bytesToWrite = count - bytesWrittenTotal; + return tolerable; +} - const int bytesWrittenMore = -# if defined(HAVE_GETRANDOM) - getrandom(currentTarget, bytesToWrite, getrandomFlags); -# else - syscall(SYS_getrandom, currentTarget, bytesToWrite, getrandomFlags); -# endif +# if defined(XML_TESTING) +void * +# else +static void * +# endif +expat_malloc(XML_Parser parser, size_t size, int sourceLine) { + // Detect integer overflow + if (SIZE_MAX - size < sizeof(size_t) + EXPAT_MALLOC_PADDING) { + return NULL; + } - if (bytesWrittenMore > 0) { - bytesWrittenTotal += bytesWrittenMore; - if (bytesWrittenTotal >= count) - success = 1; - } - } while (! success && (errno == EINTR)); + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(rootParser->m_parentParser == NULL); - return success; -} + const size_t bytesToAllocate = sizeof(size_t) + EXPAT_MALLOC_PADDING + size; -# endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ + if (SIZE_MAX - rootParser->m_alloc_tracker.bytesAllocated < bytesToAllocate) { + return NULL; // i.e. signal integer overflow as out-of-memory + } -# if ! defined(_WIN32) && defined(XML_DEV_URANDOM) + if (! expat_heap_increase_tolerable(rootParser, bytesToAllocate, + sourceLine)) { + return NULL; // i.e. signal violation as out-of-memory + } -/* Extract entropy from /dev/urandom */ -static int -writeRandomBytes_dev_urandom(void *target, size_t count) { - int success = 0; /* full count bytes written? */ - size_t bytesWrittenTotal = 0; + // Actually allocate + void *const mallocedPtr = parser->m_mem.malloc_fcn(bytesToAllocate); - const int fd = open("/dev/urandom", O_RDONLY); - if (fd < 0) { - return 0; + if (mallocedPtr == NULL) { + return NULL; } - do { - void *const currentTarget = (void *)((char *)target + bytesWrittenTotal); - const size_t bytesToWrite = count - bytesWrittenTotal; + // Update in-block recorded size + *(size_t *)mallocedPtr = size; - const ssize_t bytesWrittenMore = read(fd, currentTarget, bytesToWrite); + // Update accounting + rootParser->m_alloc_tracker.bytesAllocated += bytesToAllocate; - if (bytesWrittenMore > 0) { - bytesWrittenTotal += bytesWrittenMore; - if (bytesWrittenTotal >= count) - success = 1; + // Report as needed + if (rootParser->m_alloc_tracker.debugLevel >= 2) { + if (rootParser->m_alloc_tracker.bytesAllocated + > rootParser->m_alloc_tracker.peakBytesAllocated) { + rootParser->m_alloc_tracker.peakBytesAllocated + = rootParser->m_alloc_tracker.bytesAllocated; } - } while (! success && (errno == EINTR)); + expat_heap_stat(rootParser, '+', bytesToAllocate, + rootParser->m_alloc_tracker.bytesAllocated, + rootParser->m_alloc_tracker.peakBytesAllocated, sourceLine); + } - close(fd); - return success; + return (char *)mallocedPtr + sizeof(size_t) + EXPAT_MALLOC_PADDING; } -# endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */ +# if defined(XML_TESTING) +void +# else +static void +# endif +expat_free(XML_Parser parser, void *ptr, int sourceLine) { + assert(parser != NULL); -#endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */ + if (ptr == NULL) { + return; + } -#if defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF) + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(rootParser->m_parentParser == NULL); -static void -writeRandomBytes_arc4random(void *target, size_t count) { - size_t bytesWrittenTotal = 0; + // Extract size (to the eyes of malloc_fcn/realloc_fcn) and + // the original pointer returned by malloc/realloc + void *const mallocedPtr = (char *)ptr - EXPAT_MALLOC_PADDING - sizeof(size_t); + const size_t bytesAllocated + = sizeof(size_t) + EXPAT_MALLOC_PADDING + *(size_t *)mallocedPtr; - while (bytesWrittenTotal < count) { - const uint32_t random32 = arc4random(); - size_t i = 0; + // Update accounting + assert(rootParser->m_alloc_tracker.bytesAllocated >= bytesAllocated); + rootParser->m_alloc_tracker.bytesAllocated -= bytesAllocated; - for (; (i < sizeof(random32)) && (bytesWrittenTotal < count); - i++, bytesWrittenTotal++) { - const uint8_t random8 = (uint8_t)(random32 >> (i * 8)); - ((uint8_t *)target)[bytesWrittenTotal] = random8; - } + // Report as needed + if (rootParser->m_alloc_tracker.debugLevel >= 2) { + expat_heap_stat(rootParser, '-', bytesAllocated, + rootParser->m_alloc_tracker.bytesAllocated, + rootParser->m_alloc_tracker.peakBytesAllocated, sourceLine); } + + // NOTE: This may be freeing rootParser, so freeing has to come last + parser->m_mem.free_fcn(mallocedPtr); } -#endif /* defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF) */ +# if defined(XML_TESTING) +void * +# else +static void * +# endif +expat_realloc(XML_Parser parser, void *ptr, size_t size, int sourceLine) { + assert(parser != NULL); -#ifdef _WIN32 + if (ptr == NULL) { + return expat_malloc(parser, size, sourceLine); + } -/* Provide declaration of rand_s() for MinGW-32 (not 64, which has it), - as it didn't declare it in its header prior to version 5.3.0 of its - runtime package (mingwrt, containing stdlib.h). The upstream fix - was introduced at https://osdn.net/projects/mingw/ticket/39658 . */ -# if defined(__MINGW32__) && defined(__MINGW32_VERSION) \ - && __MINGW32_VERSION < 5003000L && ! defined(__MINGW64_VERSION_MAJOR) -__declspec(dllimport) int rand_s(unsigned int *); -# endif + if (size == 0) { + expat_free(parser, ptr, sourceLine); + return NULL; + } -/* Obtain entropy on Windows using the rand_s() function which - * generates cryptographically secure random numbers. Internally it - * uses RtlGenRandom API which is present in Windows XP and later. - */ -static int -writeRandomBytes_rand_s(void *target, size_t count) { - size_t bytesWrittenTotal = 0; + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(rootParser->m_parentParser == NULL); + + // Extract original size (to the eyes of the caller) and the original + // pointer returned by malloc/realloc + void *mallocedPtr = (char *)ptr - EXPAT_MALLOC_PADDING - sizeof(size_t); + const size_t prevSize = *(size_t *)mallocedPtr; + + // Classify upcoming change + const bool isIncrease = (size > prevSize); + const size_t absDiff + = (size > prevSize) ? (size - prevSize) : (prevSize - size); + + // Ask for permission from accounting + if (isIncrease) { + if (! expat_heap_increase_tolerable(rootParser, absDiff, sourceLine)) { + return NULL; // i.e. signal violation as out-of-memory + } + } + + // Detect and prevent integer overflow + if (size > SIZE_MAX - sizeof(size_t) - EXPAT_MALLOC_PADDING) { + return NULL; + } + + // Actually allocate + mallocedPtr = parser->m_mem.realloc_fcn( + mallocedPtr, sizeof(size_t) + EXPAT_MALLOC_PADDING + size); - while (bytesWrittenTotal < count) { - unsigned int random32 = 0; - size_t i = 0; + if (mallocedPtr == NULL) { + return NULL; + } - if (rand_s(&random32)) - return 0; /* failure */ + // Update accounting + if (isIncrease) { + assert(SIZE_MAX - rootParser->m_alloc_tracker.bytesAllocated >= absDiff); + rootParser->m_alloc_tracker.bytesAllocated += absDiff; + } else { // i.e. decrease + assert(rootParser->m_alloc_tracker.bytesAllocated >= absDiff); + rootParser->m_alloc_tracker.bytesAllocated -= absDiff; + } - for (; (i < sizeof(random32)) && (bytesWrittenTotal < count); - i++, bytesWrittenTotal++) { - const uint8_t random8 = (uint8_t)(random32 >> (i * 8)); - ((uint8_t *)target)[bytesWrittenTotal] = random8; + // Report as needed + if (rootParser->m_alloc_tracker.debugLevel >= 2) { + if (rootParser->m_alloc_tracker.bytesAllocated + > rootParser->m_alloc_tracker.peakBytesAllocated) { + rootParser->m_alloc_tracker.peakBytesAllocated + = rootParser->m_alloc_tracker.bytesAllocated; } + expat_heap_stat(rootParser, isIncrease ? '+' : '-', absDiff, + rootParser->m_alloc_tracker.bytesAllocated, + rootParser->m_alloc_tracker.peakBytesAllocated, sourceLine); } - return 1; /* success */ + + // Update in-block recorded size + *(size_t *)mallocedPtr = size; + + return (char *)mallocedPtr + sizeof(size_t) + EXPAT_MALLOC_PADDING; +} +#endif // XML_GE == 1 + +XML_Parser XMLCALL +XML_ParserCreate(const XML_Char *encodingName) { + return XML_ParserCreate_MM(encodingName, NULL, NULL); } -#endif /* _WIN32 */ +XML_Parser XMLCALL +XML_ParserCreateNS(const XML_Char *encodingName, XML_Char nsSep) { + XML_Char tmp[2] = {nsSep, 0}; + return XML_ParserCreate_MM(encodingName, NULL, tmp); +} + +// "xml=http://www.w3.org/XML/1998/namespace" +static const XML_Char implicitContext[] + = {ASCII_x, ASCII_m, ASCII_l, ASCII_EQUALS, ASCII_h, + ASCII_t, ASCII_t, ASCII_p, ASCII_COLON, ASCII_SLASH, + ASCII_SLASH, ASCII_w, ASCII_w, ASCII_w, ASCII_PERIOD, + ASCII_w, ASCII_3, ASCII_PERIOD, ASCII_o, ASCII_r, + ASCII_g, ASCII_SLASH, ASCII_X, ASCII_M, ASCII_L, + ASCII_SLASH, ASCII_1, ASCII_9, ASCII_9, ASCII_8, + ASCII_SLASH, ASCII_n, ASCII_a, ASCII_m, ASCII_e, + ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e, + '\0'}; #if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) @@ -938,63 +1107,104 @@ gather_time_entropy(void) { #endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */ -static unsigned long -ENTROPY_DEBUG(const char *label, unsigned long entropy) { +static struct sipkey +ENTROPY_DEBUG(const char *label, struct sipkey entropy_128) { if (getDebugLevel("EXPAT_ENTROPY_DEBUG", 0) >= 1u) { - fprintf(stderr, "expat: Entropy: %s --> 0x%0*lx (%lu bytes)\n", label, - (int)sizeof(entropy) * 2, entropy, (unsigned long)sizeof(entropy)); + fprintf(stderr, + "expat: Entropy: %s --> [0x" EXPAT_FMT_LLX( + "016") ", 0x" EXPAT_FMT_LLX("016") "] (16 bytes)\n", + label, (unsigned long long)entropy_128.k[0], + (unsigned long long)entropy_128.k[1]); } - return entropy; + return entropy_128; } -static unsigned long -generate_hash_secret_salt(XML_Parser parser) { - unsigned long entropy; - (void)parser; +static struct sipkey +generate_hash_secret_salt(void) { + struct sipkey entropy; /* "Failproof" high quality providers: */ #if defined(HAVE_ARC4RANDOM_BUF) - arc4random_buf(&entropy, sizeof(entropy)); + writeRandomBytes_arc4random_buf(&entropy, sizeof(entropy)); return ENTROPY_DEBUG("arc4random_buf", entropy); #elif defined(HAVE_ARC4RANDOM) - writeRandomBytes_arc4random((void *)&entropy, sizeof(entropy)); + writeRandomBytes_arc4random(&entropy, sizeof(entropy)); return ENTROPY_DEBUG("arc4random", entropy); #else /* Try high quality providers first .. */ -# ifdef _WIN32 - if (writeRandomBytes_rand_s((void *)&entropy, sizeof(entropy))) { +# if defined(_WIN32) && ! defined(XML_POOR_ENTROPY) + if (writeRandomBytes_rand_s(&entropy, sizeof(entropy))) { return ENTROPY_DEBUG("rand_s", entropy); } +# elif defined(HAVE_GETENTROPY) + if (writeRandomBytes_getentropy(&entropy, sizeof(entropy))) { + return ENTROPY_DEBUG("getentropy", entropy); + } + errno = 0; # elif defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) - if (writeRandomBytes_getrandom_nonblock((void *)&entropy, sizeof(entropy))) { + if (writeRandomBytes_getrandom_nonblock(&entropy, sizeof(entropy))) { return ENTROPY_DEBUG("getrandom", entropy); } # endif # if ! defined(_WIN32) && defined(XML_DEV_URANDOM) - if (writeRandomBytes_dev_urandom((void *)&entropy, sizeof(entropy))) { + if (writeRandomBytes_dev_urandom(&entropy, sizeof(entropy))) { return ENTROPY_DEBUG("/dev/urandom", entropy); } # endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */ /* .. and self-made low quality for backup: */ + entropy.k[0] = 0; + entropy.k[1] = gather_time_entropy(); +# if ! defined(__wasi__) /* Process ID is 0 bits entropy if attacker has local access */ - entropy = gather_time_entropy() ^ getpid(); + entropy.k[1] ^= getpid(); +# endif /* Factors are 2^31-1 and 2^61-1 (Mersenne primes M31 and M61) */ if (sizeof(unsigned long) == 4) { - return ENTROPY_DEBUG("fallback(4)", entropy * 2147483647); + entropy.k[1] *= 2147483647; + return ENTROPY_DEBUG("fallback(4)", entropy); } else { - return ENTROPY_DEBUG("fallback(8)", - entropy * (unsigned long)2305843009213693951ULL); + entropy.k[1] *= 2305843009213693951ULL; + return ENTROPY_DEBUG("fallback(8)", entropy); } #endif } -static unsigned long -get_hash_secret_salt(XML_Parser parser) { - if (parser->m_parentParser != NULL) - return get_hash_secret_salt(parser->m_parentParser); - return parser->m_hash_secret_salt; +static void +beforeHandler(XML_Parser parser) { + assert(parser->m_handlerCallDepth < UINT_MAX); + parser->m_handlerCallDepth++; +} + +static void +afterHandler(XML_Parser parser) { + assert(parser->m_handlerCallDepth > 0); + parser->m_handlerCallDepth--; +} + +static bool +isCalledFromInsideHandler(XML_Parser parser) { + return parser->m_handlerCallDepth > 0; +} + +static void +callUnknownEncodingRelease(XML_Parser parser) { + beforeHandler(parser); + parser->m_unknownEncodingRelease(parser->m_unknownEncodingData); + afterHandler(parser); + parser->m_unknownEncodingRelease = NULL; + parser->m_unknownEncodingData = NULL; +} + +static int XMLCALL +callUnknownEncodingConvert(void *data, const char *p) { + XML_Parser parser = data; + beforeHandler(parser); + const int result + = parser->m_unknownEncodingConvert(parser->m_unknownEncodingData, p); + afterHandler(parser); + return result; } static enum XML_Error @@ -1028,7 +1238,29 @@ callProcessor(XML_Parser parser, const char *start, const char *end, #if defined(XML_TESTING) g_bytesScanned += (unsigned)have_now; #endif - const enum XML_Error ret = parser->m_processor(parser, start, end, endPtr); + // Run in a loop to eliminate dangerous recursion depths + enum XML_Error ret; + *endPtr = start; + while (1) { + // Use endPtr as the new start in each iteration, since it will + // be set to the next start point by m_processor. + ret = parser->m_processor(parser, *endPtr, end, endPtr); + + // Make parsing status (and in particular XML_SUSPENDED) take + // precedence over re-enter flag when they disagree + if (parser->m_parsingStatus.parsing != XML_PARSING) { + parser->m_reenter = XML_FALSE; + } + + if (! parser->m_reenter) { + break; + } + + parser->m_reenter = XML_FALSE; + if (ret != XML_ERROR_NONE) + return ret; + } + if (ret == XML_ERROR_NONE) { // if we consumed nothing, remember what we had on this parse attempt. if (*endPtr == start) { @@ -1043,8 +1275,10 @@ callProcessor(XML_Parser parser, const char *start, const char *end, static XML_Bool /* only valid for root parser */ startParsing(XML_Parser parser) { /* hash functions must be initialized before setContext() is called */ - if (parser->m_hash_secret_salt == 0) - parser->m_hash_secret_salt = generate_hash_secret_salt(parser); + if (parser->m_hash_secret_salt_set != XML_TRUE) { + parser->m_hash_secret_salt_128 = generate_hash_secret_salt(); + parser->m_hash_secret_salt_set = XML_TRUE; + } if (parser->m_ns) { /* implicit context only set for root parser, since child parsers (i.e. external entity parsers) will inherit it @@ -1058,19 +1292,43 @@ XML_Parser XMLCALL XML_ParserCreate_MM(const XML_Char *encodingName, const XML_Memory_Handling_Suite *memsuite, const XML_Char *nameSep) { - return parserCreate(encodingName, memsuite, nameSep, NULL); + return parserCreate(encodingName, memsuite, nameSep, NULL, NULL); } static XML_Parser parserCreate(const XML_Char *encodingName, const XML_Memory_Handling_Suite *memsuite, const XML_Char *nameSep, - DTD *dtd) { - XML_Parser parser; + DTD *dtd, XML_Parser parentParser) { + XML_Parser parser = NULL; + +#if XML_GE == 1 + const size_t increase + = sizeof(size_t) + EXPAT_MALLOC_PADDING + sizeof(struct XML_ParserStruct); + + if (parentParser != NULL) { + const XML_Parser rootParser = getRootParserOf(parentParser, NULL); + if (! expat_heap_increase_tolerable(rootParser, increase, __LINE__)) { + return NULL; + } + } +#else + UNUSED_P(parentParser); +#endif if (memsuite) { XML_Memory_Handling_Suite *mtemp; +#if XML_GE == 1 + void *const sizeAndParser + = memsuite->malloc_fcn(sizeof(size_t) + EXPAT_MALLOC_PADDING + + sizeof(struct XML_ParserStruct)); + if (sizeAndParser != NULL) { + *(size_t *)sizeAndParser = sizeof(struct XML_ParserStruct); + parser = (XML_Parser)((char *)sizeAndParser + sizeof(size_t) + + EXPAT_MALLOC_PADDING); +#else parser = memsuite->malloc_fcn(sizeof(struct XML_ParserStruct)); if (parser != NULL) { +#endif mtemp = (XML_Memory_Handling_Suite *)&(parser->m_mem); mtemp->malloc_fcn = memsuite->malloc_fcn; mtemp->realloc_fcn = memsuite->realloc_fcn; @@ -1078,44 +1336,80 @@ parserCreate(const XML_Char *encodingName, } } else { XML_Memory_Handling_Suite *mtemp; - parser = (XML_Parser)malloc(sizeof(struct XML_ParserStruct)); +#if XML_GE == 1 + void *const sizeAndParser = malloc(sizeof(size_t) + EXPAT_MALLOC_PADDING + + sizeof(struct XML_ParserStruct)); + if (sizeAndParser != NULL) { + *(size_t *)sizeAndParser = sizeof(struct XML_ParserStruct); + parser = (XML_Parser)((char *)sizeAndParser + sizeof(size_t) + + EXPAT_MALLOC_PADDING); +#else + parser = malloc(sizeof(struct XML_ParserStruct)); if (parser != NULL) { +#endif mtemp = (XML_Memory_Handling_Suite *)&(parser->m_mem); mtemp->malloc_fcn = malloc; mtemp->realloc_fcn = realloc; mtemp->free_fcn = free; } - } + } // cppcheck-suppress[memleak symbolName=sizeAndParser] // Cppcheck >=2.18.0 if (! parser) return parser; +#if XML_GE == 1 + // Initialize .m_alloc_tracker + memset(&parser->m_alloc_tracker, 0, sizeof(MALLOC_TRACKER)); + if (parentParser == NULL) { + parser->m_alloc_tracker.debugLevel + = getDebugLevel("EXPAT_MALLOC_DEBUG", 0u); + parser->m_alloc_tracker.maximumAmplificationFactor + = EXPAT_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT; + parser->m_alloc_tracker.activationThresholdBytes + = EXPAT_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT; + + // NOTE: This initialization needs to come this early because these fields + // are read by allocation tracking code + parser->m_parentParser = NULL; + parser->m_accounting.countBytesDirect = 0; + } else { + parser->m_parentParser = parentParser; + } + + // Record XML_ParserStruct allocation we did a few lines up before + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(rootParser->m_parentParser == NULL); + assert(SIZE_MAX - rootParser->m_alloc_tracker.bytesAllocated >= increase); + rootParser->m_alloc_tracker.bytesAllocated += increase; + + // Report on allocation + if (rootParser->m_alloc_tracker.debugLevel >= 2) { + if (rootParser->m_alloc_tracker.bytesAllocated + > rootParser->m_alloc_tracker.peakBytesAllocated) { + rootParser->m_alloc_tracker.peakBytesAllocated + = rootParser->m_alloc_tracker.bytesAllocated; + } + + expat_heap_stat(rootParser, '+', increase, + rootParser->m_alloc_tracker.bytesAllocated, + rootParser->m_alloc_tracker.peakBytesAllocated, __LINE__); + } +#else + parser->m_parentParser = NULL; +#endif // XML_GE == 1 + parser->m_buffer = NULL; parser->m_bufferLim = NULL; parser->m_attsSize = INIT_ATTS_SIZE; - parser->m_atts - = (ATTRIBUTE *)MALLOC(parser, parser->m_attsSize * sizeof(ATTRIBUTE)); + parser->m_atts = MALLOC(parser, parser->m_attsSize * sizeof(ATTRIBUTE)); if (parser->m_atts == NULL) { FREE(parser, parser); return NULL; } -#ifdef XML_ATTR_INFO - parser->m_attInfo = (XML_AttrInfo *)MALLOC( - parser, parser->m_attsSize * sizeof(XML_AttrInfo)); - if (parser->m_attInfo == NULL) { - FREE(parser, parser->m_atts); - FREE(parser, parser); - return NULL; - } -#endif - parser->m_dataBuf - = (XML_Char *)MALLOC(parser, INIT_DATA_BUF_SIZE * sizeof(XML_Char)); + parser->m_dataBuf = MALLOC(parser, INIT_DATA_BUF_SIZE * sizeof(XML_Char)); if (parser->m_dataBuf == NULL) { FREE(parser, parser->m_atts); -#ifdef XML_ATTR_INFO - FREE(parser, parser->m_attInfo); -#endif FREE(parser, parser); return NULL; } @@ -1124,13 +1418,10 @@ parserCreate(const XML_Char *encodingName, if (dtd) parser->m_dtd = dtd; else { - parser->m_dtd = dtdCreate(&parser->m_mem); + parser->m_dtd = dtdCreate(parser); if (parser->m_dtd == NULL) { FREE(parser, parser->m_dataBuf); FREE(parser, parser->m_atts); -#ifdef XML_ATTR_INFO - FREE(parser, parser->m_attInfo); -#endif FREE(parser, parser); return NULL; } @@ -1138,7 +1429,7 @@ parserCreate(const XML_Char *encodingName, parser->m_freeBindingList = NULL; parser->m_freeTagList = NULL; - parser->m_freeInternalEntities = NULL; + parser->m_freeEntities = NULL; parser->m_groupSize = 0; parser->m_groupConnector = NULL; @@ -1156,8 +1447,8 @@ parserCreate(const XML_Char *encodingName, parser->m_protocolEncodingName = NULL; - poolInit(&parser->m_tempPool, &(parser->m_mem)); - poolInit(&parser->m_temp2Pool, &(parser->m_mem)); + poolInit(&parser->m_tempPool, parser); + poolInit(&parser->m_temp2Pool, parser); parserInit(parser, encodingName); if (encodingName && ! parser->m_protocolEncodingName) { @@ -1189,7 +1480,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_processor = prologInitProcessor; XmlPrologStateInit(&parser->m_prologState); if (encodingName != NULL) { - parser->m_protocolEncodingName = copyString(encodingName, &(parser->m_mem)); + parser->m_protocolEncodingName = copyString(encodingName, parser); } parser->m_curBase = NULL; XmlInitEncoding(&parser->m_initEncoding, &parser->m_encoding, 0); @@ -1241,22 +1532,29 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_eventEndPtr = NULL; parser->m_positionPtr = NULL; parser->m_openInternalEntities = NULL; + parser->m_openAttributeEntities = NULL; + parser->m_openValueEntities = NULL; parser->m_defaultExpandInternalEntities = XML_TRUE; parser->m_tagLevel = 0; parser->m_tagStack = NULL; parser->m_inheritedBindings = NULL; parser->m_nSpecifiedAtts = 0; parser->m_unknownEncodingMem = NULL; + parser->m_unknownEncodingConvert = NULL; parser->m_unknownEncodingRelease = NULL; parser->m_unknownEncodingData = NULL; - parser->m_parentParser = NULL; parser->m_parsingStatus.parsing = XML_INITIALIZED; + // Reentry can only be triggered inside m_processor calls + parser->m_reenter = XML_FALSE; + parser->m_handlerCallDepth = 0; #ifdef XML_DTD parser->m_isParamEntity = XML_FALSE; parser->m_useForeignDTD = XML_FALSE; parser->m_paramEntityParsing = XML_PARAM_ENTITY_PARSING_NEVER; #endif - parser->m_hash_secret_salt = 0; + parser->m_hash_secret_salt_128.k[0] = 0; + parser->m_hash_secret_salt_128.k[1] = 0; + parser->m_hash_secret_salt_set = XML_FALSE; #if XML_GE == 1 memset(&parser->m_accounting, 0, sizeof(ACCOUNTING)); @@ -1272,7 +1570,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { } /* moves list of bindings to m_freeBindingList */ -static void FASTCALL +static void moveToFreeBindingList(XML_Parser parser, BINDING *bindings) { while (bindings) { BINDING *b = bindings; @@ -1282,16 +1580,30 @@ moveToFreeBindingList(XML_Parser parser, BINDING *bindings) { } } +/* Moves a list of entities onto the start of another list. */ +static void +moveEntityList(OPEN_INTERNAL_ENTITY **dst, OPEN_INTERNAL_ENTITY **src) { + for (OPEN_INTERNAL_ENTITY *head = *src; head != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = head; + head = head->next; + openEntity->next = *dst; + *dst = openEntity; + } +} + XML_Bool XMLCALL XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { TAG *tStk; - OPEN_INTERNAL_ENTITY *openEntityList; - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return XML_FALSE; if (parser->m_parentParser) return XML_FALSE; + // The application-defined release callback may access the parser, so it + // must run before any parser state is freed. + if (parser->m_unknownEncodingRelease) + callUnknownEncodingRelease(parser); /* move m_tagStack to m_freeTagList */ tStk = parser->m_tagStack; while (tStk) { @@ -1302,27 +1614,42 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { tag->bindings = NULL; parser->m_freeTagList = tag; } - /* move m_openInternalEntities to m_freeInternalEntities */ - openEntityList = parser->m_openInternalEntities; - while (openEntityList) { - OPEN_INTERNAL_ENTITY *openEntity = openEntityList; - openEntityList = openEntity->next; - openEntity->next = parser->m_freeInternalEntities; - parser->m_freeInternalEntities = openEntity; - } + /* move m_openInternalEntities to m_freeEntities */ + moveEntityList(&parser->m_freeEntities, &parser->m_openInternalEntities); + /* move m_openAttributeEntities to m_freeEntities (i.e. same task but for + * attributes) */ + moveEntityList(&parser->m_freeEntities, &parser->m_openAttributeEntities); + /* move m_openValueEntities to m_freeEntities (i.e. same task but for value + * entities) */ + moveEntityList(&parser->m_freeEntities, &parser->m_openValueEntities); moveToFreeBindingList(parser, parser->m_inheritedBindings); FREE(parser, parser->m_unknownEncodingMem); - if (parser->m_unknownEncodingRelease) - parser->m_unknownEncodingRelease(parser->m_unknownEncodingData); poolClear(&parser->m_tempPool); poolClear(&parser->m_temp2Pool); FREE(parser, (void *)parser->m_protocolEncodingName); parser->m_protocolEncodingName = NULL; parserInit(parser, encodingName); - dtdReset(parser->m_dtd, &parser->m_mem); + dtdReset(parser->m_dtd, parser); + if (encodingName && ! parser->m_protocolEncodingName) { + parser->m_errorCode = XML_ERROR_NO_MEMORY; + return XML_FALSE; + } return XML_TRUE; } +static XML_Bool +parserBusy(XML_Parser parser) { + switch (parser->m_parsingStatus.parsing) { + case XML_PARSING: + case XML_SUSPENDED: + return XML_TRUE; + case XML_INITIALIZED: + case XML_FINISHED: + default: + return XML_FALSE; + } +} + enum XML_Status XMLCALL XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName) { if (parser == NULL) @@ -1331,8 +1658,7 @@ XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName) { XXX There's no way for the caller to determine which of the XXX possible error cases caused the XML_STATUS_ERROR return. */ - if (parser->m_parsingStatus.parsing == XML_PARSING - || parser->m_parsingStatus.parsing == XML_SUSPENDED) + if (parserBusy(parser)) return XML_STATUS_ERROR; /* Get rid of any previous encoding name */ @@ -1343,7 +1669,7 @@ XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName) { parser->m_protocolEncodingName = NULL; else { /* Copy the new encoding name into allocated memory */ - parser->m_protocolEncodingName = copyString(encodingName, &(parser->m_mem)); + parser->m_protocolEncodingName = copyString(encodingName, parser); if (! parser->m_protocolEncodingName) return XML_STATUS_ERROR; } @@ -1372,6 +1698,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, XML_ExternalEntityRefHandler oldExternalEntityRefHandler; XML_SkippedEntityHandler oldSkippedEntityHandler; XML_UnknownEncodingHandler oldUnknownEncodingHandler; + void *oldUnknownEncodingHandlerData; XML_ElementDeclHandler oldElementDeclHandler; XML_AttlistDeclHandler oldAttlistDeclHandler; XML_EntityDeclHandler oldEntityDeclHandler; @@ -1392,7 +1719,8 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, from hash tables associated with either parser without us having to worry which hash secrets each table has. */ - unsigned long oldhash_secret_salt; + struct sipkey oldhash_secret_salt_128; + XML_Bool oldhash_secret_salt_set; XML_Bool oldReparseDeferralEnabled; /* Validate the oldParser parameter before we pull everything out of it */ @@ -1417,6 +1745,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, oldExternalEntityRefHandler = parser->m_externalEntityRefHandler; oldSkippedEntityHandler = parser->m_skippedEntityHandler; oldUnknownEncodingHandler = parser->m_unknownEncodingHandler; + oldUnknownEncodingHandlerData = parser->m_unknownEncodingHandlerData; oldElementDeclHandler = parser->m_elementDeclHandler; oldAttlistDeclHandler = parser->m_attlistDeclHandler; oldEntityDeclHandler = parser->m_entityDeclHandler; @@ -1437,7 +1766,8 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, from hash tables associated with either parser without us having to worry which hash secrets each table has. */ - oldhash_secret_salt = parser->m_hash_secret_salt; + oldhash_secret_salt_128 = parser->m_hash_secret_salt_128; + oldhash_secret_salt_set = parser->m_hash_secret_salt_set; oldReparseDeferralEnabled = parser->m_reparseDeferralEnabled; #ifdef XML_DTD @@ -1445,16 +1775,12 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, newDtd = oldDtd; #endif /* XML_DTD */ - /* Note that the magical uses of the pre-processor to make field - access look more like C++ require that `parser' be overwritten - here. This makes this function more painful to follow than it - would be otherwise. - */ if (parser->m_ns) { XML_Char tmp[2] = {parser->m_namespaceSeparator, 0}; - parser = parserCreate(encodingName, &parser->m_mem, tmp, newDtd); + parser = parserCreate(encodingName, &parser->m_mem, tmp, newDtd, oldParser); } else { - parser = parserCreate(encodingName, &parser->m_mem, NULL, newDtd); + parser + = parserCreate(encodingName, &parser->m_mem, NULL, newDtd, oldParser); } if (! parser) @@ -1476,6 +1802,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, parser->m_externalEntityRefHandler = oldExternalEntityRefHandler; parser->m_skippedEntityHandler = oldSkippedEntityHandler; parser->m_unknownEncodingHandler = oldUnknownEncodingHandler; + parser->m_unknownEncodingHandlerData = oldUnknownEncodingHandlerData; parser->m_elementDeclHandler = oldElementDeclHandler; parser->m_attlistDeclHandler = oldAttlistDeclHandler; parser->m_entityDeclHandler = oldEntityDeclHandler; @@ -1490,7 +1817,8 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, parser->m_externalEntityRefHandlerArg = oldExternalEntityRefHandlerArg; parser->m_defaultExpandInternalEntities = oldDefaultExpandInternalEntities; parser->m_ns_triplets = oldns_triplets; - parser->m_hash_secret_salt = oldhash_secret_salt; + parser->m_hash_secret_salt_128 = oldhash_secret_salt_128; + parser->m_hash_secret_salt_set = oldhash_secret_salt_set; parser->m_reparseDeferralEnabled = oldReparseDeferralEnabled; parser->m_parentParser = oldParser; #ifdef XML_DTD @@ -1498,7 +1826,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, parser->m_prologState.inEntityValue = oldInEntityValue; if (context) { #endif /* XML_DTD */ - if (! dtdCopy(oldParser, parser->m_dtd, oldDtd, &parser->m_mem) + if (! dtdCopy(oldParser, parser->m_dtd, oldDtd, parser) || ! setContext(parser, context)) { XML_ParserFree(parser); return NULL; @@ -1521,7 +1849,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, return parser; } -static void FASTCALL +static void destroyBindings(BINDING *bindings, XML_Parser parser) { for (;;) { BINDING *b = bindings; @@ -1536,9 +1864,12 @@ destroyBindings(BINDING *bindings, XML_Parser parser) { void XMLCALL XML_ParserFree(XML_Parser parser) { TAG *tagList; - OPEN_INTERNAL_ENTITY *entityList; - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return; + // The application-defined release callback may access the parser, so it + // must run before any parser state is freed. + if (parser->m_unknownEncodingRelease) + callUnknownEncodingRelease(parser); /* free m_tagStack and m_freeTagList */ tagList = parser->m_tagStack; for (;;) { @@ -1551,25 +1882,39 @@ XML_ParserFree(XML_Parser parser) { } p = tagList; tagList = tagList->parent; - FREE(parser, p->buf); + FREE(parser, p->buf.raw); destroyBindings(p->bindings, parser); FREE(parser, p); } - /* free m_openInternalEntities and m_freeInternalEntities */ - entityList = parser->m_openInternalEntities; - for (;;) { - OPEN_INTERNAL_ENTITY *openEntity; - if (entityList == NULL) { - if (parser->m_freeInternalEntities == NULL) - break; - entityList = parser->m_freeInternalEntities; - parser->m_freeInternalEntities = NULL; - } - openEntity = entityList; + /* free m_openInternalEntities */ + for (OPEN_INTERNAL_ENTITY *entityList = parser->m_openInternalEntities; + entityList != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = entityList; entityList = entityList->next; FREE(parser, openEntity); } - + /* free m_openAttributeEntities */ + for (OPEN_INTERNAL_ENTITY *entityList = parser->m_openAttributeEntities; + entityList != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = entityList; + entityList = entityList->next; + FREE(parser, openEntity); + } + /* free m_openValueEntities */ + for (OPEN_INTERNAL_ENTITY *entityList = parser->m_openValueEntities; + entityList != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = entityList; + entityList = entityList->next; + FREE(parser, openEntity); + } + /* free m_freeEntities */ + for (OPEN_INTERNAL_ENTITY *entityList = parser->m_freeEntities; + entityList != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = entityList; + entityList = entityList->next; + FREE(parser, openEntity); + } + parser->m_freeEntities = NULL; destroyBindings(parser->m_freeBindingList, parser); destroyBindings(parser->m_inheritedBindings, parser); poolDestroy(&parser->m_tempPool); @@ -1583,19 +1928,16 @@ XML_ParserFree(XML_Parser parser) { #else if (parser->m_dtd) #endif /* XML_DTD */ - dtdDestroy(parser->m_dtd, (XML_Bool)! parser->m_parentParser, - &parser->m_mem); - FREE(parser, (void *)parser->m_atts); -#ifdef XML_ATTR_INFO - FREE(parser, (void *)parser->m_attInfo); -#endif + dtdDestroy(parser->m_dtd, (XML_Bool)! parser->m_parentParser, parser); + FREE(parser, parser->m_atts); FREE(parser, parser->m_groupConnector); - FREE(parser, parser->m_buffer); + // NOTE: We are avoiding FREE(..) here because parser->m_buffer + // is not being allocated with MALLOC(..) but with plain + // .malloc_fcn(..). + parser->m_mem.free_fcn(parser->m_buffer); FREE(parser, parser->m_dataBuf); FREE(parser, parser->m_nsAtts); FREE(parser, parser->m_unknownEncodingMem); - if (parser->m_unknownEncodingRelease) - parser->m_unknownEncodingRelease(parser->m_unknownEncodingData); FREE(parser, parser); } @@ -1611,8 +1953,7 @@ XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD) { return XML_ERROR_INVALID_ARGUMENT; #ifdef XML_DTD /* block after XML_Parse()/XML_ParseBuffer() has been called */ - if (parser->m_parsingStatus.parsing == XML_PARSING - || parser->m_parsingStatus.parsing == XML_SUSPENDED) + if (parserBusy(parser)) return XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING; parser->m_useForeignDTD = useDTD; return XML_ERROR_NONE; @@ -1627,8 +1968,7 @@ XML_SetReturnNSTriplet(XML_Parser parser, int do_nst) { if (parser == NULL) return; /* block after XML_Parse()/XML_ParseBuffer() has been called */ - if (parser->m_parsingStatus.parsing == XML_PARSING - || parser->m_parsingStatus.parsing == XML_SUSPENDED) + if (parserBusy(parser)) return; parser->m_ns_triplets = do_nst ? XML_TRUE : XML_FALSE; } @@ -1678,15 +2018,6 @@ XML_GetIdAttributeIndex(XML_Parser parser) { return parser->m_idAttIndex; } -#ifdef XML_ATTR_INFO -const XML_AttrInfo *XMLCALL -XML_GetAttributeInfo(XML_Parser parser) { - if (parser == NULL) - return NULL; - return parser->m_attInfo; -} -#endif - void XMLCALL XML_SetElementHandler(XML_Parser parser, XML_StartElementHandler start, XML_EndElementHandler end) { @@ -1897,8 +2228,7 @@ XML_SetParamEntityParsing(XML_Parser parser, if (parser == NULL) return 0; /* block after XML_Parse()/XML_ParseBuffer() has been called */ - if (parser->m_parsingStatus.parsing == XML_PARSING - || parser->m_parsingStatus.parsing == XML_SUSPENDED) + if (parserBusy(parser)) return 0; #ifdef XML_DTD parser->m_paramEntityParsing = peParsing; @@ -1908,20 +2238,65 @@ XML_SetParamEntityParsing(XML_Parser parser, #endif } -int XMLCALL -XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) { +/* The body of XML_SetHashSalt, so that Expat's own tests can reach it + without tripping the deprecation of the public function. */ +XML_NONTESTING_STATIC int +xmlSetHashSalt(XML_Parser parser, unsigned long hash_salt) { if (parser == NULL) return 0; - if (parser->m_parentParser) - return XML_SetHashSalt(parser->m_parentParser, hash_salt); + + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(! rootParser->m_parentParser); + /* block after XML_Parse()/XML_ParseBuffer() has been called */ - if (parser->m_parsingStatus.parsing == XML_PARSING - || parser->m_parsingStatus.parsing == XML_SUSPENDED) + if (parserBusy(rootParser)) return 0; - parser->m_hash_secret_salt = hash_salt; + + rootParser->m_hash_secret_salt_128.k[0] = 0; + rootParser->m_hash_secret_salt_128.k[1] = hash_salt; + + if (hash_salt != 0) { // to remain backwards compatible + rootParser->m_hash_secret_salt_set = XML_TRUE; + + if (sizeof(unsigned long) == 4) + ENTROPY_DEBUG("explicit(4)", rootParser->m_hash_secret_salt_128); + else + ENTROPY_DEBUG("explicit(8)", rootParser->m_hash_secret_salt_128); + } + return 1; } +// DEPRECATED since Expat 2.8.0. +int XMLCALL +XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) { + return xmlSetHashSalt(parser, hash_salt); +} + +XML_Bool XMLCALL +XML_SetHashSalt16Bytes(XML_Parser parser, const uint8_t entropy[16]) { + if (parser == NULL) + return XML_FALSE; + + if (entropy == NULL) + return XML_FALSE; + + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(! rootParser->m_parentParser); + + /* block after XML_Parse()/XML_ParseBuffer() has been called */ + if (parserBusy(rootParser)) + return XML_FALSE; + + sip_tokey(&(rootParser->m_hash_secret_salt_128), entropy); + + rootParser->m_hash_secret_salt_set = XML_TRUE; + + ENTROPY_DEBUG("explicit(16)", rootParser->m_hash_secret_salt_128); + + return XML_TRUE; +} + enum XML_Status XMLCALL XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { if ((parser == NULL) || (len < 0) || ((s == NULL) && (len != 0))) { @@ -1929,6 +2304,8 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT; return XML_STATUS_ERROR; } + if (isCalledFromInsideHandler(parser)) + return XML_STATUS_ERROR; switch (parser->m_parsingStatus.parsing) { case XML_SUSPENDED: parser->m_errorCode = XML_ERROR_SUSPENDED; @@ -1941,7 +2318,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return XML_STATUS_ERROR; } - /* fall through */ + EXPAT_FALLTHROUGH; default: parser->m_parsingStatus.parsing = XML_PARSING; } @@ -1952,7 +2329,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { int nLeftOver; enum XML_Status result; /* Detect overflow (a+b > MAX <==> b > MAX-a) */ - if ((XML_Size)len > ((XML_Size)-1) / 2 - parser->m_parseEndByteIndex) { + if ((uint64_t)len > UINT64_MAX - parser->m_parseEndByteIndex) { parser->m_errorCode = XML_ERROR_NO_MEMORY; parser->m_eventPtr = parser->m_eventEndPtr = NULL; parser->m_processor = errorProcessor; @@ -1983,7 +2360,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_parsingStatus.parsing = XML_FINISHED; return XML_STATUS_OK; } - /* fall through */ + EXPAT_FALLTHROUGH; default: result = XML_STATUS_OK; } @@ -2038,7 +2415,7 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { const char *start; enum XML_Status result = XML_STATUS_OK; - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return XML_STATUS_ERROR; if (len < 0) { @@ -2064,11 +2441,24 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return XML_STATUS_ERROR; } - /* fall through */ + EXPAT_FALLTHROUGH; default: parser->m_parsingStatus.parsing = XML_PARSING; } + if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd)) { + parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT; + return XML_STATUS_ERROR; + } + + // Detect and avoid integer overflow + if ((uint64_t)len > UINT64_MAX - parser->m_parseEndByteIndex) { + parser->m_errorCode = XML_ERROR_NO_MEMORY; + parser->m_eventPtr = parser->m_eventEndPtr = NULL; + parser->m_processor = errorProcessor; + return XML_STATUS_ERROR; + } + start = parser->m_bufferPtr; parser->m_positionPtr = start; parser->m_bufferEnd += len; @@ -2094,6 +2484,7 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { parser->m_parsingStatus.parsing = XML_FINISHED; return result; } + break; default:; /* should not happen */ } } @@ -2104,9 +2495,32 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { return result; } +/* Modifies `parser`’s buffer to be backed by `newBuf`. */ +static void +setParserBuffer(XML_Parser parser, char *newBuf, int newBufSize, int keep) { + parser->m_bufferLim = newBuf + newBufSize; + if (parser->m_bufferPtr) { + const int parsing + = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr); + memcpy(newBuf, parser->m_bufferPtr - keep, parsing + keep); + // NOTE: We are avoiding FREE(..) here because parser->m_buffer + // is not being allocated with MALLOC(..) but with plain + // .malloc_fcn(..). + parser->m_mem.free_fcn(parser->m_buffer); + parser->m_buffer = newBuf; + parser->m_bufferEnd = newBuf + parsing + keep; + parser->m_bufferPtr = newBuf + keep; + } else { + /* This must be a brand new buffer with no data in it yet */ + parser->m_buffer = newBuf; + parser->m_bufferEnd = newBuf; + parser->m_bufferPtr = newBuf; + } +} + void *XMLCALL XML_GetBuffer(XML_Parser parser, int len) { - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return NULL; if (len < 0) { parser->m_errorCode = XML_ERROR_NO_MEMORY; @@ -2127,9 +2541,6 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_lastBufferRequestSize = len; if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd) || parser->m_buffer == NULL) { -#if XML_CONTEXT_BYTES > 0 - int keep; -#endif /* XML_CONTEXT_BYTES > 0 */ /* Do not invoke signed arithmetic overflow: */ int neededSize = (int)((unsigned)len + (unsigned)EXPAT_SAFE_PTR_DIFF( @@ -2139,7 +2550,9 @@ XML_GetBuffer(XML_Parser parser, int len) { return NULL; } #if XML_CONTEXT_BYTES > 0 - keep = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer); + const int parsed + = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer); + int keep = parsed; if (keep > XML_CONTEXT_BYTES) keep = XML_CONTEXT_BYTES; /* Detect and prevent integer overflow */ @@ -2147,16 +2560,16 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return NULL; } - neededSize += keep; +#else + int keep = 0; #endif /* XML_CONTEXT_BYTES > 0 */ + neededSize += keep; if (parser->m_buffer && parser->m_bufferPtr && neededSize <= EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer)) { #if XML_CONTEXT_BYTES > 0 - if (keep < EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer)) { - int offset - = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer) - - keep; + if (keep < parsed) { + int offset = parsed - keep; /* The buffer pointers cannot be NULL here; we have at least some bytes * in the buffer */ memmove(parser->m_buffer, &parser->m_buffer[offset], @@ -2173,7 +2586,6 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_bufferPtr = parser->m_buffer; #endif /* XML_CONTEXT_BYTES > 0 */ } else { - char *newBuf; int bufferSize = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer); if (bufferSize == 0) @@ -2186,43 +2598,14 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return NULL; } - newBuf = (char *)MALLOC(parser, bufferSize); - if (newBuf == 0) { + // NOTE: We are avoiding MALLOC(..) here to leave limiting + // the input size to the application using Expat. + char *const newBuf = parser->m_mem.malloc_fcn(bufferSize); + if (newBuf == NULL) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return NULL; } - parser->m_bufferLim = newBuf + bufferSize; -#if XML_CONTEXT_BYTES > 0 - if (parser->m_bufferPtr) { - memcpy(newBuf, &parser->m_bufferPtr[-keep], - EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) - + keep); - FREE(parser, parser->m_buffer); - parser->m_buffer = newBuf; - parser->m_bufferEnd - = parser->m_buffer - + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) - + keep; - parser->m_bufferPtr = parser->m_buffer + keep; - } else { - /* This must be a brand new buffer with no data in it yet */ - parser->m_bufferEnd = newBuf; - parser->m_bufferPtr = parser->m_buffer = newBuf; - } -#else - if (parser->m_bufferPtr) { - memcpy(newBuf, parser->m_bufferPtr, - EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)); - FREE(parser, parser->m_buffer); - parser->m_bufferEnd - = newBuf - + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr); - } else { - /* This must be a brand new buffer with no data in it yet */ - parser->m_bufferEnd = newBuf; - } - parser->m_bufferPtr = parser->m_buffer = newBuf; -#endif /* XML_CONTEXT_BYTES > 0 */ + setParserBuffer(parser, newBuf, bufferSize, keep); } parser->m_eventPtr = parser->m_eventEndPtr = NULL; parser->m_positionPtr = NULL; @@ -2230,6 +2613,11 @@ XML_GetBuffer(XML_Parser parser, int len) { return parser->m_bufferEnd; } +static void +triggerReenter(XML_Parser parser) { + parser->m_reenter = XML_TRUE; +} + enum XML_Status XMLCALL XML_StopParser(XML_Parser parser, XML_Bool resumable) { if (parser == NULL) @@ -2270,7 +2658,7 @@ enum XML_Status XMLCALL XML_ResumeParser(XML_Parser parser) { enum XML_Status result = XML_STATUS_OK; - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return XML_STATUS_ERROR; if (parser->m_parsingStatus.parsing != XML_SUSPENDED) { parser->m_errorCode = XML_ERROR_NOT_SUSPENDED; @@ -2296,6 +2684,7 @@ XML_ResumeParser(XML_Parser parser) { parser->m_parsingStatus.parsing = XML_FINISHED; return result; } + break; default:; } } @@ -2321,35 +2710,57 @@ XML_GetErrorCode(XML_Parser parser) { return parser->m_errorCode; } -XML_Index XMLCALL -XML_GetCurrentByteIndex(XML_Parser parser) { +int64_t XMLCALL +XML_GetCurrentByteIndex64(XML_Parser parser) { if (parser == NULL) return -1; - if (parser->m_eventPtr) - return (XML_Index)(parser->m_parseEndByteIndex - - (parser->m_parseEndPtr - parser->m_eventPtr)); + if (parser->m_eventPtr) { + return (int64_t)(parser->m_parseEndByteIndex + - (parser->m_parseEndPtr - parser->m_eventPtr)); + } return -1; } -int XMLCALL -XML_GetCurrentByteCount(XML_Parser parser) { +// DEPRECATED since Expat 2.9.0. +XML_Index XMLCALL +XML_GetCurrentByteIndex(XML_Parser parser) { + // NOTE: XML_Index is known to wrap around for >2 GiB content + // on 32bit machines and 64bit Windows, unless (non-default and + // uncommon) XML_LARGE_SIZE is defined. + // That's a bug and it only lives on because we cannot break + // ABI compatibility of public API. + return (XML_Index)XML_GetCurrentByteIndex64(parser); +} + +uint64_t XMLCALL +XML_GetCurrentByteCount64(XML_Parser parser) { if (parser == NULL) return 0; - if (parser->m_eventEndPtr && parser->m_eventPtr) - return (int)(parser->m_eventEndPtr - parser->m_eventPtr); + if (parser->m_eventEndPtr && parser->m_eventPtr) { + return parser->m_eventEndPtr - parser->m_eventPtr; + } return 0; } +// DEPRECATED since Expat 2.9.0. +int XMLCALL +XML_GetCurrentByteCount(XML_Parser parser) { + // NOTE: int is known to wrap around for >2 GiB content. + // That's a bug and it only lives on because we cannot break + // ABI compatibility of public API. + return (int)XML_GetCurrentByteCount64(parser); +} + const char *XMLCALL -XML_GetInputContext(XML_Parser parser, int *offset, int *size) { +XML_GetInputContext64(XML_Parser parser, int64_t *offset, uint64_t *size) { #if XML_CONTEXT_BYTES > 0 if (parser == NULL) return NULL; if (parser->m_eventPtr && parser->m_buffer) { if (offset != NULL) - *offset = (int)(parser->m_eventPtr - parser->m_buffer); + *offset = parser->m_eventPtr - parser->m_buffer; if (size != NULL) - *size = (int)(parser->m_bufferEnd - parser->m_buffer); + *size = parser->m_bufferEnd - parser->m_buffer; return parser->m_buffer; } #else @@ -2357,11 +2768,44 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size) { (void)offset; (void)size; #endif /* XML_CONTEXT_BYTES > 0 */ - return (const char *)0; + return NULL; } -XML_Size XMLCALL -XML_GetCurrentLineNumber(XML_Parser parser) { +// DEPRECATED since Expat 2.9.0. +const char *XMLCALL +XML_GetInputContext(XML_Parser parser, int *offset, int *size) { +#if XML_CONTEXT_BYTES > 0 + if (parser == NULL) + return NULL; + + int64_t offset64; + uint64_t size64; + + const char *const buffer = XML_GetInputContext64(parser, &offset64, &size64); + + if (buffer == NULL) + return NULL; + + // NOTE: int is known to wrap around for >2 GiB content. + // That's a bug and it only lives on because we cannot break + // ABI compatibility of public API. + if (offset != NULL) + *offset = (int)offset64; + + if (size != NULL) + *size = (int)size64; + + return buffer; +#else + (void)parser; + (void)offset; + (void)size; +#endif /* XML_CONTEXT_BYTES > 0 */ + return NULL; +} + +uint64_t XMLCALL +XML_GetCurrentLineNumber64(XML_Parser parser) { if (parser == NULL) return 0; if (parser->m_eventPtr && parser->m_eventPtr >= parser->m_positionPtr) { @@ -2372,8 +2816,19 @@ XML_GetCurrentLineNumber(XML_Parser parser) { return parser->m_position.lineNumber + 1; } +// DEPRECATED since Expat 2.9.0. XML_Size XMLCALL -XML_GetCurrentColumnNumber(XML_Parser parser) { +XML_GetCurrentLineNumber(XML_Parser parser) { + // NOTE: XML_Size is known to wrap around for >4 GiB content + // on 32bit machines and 64bit Windows, unless (non-default and + // uncommon) XML_LARGE_SIZE is defined. + // That's a bug and it only lives on because we cannot break + // ABI compatibility of public API. + return (XML_Size)XML_GetCurrentLineNumber64(parser); +} + +uint64_t XMLCALL +XML_GetCurrentColumnNumber64(XML_Parser parser) { if (parser == NULL) return 0; if (parser->m_eventPtr && parser->m_eventPtr >= parser->m_positionPtr) { @@ -2384,35 +2839,61 @@ XML_GetCurrentColumnNumber(XML_Parser parser) { return parser->m_position.columnNumber; } +// DEPRECATED since Expat 2.9.0. +XML_Size XMLCALL +XML_GetCurrentColumnNumber(XML_Parser parser) { + // NOTE: XML_Size is known to wrap around for >4 GiB content + // on 32bit machines and 64bit Windows, unless (non-default and + // uncommon) XML_LARGE_SIZE is defined. + // That's a bug and it only lives on because we cannot break + // ABI compatibility of public API. + return (XML_Size)XML_GetCurrentColumnNumber64(parser); +} + void XMLCALL XML_FreeContentModel(XML_Parser parser, XML_Content *model) { - if (parser != NULL) - FREE(parser, model); + if (parser == NULL) + return; + + // NOTE: We are avoiding FREE(..) here because the content model + // has been created using plain .malloc_fcn(..) rather than MALLOC(..). + parser->m_mem.free_fcn(model); } void *XMLCALL XML_MemMalloc(XML_Parser parser, size_t size) { if (parser == NULL) return NULL; - return MALLOC(parser, size); + + // NOTE: We are avoiding MALLOC(..) here to not include + // user allocations with allocation tracking and limiting. + return parser->m_mem.malloc_fcn(size); } void *XMLCALL XML_MemRealloc(XML_Parser parser, void *ptr, size_t size) { if (parser == NULL) return NULL; - return REALLOC(parser, ptr, size); + + // NOTE: We are avoiding REALLOC(..) here to not include + // user allocations with allocation tracking and limiting. + return parser->m_mem.realloc_fcn(ptr, size); } void XMLCALL XML_MemFree(XML_Parser parser, void *ptr) { - if (parser != NULL) - FREE(parser, ptr); + if (parser == NULL) + return; + + // NOTE: We are avoiding FREE(..) here because XML_MemMalloc and + // XML_MemRealloc are not using MALLOC(..) and REALLOC(..) + // but plain .malloc_fcn(..) and .realloc_fcn(..), internally. + parser->m_mem.free_fcn(ptr); } void XMLCALL XML_DefaultCurrent(XML_Parser parser) { - if (parser == NULL) + if (parser == NULL || ! isCalledFromInsideHandler(parser)) return; if (parser->m_defaultHandler) { if (parser->m_openInternalEntities) @@ -2583,67 +3064,326 @@ XML_GetFeatureList(void) { {XML_FEATURE_CONTEXT_BYTES, XML_L("XML_CONTEXT_BYTES"), XML_CONTEXT_BYTES}, #endif -#ifdef XML_MIN_SIZE - {XML_FEATURE_MIN_SIZE, XML_L("XML_MIN_SIZE"), 0}, -#endif #ifdef XML_NS {XML_FEATURE_NS, XML_L("XML_NS"), 0}, #endif #ifdef XML_LARGE_SIZE {XML_FEATURE_LARGE_SIZE, XML_L("XML_LARGE_SIZE"), 0}, #endif -#ifdef XML_ATTR_INFO - {XML_FEATURE_ATTR_INFO, XML_L("XML_ATTR_INFO"), 0}, +#if XML_GE == 1 + /* Added in Expat 2.4.0 for XML_DTD defined and + * added in Expat 2.6.0 for XML_GE == 1. */ + {XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT, + XML_L("XML_BLAP_MAX_AMP"), + (long int) + EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT}, + {XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT, + XML_L("XML_BLAP_ACT_THRES"), + EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT}, + /* Added in Expat 2.6.0. */ + {XML_FEATURE_GE, XML_L("XML_GE"), 0}, + /* Added in Expat 2.7.2. */ + {XML_FEATURE_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT, + XML_L("XML_AT_MAX_AMP"), + (long int)EXPAT_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT}, + {XML_FEATURE_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT, + XML_L("XML_AT_ACT_THRES"), + (long int)EXPAT_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT}, +#endif + {XML_FEATURE_END, NULL, 0}}; + + return features; +} + +#if XML_GE == 1 +static enum XML_Prop_Error +setBillionLaughsAttackProtectionMaximumAmplification( + XML_Parser parser, float maximumAmplificationFactor) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (parser->m_parentParser != NULL) + return XML_PROP_ERROR_PARSER_NOT_ROOT; + + if (isnan(maximumAmplificationFactor) || (maximumAmplificationFactor < 1.0f)) + return XML_PROP_ERROR_INVALID_VALUE; + + parser->m_accounting.maximumAmplificationFactor = maximumAmplificationFactor; + + return XML_PROP_ERROR_NONE; +} + +XML_Bool XMLCALL +XML_SetBillionLaughsAttackProtectionMaximumAmplification( + XML_Parser parser, float maximumAmplificationFactor) { + return (setBillionLaughsAttackProtectionMaximumAmplification( + parser, maximumAmplificationFactor) + == XML_PROP_ERROR_NONE) + ? XML_TRUE + : XML_FALSE; +} + +static enum XML_Prop_Error +setBillionLaughsAttackProtectionActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (parser->m_parentParser != NULL) + return XML_PROP_ERROR_PARSER_NOT_ROOT; + + parser->m_accounting.activationThresholdBytes = activationThresholdBytes; + + return XML_PROP_ERROR_NONE; +} + +XML_Bool XMLCALL +XML_SetBillionLaughsAttackProtectionActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes) { + return (setBillionLaughsAttackProtectionActivationThreshold( + parser, activationThresholdBytes) + == XML_PROP_ERROR_NONE) + ? XML_TRUE + : XML_FALSE; +} + +static enum XML_Prop_Error +setAllocTrackerMaximumAmplification(XML_Parser parser, + float maximumAmplificationFactor) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (parser->m_parentParser != NULL) + return XML_PROP_ERROR_PARSER_NOT_ROOT; + + if (isnan(maximumAmplificationFactor) || (maximumAmplificationFactor < 1.0f)) + return XML_PROP_ERROR_INVALID_VALUE; + + parser->m_alloc_tracker.maximumAmplificationFactor + = maximumAmplificationFactor; + + return XML_PROP_ERROR_NONE; +} + +XML_Bool XMLCALL +XML_SetAllocTrackerMaximumAmplification(XML_Parser parser, + float maximumAmplificationFactor) { + return (setAllocTrackerMaximumAmplification(parser, + maximumAmplificationFactor) + == XML_PROP_ERROR_NONE) + ? XML_TRUE + : XML_FALSE; +} + +static enum XML_Prop_Error +setAllocTrackerActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (parser->m_parentParser != NULL) + return XML_PROP_ERROR_PARSER_NOT_ROOT; + + parser->m_alloc_tracker.activationThresholdBytes = activationThresholdBytes; + + return XML_PROP_ERROR_NONE; +} + +XML_Bool XMLCALL +XML_SetAllocTrackerActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes) { + return (setAllocTrackerActivationThreshold(parser, activationThresholdBytes) + == XML_PROP_ERROR_NONE) + ? XML_TRUE + : XML_FALSE; +} +#endif /* XML_GE == 1 */ + +static enum XML_Prop_Error +setReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (enabled != XML_TRUE && enabled != XML_FALSE) + return XML_PROP_ERROR_INVALID_VALUE; + + parser->m_reparseDeferralEnabled = enabled; + + return XML_PROP_ERROR_NONE; +} + +XML_Bool XMLCALL +XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled) { + return (setReparseDeferralEnabled(parser, enabled) == XML_PROP_ERROR_NONE) + ? XML_TRUE + : XML_FALSE; +} + +enum XML_Prop_Error XMLCALL +XML_SetPropertyBool(XML_Parser parser, enum XML_Parser_Property property, + XML_Bool value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + switch (property) { +#if XML_GE == 1 + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: + return XML_PROP_ERROR_INVALID_TYPE; +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + return setReparseDeferralEnabled(parser, value); + default: + return XML_PROP_ERROR_INVALID_KEY; + } + + assert(0 && "considered unreachable"); +} + +enum XML_Prop_Error XMLCALL +XML_SetPropertyDouble(XML_Parser parser, enum XML_Parser_Property property, + double value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + +#if XML_GE == 0 + UNUSED_P(value); #endif + + switch (property) { #if XML_GE == 1 - /* Added in Expat 2.4.0 for XML_DTD defined and - * added in Expat 2.6.0 for XML_GE == 1. */ - {XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT, - XML_L("XML_BLAP_MAX_AMP"), - (long int) - EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT}, - {XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT, - XML_L("XML_BLAP_ACT_THRES"), - EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT}, - /* Added in Expat 2.6.0. */ - {XML_FEATURE_GE, XML_L("XML_GE"), 0}, -#endif - {XML_FEATURE_END, NULL, 0}}; + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + return setAllocTrackerMaximumAmplification(parser, (float)value); + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: + return setBillionLaughsAttackProtectionMaximumAmplification(parser, + (float)value); + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + return XML_PROP_ERROR_INVALID_TYPE; + default: + return XML_PROP_ERROR_INVALID_KEY; + } - return features; + assert(0 && "considered unreachable"); } +enum XML_Prop_Error XMLCALL +XML_SetPropertyUInt64(XML_Parser parser, enum XML_Parser_Property property, + uint64_t value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + +#if XML_GE == 0 + UNUSED_P(value); +#endif + + switch (property) { #if XML_GE == 1 -XML_Bool XMLCALL -XML_SetBillionLaughsAttackProtectionMaximumAmplification( - XML_Parser parser, float maximumAmplificationFactor) { - if ((parser == NULL) || (parser->m_parentParser != NULL) - || isnan(maximumAmplificationFactor) - || (maximumAmplificationFactor < 1.0f)) { - return XML_FALSE; + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + return setAllocTrackerActivationThreshold(parser, + (unsigned long long)value); + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: + return setBillionLaughsAttackProtectionActivationThreshold( + parser, (unsigned long long)value); + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + return XML_PROP_ERROR_INVALID_TYPE; + default: + return XML_PROP_ERROR_INVALID_KEY; } - parser->m_accounting.maximumAmplificationFactor = maximumAmplificationFactor; - return XML_TRUE; + + assert(0 && "considered unreachable"); } -XML_Bool XMLCALL -XML_SetBillionLaughsAttackProtectionActivationThreshold( - XML_Parser parser, unsigned long long activationThresholdBytes) { - if ((parser == NULL) || (parser->m_parentParser != NULL)) { - return XML_FALSE; +enum XML_Prop_Error XMLCALL +XML_GetPropertyBool(XML_Parser parser, enum XML_Parser_Property property, + XML_Bool *value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (value == NULL) + return XML_PROP_ERROR_INVALID_VALUE; + + switch (property) { +#if XML_GE == 1 + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: + return XML_PROP_ERROR_INVALID_TYPE; +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + *value = parser->m_reparseDeferralEnabled; + break; + default: + return XML_PROP_ERROR_INVALID_KEY; } - parser->m_accounting.activationThresholdBytes = activationThresholdBytes; - return XML_TRUE; + + return XML_PROP_ERROR_NONE; } + +enum XML_Prop_Error XMLCALL +XML_GetPropertyDouble(XML_Parser parser, enum XML_Parser_Property property, + double *value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (value == NULL) + return XML_PROP_ERROR_INVALID_VALUE; + + switch (property) { +#if XML_GE == 1 + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: + *value = (double)parser->m_accounting.maximumAmplificationFactor; + break; + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + *value = (double)parser->m_alloc_tracker.maximumAmplificationFactor; + break; + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: #endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + return XML_PROP_ERROR_INVALID_TYPE; + default: + return XML_PROP_ERROR_INVALID_KEY; + } -XML_Bool XMLCALL -XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled) { - if (parser != NULL && (enabled == XML_TRUE || enabled == XML_FALSE)) { - parser->m_reparseDeferralEnabled = enabled; - return XML_TRUE; + return XML_PROP_ERROR_NONE; +} + +enum XML_Prop_Error XMLCALL +XML_GetPropertyUInt64(XML_Parser parser, enum XML_Parser_Property property, + uint64_t *value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (value == NULL) + return XML_PROP_ERROR_INVALID_VALUE; + + switch (property) { +#if XML_GE == 1 + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: + *value = parser->m_accounting.activationThresholdBytes; + break; + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + *value = parser->m_alloc_tracker.activationThresholdBytes; + break; + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + return XML_PROP_ERROR_INVALID_TYPE; + default: + return XML_PROP_ERROR_INVALID_KEY; } - return XML_FALSE; + + return XML_PROP_ERROR_NONE; } /* Initially tag->rawName always points into the parse buffer; @@ -2655,10 +3395,10 @@ static XML_Bool storeRawNames(XML_Parser parser) { TAG *tag = parser->m_tagStack; while (tag) { - int bufSize; - int nameLen = sizeof(XML_Char) * (tag->name.strLen + 1); + size_t bufSize; + size_t nameLen = sizeof(XML_Char) * (tag->name.strLen + 1); size_t rawNameLen; - char *rawNameBuf = tag->buf + nameLen; + char *rawNameBuf = tag->buf.raw + nameLen; /* Stop if already stored. Since m_tagStack is a stack, we can stop at the first entry that has already been copied; everything below it in the stack is already been accounted for in a @@ -2671,25 +3411,25 @@ storeRawNames(XML_Parser parser) { */ rawNameLen = ROUND_UP(tag->rawNameLength, sizeof(XML_Char)); /* Detect and prevent integer overflow. */ - if (rawNameLen > (size_t)INT_MAX - nameLen) + if (rawNameLen > SIZE_MAX - nameLen) return XML_FALSE; - bufSize = nameLen + (int)rawNameLen; - if (bufSize > tag->bufEnd - tag->buf) { - char *temp = (char *)REALLOC(parser, tag->buf, bufSize); + bufSize = nameLen + rawNameLen; + if (bufSize > (size_t)(tag->bufEnd - tag->buf.raw)) { + char *temp = REALLOC(parser, tag->buf.raw, bufSize); if (temp == NULL) return XML_FALSE; - /* if tag->name.str points to tag->buf (only when namespace + /* if tag->name.str points to tag->buf.str (only when namespace processing is off) then we have to update it */ - if (tag->name.str == (XML_Char *)tag->buf) + if (tag->name.str == tag->buf.str) tag->name.str = (XML_Char *)temp; /* if tag->name.localPart is set (when namespace processing is on) then update it as well, since it will always point into tag->buf */ if (tag->name.localPart) tag->name.localPart - = (XML_Char *)temp + (tag->name.localPart - (XML_Char *)tag->buf); - tag->buf = temp; + = (XML_Char *)temp + (tag->name.localPart - tag->buf.str); + tag->buf.raw = temp; tag->bufEnd = temp + bufSize; rawNameBuf = temp + nameLen; } @@ -2700,12 +3440,13 @@ storeRawNames(XML_Parser parser) { return XML_TRUE; } -static enum XML_Error PTRCALL +static enum XML_Error contentProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { enum XML_Error result = doContent( - parser, 0, parser->m_encoding, start, end, endPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_ACCOUNT_DIRECT); + parser, parser->m_parentParser ? 1 : 0, parser->m_encoding, start, end, + endPtr, (XML_Bool)! parser->m_parsingStatus.finalBuffer, + XML_ACCOUNT_DIRECT); if (result == XML_ERROR_NONE) { if (! storeRawNames(parser)) return XML_ERROR_NO_MEMORY; @@ -2713,7 +3454,7 @@ contentProcessor(XML_Parser parser, const char *start, const char *end, return result; } -static enum XML_Error PTRCALL +static enum XML_Error externalEntityInitProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { enum XML_Error result = initializeEncoding(parser); @@ -2723,7 +3464,7 @@ externalEntityInitProcessor(XML_Parser parser, const char *start, return externalEntityInitProcessor2(parser, start, end, endPtr); } -static enum XML_Error PTRCALL +static enum XML_Error externalEntityInitProcessor2(XML_Parser parser, const char *start, const char *end, const char **endPtr) { const char *next = start; /* XmlContentTok doesn't always set the last arg */ @@ -2768,7 +3509,7 @@ externalEntityInitProcessor2(XML_Parser parser, const char *start, return externalEntityInitProcessor3(parser, start, end, endPtr); } -static enum XML_Error PTRCALL +static enum XML_Error externalEntityInitProcessor3(XML_Parser parser, const char *start, const char *end, const char **endPtr) { int tok; @@ -2793,6 +3534,11 @@ externalEntityInitProcessor3(XML_Parser parser, const char *start, return XML_ERROR_NONE; case XML_FINISHED: return XML_ERROR_ABORTED; + case XML_PARSING: + if (parser->m_reenter) { + return XML_ERROR_UNEXPECTED_STATE; // LCOV_EXCL_LINE + } + EXPAT_FALLTHROUGH; default: start = next; } @@ -2815,7 +3561,7 @@ externalEntityInitProcessor3(XML_Parser parser, const char *start, return externalEntityContentProcessor(parser, start, end, endPtr); } -static enum XML_Error PTRCALL +static enum XML_Error externalEntityContentProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { enum XML_Error result @@ -2871,7 +3617,9 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, *eventEndPP = end; if (parser->m_characterDataHandler) { XML_Char c = 0xA; + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, &c, 1); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, end); /* We are at the end of the final buffer, should we check for @@ -2924,9 +3672,11 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, ((char *)&ch) + sizeof(XML_Char), __LINE__, XML_ACCOUNT_ENTITY_EXPANSION); #endif /* XML_GE == 1 */ - if (parser->m_characterDataHandler) + if (parser->m_characterDataHandler) { + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, &ch, 1); - else if (parser->m_defaultHandler) + afterHandler(parser); + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; } @@ -2946,9 +3696,11 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, else if (! entity->is_internal) return XML_ERROR_ENTITY_DECLARED_IN_PE; } else if (! entity) { - if (parser->m_skippedEntityHandler) + if (parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0); - else if (parser->m_defaultHandler) + afterHandler(parser); + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; } @@ -2959,26 +3711,31 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, if (entity->textPtr) { enum XML_Error result; if (! parser->m_defaultExpandInternalEntities) { - if (parser->m_skippedEntityHandler) + if (parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, entity->name, 0); - else if (parser->m_defaultHandler) + afterHandler(parser); + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; } - result = processInternalEntity(parser, entity, XML_FALSE); + result = processEntity(parser, entity, XML_FALSE, ENTITY_INTERNAL); if (result != XML_ERROR_NONE) return result; } else if (parser->m_externalEntityRefHandler) { const XML_Char *context; - entity->open = XML_TRUE; + entity->open = true; context = getContext(parser); - entity->open = XML_FALSE; + entity->open = false; if (! context) return XML_ERROR_NO_MEMORY; - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, context, entity->base, - entity->systemId, entity->publicId)) + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, context, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) return XML_ERROR_EXTERNAL_ENTITY_HANDLING; poolDiscard(&parser->m_tempPool); } else if (parser->m_defaultHandler) @@ -2986,7 +3743,6 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, break; } case XML_TOK_START_TAG_NO_ATTS: - /* fall through */ case XML_TOK_START_TAG_WITH_ATTS: { TAG *tag; enum XML_Error result; @@ -2995,15 +3751,15 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, tag = parser->m_freeTagList; parser->m_freeTagList = parser->m_freeTagList->parent; } else { - tag = (TAG *)MALLOC(parser, sizeof(TAG)); + tag = MALLOC(parser, sizeof(TAG)); if (! tag) return XML_ERROR_NO_MEMORY; - tag->buf = (char *)MALLOC(parser, INIT_TAG_BUF_SIZE); - if (! tag->buf) { + tag->buf.raw = MALLOC(parser, INIT_TAG_BUF_SIZE); + if (! tag->buf.raw) { FREE(parser, tag); return XML_ERROR_NO_MEMORY; } - tag->bufEnd = tag->buf + INIT_TAG_BUF_SIZE; + tag->bufEnd = tag->buf.raw + INIT_TAG_BUF_SIZE; } tag->bindings = NULL; tag->parent = parser->m_tagStack; @@ -3016,46 +3772,47 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, { const char *rawNameEnd = tag->rawName + tag->rawNameLength; const char *fromPtr = tag->rawName; - toPtr = (XML_Char *)tag->buf; + toPtr = tag->buf.str; for (;;) { - int bufSize; - int convLen; const enum XML_Convert_Result convert_res = XmlConvert(enc, &fromPtr, rawNameEnd, (ICHAR **)&toPtr, (ICHAR *)tag->bufEnd - 1); - convLen = (int)(toPtr - (XML_Char *)tag->buf); + const size_t convLen = (size_t)(toPtr - tag->buf.str); if ((fromPtr >= rawNameEnd) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) { tag->name.strLen = convLen; break; } - bufSize = (int)(tag->bufEnd - tag->buf) << 1; + if (SIZE_MAX / 2 < (size_t)(tag->bufEnd - tag->buf.raw)) + return XML_ERROR_NO_MEMORY; + const size_t bufSize = (size_t)(tag->bufEnd - tag->buf.raw) * 2; { - char *temp = (char *)REALLOC(parser, tag->buf, bufSize); + char *temp = REALLOC(parser, tag->buf.raw, bufSize); if (temp == NULL) return XML_ERROR_NO_MEMORY; - tag->buf = temp; + tag->buf.raw = temp; tag->bufEnd = temp + bufSize; toPtr = (XML_Char *)temp + convLen; } } } - tag->name.str = (XML_Char *)tag->buf; + tag->name.str = tag->buf.str; *toPtr = XML_T('\0'); result = storeAtts(parser, enc, s, &(tag->name), &(tag->bindings), account); if (result) return result; - if (parser->m_startElementHandler) + if (parser->m_startElementHandler) { + beforeHandler(parser); parser->m_startElementHandler(parser->m_handlerArg, tag->name.str, (const XML_Char **)parser->m_atts); - else if (parser->m_defaultHandler) + afterHandler(parser); + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); poolClear(&parser->m_tempPool); break; } case XML_TOK_EMPTY_ELEMENT_NO_ATTS: - /* fall through */ case XML_TOK_EMPTY_ELEMENT_WITH_ATTS: { const char *rawName = s + enc->minBytesPerChar; enum XML_Error result; @@ -3075,14 +3832,18 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, } poolFinish(&parser->m_tempPool); if (parser->m_startElementHandler) { + beforeHandler(parser); parser->m_startElementHandler(parser->m_handlerArg, name.str, (const XML_Char **)parser->m_atts); + afterHandler(parser); noElmHandlers = XML_FALSE; } if (parser->m_endElementHandler) { if (parser->m_startElementHandler) *eventPP = *eventEndPP; + beforeHandler(parser); parser->m_endElementHandler(parser->m_handlerArg, name.str); + afterHandler(parser); noElmHandlers = XML_FALSE; } if (noElmHandlers && parser->m_defaultHandler) @@ -3092,7 +3853,9 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, } if ((parser->m_tagLevel == 0) && (parser->m_parsingStatus.parsing != XML_FINISHED)) { - if (parser->m_parsingStatus.parsing == XML_SUSPENDED) + if (parser->m_parsingStatus.parsing == XML_SUSPENDED + || (parser->m_parsingStatus.parsing == XML_PARSING + && parser->m_reenter)) parser->m_processor = epilogProcessor; else return epilogProcessor(parser, next, end, nextPtr); @@ -3138,14 +3901,19 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, } *uri = XML_T('\0'); } + beforeHandler(parser); parser->m_endElementHandler(parser->m_handlerArg, tag->name.str); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); while (tag->bindings) { BINDING *b = tag->bindings; - if (parser->m_endNamespaceDeclHandler) + if (parser->m_endNamespaceDeclHandler) { + beforeHandler(parser); parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name); + afterHandler(parser); + } tag->bindings = tag->bindings->nextTagBinding; b->nextTagBinding = parser->m_freeBindingList; parser->m_freeBindingList = b; @@ -3153,7 +3921,9 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, } if ((parser->m_tagLevel == 0) && (parser->m_parsingStatus.parsing != XML_FINISHED)) { - if (parser->m_parsingStatus.parsing == XML_SUSPENDED) + if (parser->m_parsingStatus.parsing == XML_SUSPENDED + || (parser->m_parsingStatus.parsing == XML_PARSING + && parser->m_reenter)) parser->m_processor = epilogProcessor; else return epilogProcessor(parser, next, end, nextPtr); @@ -3166,8 +3936,10 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, return XML_ERROR_BAD_CHAR_REF; if (parser->m_characterDataHandler) { XML_Char buf[XML_ENCODE_MAX]; + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, buf, XmlEncode(n, (ICHAR *)buf)); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); } break; @@ -3176,32 +3948,38 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, case XML_TOK_DATA_NEWLINE: if (parser->m_characterDataHandler) { XML_Char c = 0xA; + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, &c, 1); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; case XML_TOK_CDATA_SECT_OPEN: { enum XML_Error result; - if (parser->m_startCdataSectionHandler) + if (parser->m_startCdataSectionHandler) { + beforeHandler(parser); parser->m_startCdataSectionHandler(parser->m_handlerArg); - /* BEGIN disabled code */ - /* Suppose you doing a transformation on a document that involves - changing only the character data. You set up a defaultHandler - and a characterDataHandler. The defaultHandler simply copies - characters through. The characterDataHandler does the - transformation and writes the characters out escaping them as - necessary. This case will fail to work if we leave out the - following two lines (because & and < inside CDATA sections will - be incorrectly escaped). - - However, now we have a start/endCdataSectionHandler, so it seems - easier to let the user deal with this. - */ - else if ((0) && parser->m_characterDataHandler) + afterHandler(parser); + /* BEGIN disabled code */ + /* Suppose you doing a transformation on a document that involves + changing only the character data. You set up a defaultHandler + and a characterDataHandler. The defaultHandler simply copies + characters through. The characterDataHandler does the + transformation and writes the characters out escaping them as + necessary. This case will fail to work if we leave out the + following two lines (because & and < inside CDATA sections will + be incorrectly escaped). + + However, now we have a start/endCdataSectionHandler, so it seems + easier to let the user deal with this. + */ + } else if ((0) && parser->m_characterDataHandler) { + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0); - /* END disabled code */ - else if (parser->m_defaultHandler) + afterHandler(parser); + /* END disabled code */ + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); result = doCdataSection(parser, enc, &next, end, nextPtr, haveMore, account); @@ -3221,13 +3999,18 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, if (MUST_CONVERT(enc, s)) { ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf; XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd); + beforeHandler(parser); parser->m_characterDataHandler( parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); - } else + afterHandler(parser); + } else { + beforeHandler(parser); parser->m_characterDataHandler( parser->m_handlerArg, (const XML_Char *)s, (int)((const XML_Char *)end - (const XML_Char *)s)); + afterHandler(parser); + } } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, end); /* We are at the end of the final buffer, should we check for @@ -3252,16 +4035,21 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, const enum XML_Convert_Result convert_res = XmlConvert( enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd); *eventEndPP = s; + beforeHandler(parser); charDataHandler(parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); + afterHandler(parser); if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; *eventPP = s; } - } else + } else { + beforeHandler(parser); charDataHandler(parser->m_handlerArg, (const XML_Char *)s, (int)((const XML_Char *)next - (const XML_Char *)s)); + afterHandler(parser); + } } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); } break; @@ -3286,14 +4074,22 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, break; /* LCOV_EXCL_STOP */ } - *eventPP = s = next; switch (parser->m_parsingStatus.parsing) { case XML_SUSPENDED: + *eventPP = next; *nextPtr = next; return XML_ERROR_NONE; case XML_FINISHED: + *eventPP = next; return XML_ERROR_ABORTED; + case XML_PARSING: + if (parser->m_reenter) { + *nextPtr = next; + return XML_ERROR_NONE; + } + EXPAT_FALLTHROUGH; default:; + *eventPP = s = next; } } /* not reached */ @@ -3311,8 +4107,11 @@ freeBindings(XML_Parser parser, BINDING *bindings) { /* m_startNamespaceDeclHandler will have been called for this * binding in addBindings(), so call the end handler now. */ - if (parser->m_endNamespaceDeclHandler) + if (parser->m_endNamespaceDeclHandler) { + beforeHandler(parser); parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name); + afterHandler(parser); + } bindings = bindings->nextTagBinding; b->nextTagBinding = parser->m_freeBindingList; @@ -3336,20 +4135,14 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, TAG_NAME *tagNamePtr, BINDING **bindingsPtr, enum XML_Account account) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ - ELEMENT_TYPE *elementType; - int nDefaultAtts; - const XML_Char **appAtts; /* the attribute list for the application */ int attIndex = 0; - int prefixLen; - int i; - int n; XML_Char *uri; int nPrefixes = 0; BINDING *binding; const XML_Char *localPart; /* lookup the element type name */ - elementType + ELEMENT_TYPE *elementType = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, tagNamePtr->str, 0); if (! elementType) { const XML_Char *name = poolCopyString(&dtd->pool, tagNamePtr->str); @@ -3359,98 +4152,68 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sizeof(ELEMENT_TYPE)); if (! elementType) return XML_ERROR_NO_MEMORY; + if (! elementType->defaultAttForName.parser) + hashTableInit(&(elementType->defaultAttForName), parser); if (parser->m_ns && ! setElementTypePrefix(parser, elementType)) return XML_ERROR_NO_MEMORY; } - nDefaultAtts = elementType->nDefaultAtts; + const size_t nDefaultAtts = elementType->nDefaultAtts; + + /* Detect and prevent integer overflow. */ + if (parser->m_attsSize > (size_t)INT_MAX) + return XML_ERROR_NO_MEMORY; /* get the attributes from the tokenizer */ - n = XmlGetAttributes(enc, attStr, parser->m_attsSize, parser->m_atts); + size_t n = (size_t)XmlGetAttributes(enc, attStr, (int)parser->m_attsSize, + parser->m_atts); /* Detect and prevent integer overflow */ - if (n > INT_MAX - nDefaultAtts) { + if (n > SIZE_MAX - nDefaultAtts) { return XML_ERROR_NO_MEMORY; } if (n + nDefaultAtts > parser->m_attsSize) { - int oldAttsSize = parser->m_attsSize; - ATTRIBUTE *temp; -#ifdef XML_ATTR_INFO - XML_AttrInfo *temp2; -#endif + size_t oldAttsSize = parser->m_attsSize; /* Detect and prevent integer overflow */ - if ((nDefaultAtts > INT_MAX - INIT_ATTS_SIZE) - || (n > INT_MAX - (nDefaultAtts + INIT_ATTS_SIZE))) { + if ((nDefaultAtts > SIZE_MAX - INIT_ATTS_SIZE) + || (n > SIZE_MAX - (nDefaultAtts + INIT_ATTS_SIZE))) { return XML_ERROR_NO_MEMORY; } parser->m_attsSize = n + nDefaultAtts + INIT_ATTS_SIZE; - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)parser->m_attsSize > (size_t)(-1) / sizeof(ATTRIBUTE)) { + /* Detect and prevent integer overflow. */ + if (parser->m_attsSize > SIZE_MAX / sizeof(ATTRIBUTE)) { parser->m_attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; } -#endif - temp = (ATTRIBUTE *)REALLOC(parser, (void *)parser->m_atts, - parser->m_attsSize * sizeof(ATTRIBUTE)); + ATTRIBUTE *const temp = REALLOC(parser, parser->m_atts, + parser->m_attsSize * sizeof(ATTRIBUTE)); if (temp == NULL) { parser->m_attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; } parser->m_atts = temp; -#ifdef XML_ATTR_INFO - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -# if UINT_MAX >= SIZE_MAX - if ((unsigned)parser->m_attsSize > (size_t)(-1) / sizeof(XML_AttrInfo)) { - parser->m_attsSize = oldAttsSize; - return XML_ERROR_NO_MEMORY; - } -# endif - - temp2 = (XML_AttrInfo *)REALLOC(parser, (void *)parser->m_attInfo, - parser->m_attsSize * sizeof(XML_AttrInfo)); - if (temp2 == NULL) { - parser->m_attsSize = oldAttsSize; - return XML_ERROR_NO_MEMORY; + if (n > oldAttsSize) { + /* Detect and prevent integer overflow. */ + if (n > (size_t)INT_MAX) + return XML_ERROR_NO_MEMORY; + XmlGetAttributes(enc, attStr, (int)n, parser->m_atts); } - parser->m_attInfo = temp2; -#endif - if (n > oldAttsSize) - XmlGetAttributes(enc, attStr, n, parser->m_atts); } - appAtts = (const XML_Char **)parser->m_atts; - for (i = 0; i < n; i++) { + /* the attribute list for the application */ + const XML_Char **const appAtts = (const XML_Char **)parser->m_atts; + for (size_t i = 0; i < n; i++) { ATTRIBUTE *currAtt = &parser->m_atts[i]; -#ifdef XML_ATTR_INFO - XML_AttrInfo *currAttInfo = &parser->m_attInfo[i]; -#endif /* add the name and value to the attribute list */ ATTRIBUTE_ID *attId = getAttributeId(parser, enc, currAtt->name, currAtt->name + XmlNameLength(enc, currAtt->name)); if (! attId) return XML_ERROR_NO_MEMORY; -#ifdef XML_ATTR_INFO - currAttInfo->nameStart - = parser->m_parseEndByteIndex - (parser->m_parseEndPtr - currAtt->name); - currAttInfo->nameEnd - = currAttInfo->nameStart + XmlNameLength(enc, currAtt->name); - currAttInfo->valueStart = parser->m_parseEndByteIndex - - (parser->m_parseEndPtr - currAtt->valuePtr); - currAttInfo->valueEnd = parser->m_parseEndByteIndex - - (parser->m_parseEndPtr - currAtt->valueEnd); -#endif /* Detect duplicate attributes by their QNames. This does not work when namespace processing is turned on and different prefixes for the same namespace are used. For this case we have a check further down. @@ -3463,22 +4226,23 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, (attId->name)[-1] = 1; appAtts[attIndex++] = attId->name; if (! parser->m_atts[i].normalized) { - enum XML_Error result; XML_Bool isCdata = XML_TRUE; /* figure out whether declared as other than CDATA */ if (attId->maybeTokenized) { - int j; - for (j = 0; j < nDefaultAtts; j++) { - if (attId == elementType->defaultAtts[j].id) { - isCdata = elementType->defaultAtts[j].isCdata; - break; - } + NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute + = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup( + parser, &(elementType->defaultAttForName), attId->name, 0); + if (nameAndDefaultAttribute != NULL) { + assert(nameAndDefaultAttribute->attIndex < elementType->nDefaultAtts); + const DEFAULT_ATTRIBUTE *const att + = elementType->defaultAtts + nameAndDefaultAttribute->attIndex; + isCdata = att->isCdata; } } /* normalize the attribute value */ - result = storeAttributeValue( + const enum XML_Error result = storeAttributeValue( parser, enc, isCdata, parser->m_atts[i].valuePtr, parser->m_atts[i].valueEnd, &parser->m_tempPool, account); if (result) @@ -3516,7 +4280,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* set-up for XML_GetSpecifiedAttributeCount and XML_GetIdAttributeIndex */ parser->m_nSpecifiedAtts = attIndex; if (elementType->idAtt && (elementType->idAtt->name)[-1]) { - for (i = 0; i < attIndex; i += 2) + for (int i = 0; i < attIndex; i += 2) if (appAtts[i] == elementType->idAtt->name) { parser->m_idAttIndex = i; break; @@ -3525,7 +4289,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, parser->m_idAttIndex = -1; /* do attribute defaulting */ - for (i = 0; i < nDefaultAtts; i++) { + for (size_t i = 0; i < nDefaultAtts; i++) { const DEFAULT_ATTRIBUTE *da = elementType->defaultAtts + i; if (! (da->id->name)[-1] && da->value) { if (da->id->prefix) { @@ -3551,9 +4315,9 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* expand prefixed attribute names, check for duplicates, and clear flags that say whether attributes were specified */ - i = 0; + int i = 0; if (nPrefixes) { - int j; /* hash table index */ + unsigned int j; /* hash table index */ unsigned long version = parser->m_nsAttsVersion; /* Detect and prevent invalid shift */ @@ -3564,9 +4328,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, unsigned int nsAttsSize = 1u << parser->m_nsAttsPower; unsigned char oldNsAttsPower = parser->m_nsAttsPower; /* size of hash table must be at least 2 * (# of prefixed attributes) */ - if ((nPrefixes << 1) - >> parser->m_nsAttsPower) { /* true for m_nsAttsPower = 0 */ - NS_ATT *temp; + if (parser->m_nsAttsPower == 0 + || (nPrefixes >> (parser->m_nsAttsPower - 1))) { /* hash table size must also be a power of 2 and >= 8 */ while (nPrefixes >> parser->m_nsAttsPower++) ; @@ -3587,15 +4350,15 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, * from -Wtype-limits on platforms where * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ #if UINT_MAX >= SIZE_MAX - if (nsAttsSize > (size_t)(-1) / sizeof(NS_ATT)) { + if (nsAttsSize > SIZE_MAX / sizeof(NS_ATT)) { /* Restore actual size of memory in m_nsAtts */ parser->m_nsAttsPower = oldNsAttsPower; return XML_ERROR_NO_MEMORY; } #endif - temp = (NS_ATT *)REALLOC(parser, parser->m_nsAtts, - nsAttsSize * sizeof(NS_ATT)); + NS_ATT *const temp + = REALLOC(parser, parser->m_nsAtts, nsAttsSize * sizeof(NS_ATT)); if (! temp) { /* Restore actual size of memory in m_nsAtts */ parser->m_nsAttsPower = oldNsAttsPower; @@ -3616,9 +4379,6 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, for (; i < attIndex; i += 2) { const XML_Char *s = appAtts[i]; if (s[-1] == 2) { /* prefixed */ - ATTRIBUTE_ID *id; - const BINDING *b; - unsigned long uriHash; struct siphash sip_state; struct sipkey sip_key; @@ -3626,7 +4386,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sip24_init(&sip_state, &sip_key); ((XML_Char *)s)[-1] = 0; /* clear flag */ - id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0); + ATTRIBUTE_ID *const id + = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0); if (! id || ! id->prefix) { /* This code is walking through the appAtts array, dealing * with (in this case) a prefixed attribute name. To be in @@ -3634,7 +4395,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, * has to have passed through the hash table lookup once * already. That implies that an entry for it already * exists, so the lookup above will return a pointer to - * already allocated memory. There is no opportunaity for + * already allocated memory. There is no opportunity for * the allocator to fail, so the condition above cannot be * fulfilled. * @@ -3644,15 +4405,12 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, */ return XML_ERROR_NO_MEMORY; /* LCOV_EXCL_LINE */ } - b = id->prefix->binding; + const BINDING *const b = id->prefix->binding; if (! b) return XML_ERROR_UNBOUND_PREFIX; - for (j = 0; j < b->uriLen; j++) { - const XML_Char c = b->uri[j]; - if (! poolAppendChar(&parser->m_tempPool, c)) - return XML_ERROR_NO_MEMORY; - } + if (! poolAppendChars(&parser->m_tempPool, b->uri, b->uriLen)) + return XML_ERROR_NO_MEMORY; sip24_update(&sip_state, b->uri, b->uriLen * sizeof(XML_Char)); @@ -3661,12 +4419,13 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sip24_update(&sip_state, s, keylen(s) * sizeof(XML_Char)); - do { /* copies null terminator */ - if (! poolAppendChar(&parser->m_tempPool, *s)) + { + const size_t len = xcslen(s) + /*null terminator*/ 1; + if (! poolAppendChars(&parser->m_tempPool, s, len)) return XML_ERROR_NO_MEMORY; - } while (*s++); + } - uriHash = (unsigned long)sip24_final(&sip_state); + const unsigned long uriHash = (unsigned long)sip24_final(&sip_state); { /* Check hash table for duplicate of expanded name (uriName). Derived from code in lookup(parser, HASH_TABLE *table, ...). @@ -3694,10 +4453,9 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, if (parser->m_ns_triplets) { /* append namespace separator and prefix */ parser->m_tempPool.ptr[-1] = parser->m_namespaceSeparator; s = b->prefix->name; - do { - if (! poolAppendChar(&parser->m_tempPool, *s)) - return XML_ERROR_NO_MEMORY; - } while (*s++); + const size_t len = xcslen(s) + /*null terminator*/ 1; + if (! poolAppendChars(&parser->m_tempPool, s, len)) + return XML_ERROR_NO_MEMORY; } /* store expanded name in attribute list */ @@ -3740,48 +4498,36 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, localPart = tagNamePtr->str; } else return XML_ERROR_NONE; - prefixLen = 0; - if (parser->m_ns_triplets && binding->prefix->name) { - for (; binding->prefix->name[prefixLen++];) - ; /* prefixLen includes null terminator */ - } + size_t prefixLen = 0; + if (parser->m_ns_triplets && binding->prefix->name) + prefixLen = xcslen(binding->prefix->name) + /*null terminator*/ 1; tagNamePtr->localPart = localPart; tagNamePtr->uriLen = binding->uriLen; tagNamePtr->prefix = binding->prefix->name; tagNamePtr->prefixLen = prefixLen; - for (i = 0; localPart[i++];) - ; /* i includes null terminator */ + + const size_t localPartLen = xcslen(localPart) + /*null terminator*/ 1; /* Detect and prevent integer overflow */ - if (binding->uriLen > INT_MAX - prefixLen - || i > INT_MAX - (binding->uriLen + prefixLen)) { + if (binding->uriLen > SIZE_MAX - prefixLen + || localPartLen > SIZE_MAX - (binding->uriLen + prefixLen)) { return XML_ERROR_NO_MEMORY; } - n = i + binding->uriLen + prefixLen; - if (n > binding->uriAlloc) { - TAG *p; - + const size_t totalLen = localPartLen + binding->uriLen + prefixLen; + if (totalLen > binding->uriAlloc) { /* Detect and prevent integer overflow */ - if (n > INT_MAX - EXPAND_SPARE) { - return XML_ERROR_NO_MEMORY; - } - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)(n + EXPAND_SPARE) > (size_t)(-1) / sizeof(XML_Char)) { + if (totalLen > SIZE_MAX - EXPAND_SPARE + || totalLen + EXPAND_SPARE > SIZE_MAX / sizeof(XML_Char)) { return XML_ERROR_NO_MEMORY; } -#endif - uri = (XML_Char *)MALLOC(parser, (n + EXPAND_SPARE) * sizeof(XML_Char)); + uri = MALLOC(parser, (totalLen + EXPAND_SPARE) * sizeof(XML_Char)); if (! uri) return XML_ERROR_NO_MEMORY; - binding->uriAlloc = n + EXPAND_SPARE; + binding->uriAlloc = totalLen + EXPAND_SPARE; memcpy(uri, binding->uri, binding->uriLen * sizeof(XML_Char)); - for (p = parser->m_tagStack; p; p = p->parent) + for (TAG *p = parser->m_tagStack; p; p = p->parent) if (p->name.str == binding->uri) p->name.str = uri; FREE(parser, binding->uri); @@ -3789,10 +4535,14 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, } /* if m_namespaceSeparator != '\0' then uri includes it already */ uri = binding->uri + binding->uriLen; - memcpy(uri, localPart, i * sizeof(XML_Char)); + /* Detect and prevent integer overflow */ + if (localPartLen > SIZE_MAX / sizeof(XML_Char)) { + return XML_ERROR_NO_MEMORY; + } + memcpy(uri, localPart, localPartLen * sizeof(XML_Char)); /* we always have a namespace separator between localPart and prefix */ if (prefixLen) { - uri += i - 1; + uri += localPartLen - 1; *uri = parser->m_namespaceSeparator; /* replace null terminator */ memcpy(uri + 1, binding->prefix->name, prefixLen * sizeof(XML_Char)); } @@ -3927,7 +4677,7 @@ addBinding(XML_Parser parser, PPREFIX *prefix, const ATTRIBUTE_ID *attId, ASCII_8, ASCII_SLASH, ASCII_n, ASCII_a, ASCII_m, ASCII_e, ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e, '\0'}; - static const int xmlLen = (int)sizeof(xmlNamespace) / sizeof(XML_Char) - 1; + static const size_t xmlLen = sizeof(xmlNamespace) / sizeof(XML_Char) - 1; // "http://www.w3.org/2000/xmlns/" static const XML_Char xmlnsNamespace[] = {ASCII_h, ASCII_t, ASCII_t, ASCII_p, ASCII_COLON, ASCII_SLASH, @@ -3935,15 +4685,14 @@ addBinding(XML_Parser parser, PPREFIX *prefix, const ATTRIBUTE_ID *attId, ASCII_3, ASCII_PERIOD, ASCII_o, ASCII_r, ASCII_g, ASCII_SLASH, ASCII_2, ASCII_0, ASCII_0, ASCII_0, ASCII_SLASH, ASCII_x, ASCII_m, ASCII_l, ASCII_n, ASCII_s, ASCII_SLASH, '\0'}; - static const int xmlnsLen - = (int)sizeof(xmlnsNamespace) / sizeof(XML_Char) - 1; + static const size_t xmlnsLen = sizeof(xmlnsNamespace) / sizeof(XML_Char) - 1; XML_Bool mustBeXML = XML_FALSE; XML_Bool isXML = XML_TRUE; XML_Bool isXMLNS = XML_TRUE; BINDING *b; - int len; + size_t len; /* empty URI is only valid for default namespace per XML NS 1.0 (not 1.1) */ if (*uri == XML_T('\0') && prefix->name) @@ -3962,6 +4711,10 @@ addBinding(XML_Parser parser, PPREFIX *prefix, const ATTRIBUTE_ID *attId, } for (len = 0; uri[len]; len++) { + /* Detect and prevent integer overflow */ + if (len == SIZE_MAX) { + return XML_ERROR_NO_MEMORY; + } if (isXML && (len > xmlLen || uri[len] != xmlNamespace[len])) isXML = XML_FALSE; @@ -4002,28 +4755,24 @@ addBinding(XML_Parser parser, PPREFIX *prefix, const ATTRIBUTE_ID *attId, if (isXMLNS) return XML_ERROR_RESERVED_NAMESPACE_URI; - if (parser->m_namespaceSeparator) + if (parser->m_namespaceSeparator) { + /* Detect and prevent integer overflow */ + if (len == SIZE_MAX) { + return XML_ERROR_NO_MEMORY; + } len++; + } if (parser->m_freeBindingList) { b = parser->m_freeBindingList; if (len > b->uriAlloc) { /* Detect and prevent integer overflow */ - if (len > INT_MAX - EXPAND_SPARE) { - return XML_ERROR_NO_MEMORY; - } - - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)(len + EXPAND_SPARE) > (size_t)(-1) / sizeof(XML_Char)) { + if (len > SIZE_MAX - EXPAND_SPARE + || len + EXPAND_SPARE > SIZE_MAX / sizeof(XML_Char)) { return XML_ERROR_NO_MEMORY; } -#endif - XML_Char *temp = (XML_Char *)REALLOC( - parser, b->uri, sizeof(XML_Char) * (len + EXPAND_SPARE)); + XML_Char *temp + = REALLOC(parser, b->uri, sizeof(XML_Char) * (len + EXPAND_SPARE)); if (temp == NULL) return XML_ERROR_NO_MEMORY; b->uri = temp; @@ -4031,26 +4780,18 @@ addBinding(XML_Parser parser, PPREFIX *prefix, const ATTRIBUTE_ID *attId, } parser->m_freeBindingList = b->nextTagBinding; } else { - b = (BINDING *)MALLOC(parser, sizeof(BINDING)); + b = MALLOC(parser, sizeof(BINDING)); if (! b) return XML_ERROR_NO_MEMORY; /* Detect and prevent integer overflow */ - if (len > INT_MAX - EXPAND_SPARE) { - return XML_ERROR_NO_MEMORY; - } - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)(len + EXPAND_SPARE) > (size_t)(-1) / sizeof(XML_Char)) { + if (len > SIZE_MAX - EXPAND_SPARE + || len + EXPAND_SPARE > SIZE_MAX / sizeof(XML_Char)) { + FREE(parser, b); return XML_ERROR_NO_MEMORY; } -#endif - b->uri - = (XML_Char *)MALLOC(parser, sizeof(XML_Char) * (len + EXPAND_SPARE)); + b->uri = MALLOC(parser, sizeof(XML_Char) * (len + EXPAND_SPARE)); if (! b->uri) { FREE(parser, b); return XML_ERROR_NO_MEMORY; @@ -4072,16 +4813,19 @@ addBinding(XML_Parser parser, PPREFIX *prefix, const ATTRIBUTE_ID *attId, b->nextTagBinding = *bindingsPtr; *bindingsPtr = b; /* if attId == NULL then we are not starting a namespace scope */ - if (attId && parser->m_startNamespaceDeclHandler) + if (attId && parser->m_startNamespaceDeclHandler) { + beforeHandler(parser); parser->m_startNamespaceDeclHandler(parser->m_handlerArg, prefix->name, prefix->binding ? uri : 0); + afterHandler(parser); + } return XML_ERROR_NONE; } /* The idea here is to avoid using stack for each CDATA section when the whole file is parsed with one call. */ -static enum XML_Error PTRCALL +static enum XML_Error cdataSectionProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { enum XML_Error result = doCdataSection( @@ -4136,15 +4880,20 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, *eventEndPP = next; switch (tok) { case XML_TOK_CDATA_SECT_CLOSE: - if (parser->m_endCdataSectionHandler) + if (parser->m_endCdataSectionHandler) { + beforeHandler(parser); parser->m_endCdataSectionHandler(parser->m_handlerArg); + afterHandler(parser); + } /* BEGIN disabled code */ /* see comment under XML_TOK_CDATA_SECT_OPEN */ - else if ((0) && parser->m_characterDataHandler) + else if ((0) && parser->m_characterDataHandler) { + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0); - /* END disabled code */ - else if (parser->m_defaultHandler) + afterHandler(parser); + /* END disabled code */ + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); *startPtr = next; *nextPtr = next; @@ -4155,7 +4904,9 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, case XML_TOK_DATA_NEWLINE: if (parser->m_characterDataHandler) { XML_Char c = 0xA; + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, &c, 1); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; @@ -4168,16 +4919,21 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, const enum XML_Convert_Result convert_res = XmlConvert( enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd); *eventEndPP = next; + beforeHandler(parser); charDataHandler(parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); + afterHandler(parser); if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; *eventPP = s; } - } else + } else { + beforeHandler(parser); charDataHandler(parser->m_handlerArg, (const XML_Char *)s, (int)((const XML_Char *)next - (const XML_Char *)s)); + afterHandler(parser); + } } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); } break; @@ -4210,14 +4966,21 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, /* LCOV_EXCL_STOP */ } - *eventPP = s = next; switch (parser->m_parsingStatus.parsing) { case XML_SUSPENDED: + *eventPP = next; *nextPtr = next; return XML_ERROR_NONE; case XML_FINISHED: + *eventPP = next; return XML_ERROR_ABORTED; + case XML_PARSING: + if (parser->m_reenter) { + return XML_ERROR_UNEXPECTED_STATE; // LCOV_EXCL_LINE + } + EXPAT_FALLTHROUGH; default:; + *eventPP = s = next; } } /* not reached */ @@ -4228,7 +4991,7 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, /* The idea here is to avoid using stack for each IGNORE section when the whole file is parsed with one call. */ -static enum XML_Error PTRCALL +static enum XML_Error ignoreSectionProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { enum XML_Error result @@ -4409,8 +5172,10 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity, const char *s, if (! storedversion) return XML_ERROR_NO_MEMORY; } + beforeHandler(parser); parser->m_xmlDeclHandler(parser->m_handlerArg, storedversion, storedEncName, standalone); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, parser->m_encoding, s, next); if (parser->m_protocolEncodingName == NULL) { @@ -4460,31 +5225,43 @@ handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName) { info.convert = NULL; info.data = NULL; info.release = NULL; - if (parser->m_unknownEncodingHandler(parser->m_unknownEncodingHandlerData, - encodingName, &info)) { + beforeHandler(parser); + const int status = parser->m_unknownEncodingHandler( + parser->m_unknownEncodingHandlerData, encodingName, &info); + afterHandler(parser); + + parser->m_unknownEncodingRelease = info.release; + parser->m_unknownEncodingData = info.data; + + if (status) { ENCODING *enc; parser->m_unknownEncodingMem = MALLOC(parser, XmlSizeOfUnknownEncoding()); if (! parser->m_unknownEncodingMem) { - if (info.release) - info.release(info.data); + if (parser->m_unknownEncodingRelease) + callUnknownEncodingRelease(parser); + else + parser->m_unknownEncodingData = NULL; return XML_ERROR_NO_MEMORY; } + parser->m_unknownEncodingConvert = info.convert; enc = (parser->m_ns ? XmlInitUnknownEncodingNS : XmlInitUnknownEncoding)( - parser->m_unknownEncodingMem, info.map, info.convert, info.data); + parser->m_unknownEncodingMem, info.map, + info.convert ? callUnknownEncodingConvert : NULL, parser); if (enc) { - parser->m_unknownEncodingData = info.data; - parser->m_unknownEncodingRelease = info.release; parser->m_encoding = enc; return XML_ERROR_NONE; } + parser->m_unknownEncodingConvert = NULL; } - if (info.release != NULL) - info.release(info.data); + if (parser->m_unknownEncodingRelease != NULL) + callUnknownEncodingRelease(parser); + else + parser->m_unknownEncodingData = NULL; } return XML_ERROR_UNKNOWN_ENCODING; } -static enum XML_Error PTRCALL +static enum XML_Error prologInitProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { enum XML_Error result = initializeEncoding(parser); @@ -4496,7 +5273,7 @@ prologInitProcessor(XML_Parser parser, const char *s, const char *end, #ifdef XML_DTD -static enum XML_Error PTRCALL +static enum XML_Error externalParEntInitProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { enum XML_Error result = initializeEncoding(parser); @@ -4505,7 +5282,7 @@ externalParEntInitProcessor(XML_Parser parser, const char *s, const char *end, /* we know now that XML_Parse(Buffer) has been called, so we consider the external parameter entity read */ - parser->m_dtd->paramEntityRead = XML_TRUE; + parser->m_dtd->paramEntityRead = true; if (parser->m_prologState.inEntityValue) { parser->m_processor = entityValueInitProcessor; @@ -4516,7 +5293,7 @@ externalParEntInitProcessor(XML_Parser parser, const char *s, const char *end, } } -static enum XML_Error PTRCALL +static enum XML_Error entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { int tok; @@ -4549,7 +5326,7 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, } /* found end of entity value - can store it now */ return storeEntityValue(parser, parser->m_encoding, s, end, - XML_ACCOUNT_DIRECT); + XML_ACCOUNT_DIRECT, NULL); } else if (tok == XML_TOK_XML_DECL) { enum XML_Error result; result = processXmlDecl(parser, 0, start, next); @@ -4589,7 +5366,7 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, } /* If we get this token, we have the start of what might be a normal tag, but not a declaration (i.e. it doesn't begin with - "m_startDoctypeDeclHandler) { + beforeHandler(parser); parser->m_startDoctypeDeclHandler( parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid, parser->m_doctypePubid, 1); + afterHandler(parser); parser->m_doctypeName = NULL; poolClear(&parser->m_tempPool); handleDefault = XML_FALSE; @@ -4870,7 +5658,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, handleDefault = XML_FALSE; goto alreadyChecked; } - /* fall through */ + EXPAT_FALLTHROUGH; case XML_ROLE_ENTITY_PUBLIC_ID: if (! XmlIsPublicId(enc, s, next, eventPP)) return XML_ERROR_PUBLICID; @@ -4898,9 +5686,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } if (parser->m_doctypeName) { + beforeHandler(parser); parser->m_startDoctypeDeclHandler( parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid, parser->m_doctypePubid, 0); + afterHandler(parser); poolClear(&parser->m_tempPool); handleDefault = XML_FALSE; } @@ -4926,15 +5716,23 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } if (parser->m_useForeignDTD) entity->base = parser->m_curBase; - dtd->paramEntityRead = XML_FALSE; - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, 0, entity->base, - entity->systemId, entity->publicId)) + dtd->paramEntityRead = false; + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, 0, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) return XML_ERROR_EXTERNAL_ENTITY_HANDLING; if (dtd->paramEntityRead) { - if (! dtd->standalone && parser->m_notStandaloneHandler - && ! parser->m_notStandaloneHandler(parser->m_handlerArg)) - return XML_ERROR_NOT_STANDALONE; + if (! dtd->standalone && parser->m_notStandaloneHandler) { + beforeHandler(parser); + const int handlerStatus + = parser->m_notStandaloneHandler(parser->m_handlerArg); + afterHandler(parser); + if (! handlerStatus) + return XML_ERROR_NOT_STANDALONE; + } } /* if we didn't read the foreign DTD then this means that there is no external subset and we must reset dtd->hasParamEntityRefs @@ -4947,7 +5745,9 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } #endif /* XML_DTD */ if (parser->m_endDoctypeDeclHandler) { + beforeHandler(parser); parser->m_endDoctypeDeclHandler(parser->m_handlerArg); + afterHandler(parser); handleDefault = XML_FALSE; } break; @@ -4966,15 +5766,23 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (! entity) return XML_ERROR_NO_MEMORY; entity->base = parser->m_curBase; - dtd->paramEntityRead = XML_FALSE; - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, 0, entity->base, - entity->systemId, entity->publicId)) + dtd->paramEntityRead = false; + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, 0, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) return XML_ERROR_EXTERNAL_ENTITY_HANDLING; if (dtd->paramEntityRead) { - if (! dtd->standalone && parser->m_notStandaloneHandler - && ! parser->m_notStandaloneHandler(parser->m_handlerArg)) - return XML_ERROR_NOT_STANDALONE; + if (! dtd->standalone && parser->m_notStandaloneHandler) { + beforeHandler(parser); + const int handlerStatus + = parser->m_notStandaloneHandler(parser->m_handlerArg); + afterHandler(parser); + if (! handlerStatus) + return XML_ERROR_NOT_STANDALONE; + } } /* if we didn't read the foreign DTD then this means that there is no external subset and we must reset dtd->hasParamEntityRefs @@ -5067,10 +5875,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, poolFinish(&parser->m_tempPool); } *eventEndPP = s; + beforeHandler(parser); parser->m_attlistDeclHandler( parser->m_handlerArg, parser->m_declElementType->name, parser->m_declAttributeId->name, parser->m_declAttributeType, 0, role == XML_ROLE_REQUIRED_ATTRIBUTE_VALUE); + afterHandler(parser); handleDefault = XML_FALSE; } } @@ -5105,10 +5915,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, poolFinish(&parser->m_tempPool); } *eventEndPP = s; + beforeHandler(parser); parser->m_attlistDeclHandler( parser->m_handlerArg, parser->m_declElementType->name, parser->m_declAttributeId->name, parser->m_declAttributeType, attVal, role == XML_ROLE_FIXED_ATTRIBUTE_VALUE); + afterHandler(parser); poolClear(&parser->m_tempPool); handleDefault = XML_FALSE; } @@ -5119,20 +5931,26 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, #if XML_GE == 1 // This will store the given replacement text in // parser->m_declEntity->textPtr. - enum XML_Error result - = storeEntityValue(parser, enc, s + enc->minBytesPerChar, - next - enc->minBytesPerChar, XML_ACCOUNT_NONE); + enum XML_Error result = callStoreEntityValue( + parser, enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar, + XML_ACCOUNT_NONE); if (parser->m_declEntity) { + /* Detect and prevent signed integer overflow */ + if ((size_t)poolLength(&dtd->entityValuePool) > (size_t)INT_MAX) { + return XML_ERROR_NO_MEMORY; + } parser->m_declEntity->textPtr = poolStart(&dtd->entityValuePool); parser->m_declEntity->textLen = (int)(poolLength(&dtd->entityValuePool)); poolFinish(&dtd->entityValuePool); if (parser->m_entityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->is_param, parser->m_declEntity->textPtr, parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0); + afterHandler(parser); handleDefault = XML_FALSE; } } else @@ -5150,10 +5968,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (parser->m_entityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->is_param, parser->m_declEntity->textPtr, parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0); + afterHandler(parser); handleDefault = XML_FALSE; } } @@ -5184,9 +6004,13 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, #ifdef XML_DTD && ! parser->m_paramEntityParsing #endif /* XML_DTD */ - && parser->m_notStandaloneHandler - && ! parser->m_notStandaloneHandler(parser->m_handlerArg)) - return XML_ERROR_NOT_STANDALONE; + && parser->m_notStandaloneHandler) { + beforeHandler(parser); + const int status = parser->m_notStandaloneHandler(parser->m_handlerArg); + afterHandler(parser); + if (! status) + return XML_ERROR_NOT_STANDALONE; + } #ifndef XML_DTD break; #else /* XML_DTD */ @@ -5198,7 +6022,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, parser->m_declEntity->publicId = NULL; } #endif /* XML_DTD */ - /* fall through */ + EXPAT_FALLTHROUGH; case XML_ROLE_ENTITY_SYSTEM_ID: if (dtd->keepProcessing && parser->m_declEntity) { parser->m_declEntity->systemId @@ -5229,10 +6053,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (dtd->keepProcessing && parser->m_declEntity && parser->m_entityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->is_param, 0, 0, parser->m_declEntity->base, parser->m_declEntity->systemId, parser->m_declEntity->publicId, 0); + afterHandler(parser); handleDefault = XML_FALSE; } break; @@ -5245,17 +6071,21 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, poolFinish(&dtd->pool); if (parser->m_unparsedEntityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_unparsedEntityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->base, parser->m_declEntity->systemId, parser->m_declEntity->publicId, parser->m_declEntity->notation); + afterHandler(parser); handleDefault = XML_FALSE; } else if (parser->m_entityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, 0, 0, 0, parser->m_declEntity->base, parser->m_declEntity->systemId, parser->m_declEntity->publicId, parser->m_declEntity->notation); + afterHandler(parser); handleDefault = XML_FALSE; } } @@ -5279,7 +6109,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } else { poolFinish(&dtd->pool); parser->m_declEntity->publicId = NULL; - parser->m_declEntity->is_param = XML_FALSE; + parser->m_declEntity->is_param = false; /* if we have a parent parser or are reading an internal parameter entity, then the entity declaration is not considered "internal" */ @@ -5309,7 +6139,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } else { poolFinish(&dtd->pool); parser->m_declEntity->publicId = NULL; - parser->m_declEntity->is_param = XML_TRUE; + parser->m_declEntity->is_param = true; /* if we have a parent parser or are reading an internal parameter entity, then the entity declaration is not considered "internal" */ @@ -5362,9 +6192,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (! systemId) return XML_ERROR_NO_MEMORY; *eventEndPP = s; + beforeHandler(parser); parser->m_notationDeclHandler( parser->m_handlerArg, parser->m_declNotationName, parser->m_curBase, systemId, parser->m_declNotationPublicId); + afterHandler(parser); handleDefault = XML_FALSE; } poolClear(&parser->m_tempPool); @@ -5372,9 +6204,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, case XML_ROLE_NOTATION_NO_SYSTEM_ID: if (parser->m_declNotationPublicId && parser->m_notationDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_notationDeclHandler( parser->m_handlerArg, parser->m_declNotationName, parser->m_curBase, 0, parser->m_declNotationPublicId); + afterHandler(parser); handleDefault = XML_FALSE; } poolClear(&parser->m_tempPool); @@ -5408,41 +6242,20 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, case XML_ROLE_GROUP_OPEN: if (parser->m_prologState.level >= parser->m_groupSize) { if (parser->m_groupSize) { - { - /* Detect and prevent integer overflow */ - if (parser->m_groupSize > (unsigned int)(-1) / 2u) { - return XML_ERROR_NO_MEMORY; - } - - char *const new_connector = (char *)REALLOC( - parser, parser->m_groupConnector, parser->m_groupSize *= 2); - if (new_connector == NULL) { - parser->m_groupSize /= 2; - return XML_ERROR_NO_MEMORY; - } - parser->m_groupConnector = new_connector; + /* Detect and prevent integer overflow */ + if (parser->m_groupSize > SIZE_MAX / 2) { + return XML_ERROR_NO_MEMORY; } - if (dtd->scaffIndex) { - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if (parser->m_groupSize > (size_t)(-1) / sizeof(int)) { - return XML_ERROR_NO_MEMORY; - } -#endif - - int *const new_scaff_index = (int *)REALLOC( - parser, dtd->scaffIndex, parser->m_groupSize * sizeof(int)); - if (new_scaff_index == NULL) - return XML_ERROR_NO_MEMORY; - dtd->scaffIndex = new_scaff_index; + char *const new_connector = REALLOC(parser, parser->m_groupConnector, + parser->m_groupSize *= 2); + if (new_connector == NULL) { + parser->m_groupSize /= 2; + return XML_ERROR_NO_MEMORY; } + parser->m_groupConnector = new_connector; } else { - parser->m_groupConnector - = (char *)MALLOC(parser, parser->m_groupSize = 32); + parser->m_groupConnector = MALLOC(parser, parser->m_groupSize = 32); if (! parser->m_groupConnector) { parser->m_groupSize = 0; return XML_ERROR_NO_MEMORY; @@ -5455,6 +6268,21 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (myindex < 0) return XML_ERROR_NO_MEMORY; assert(dtd->scaffIndex != NULL); + if ((size_t)dtd->scaffLevel >= dtd->scaffIndexSize) { + /* Detect and prevent integer overflow */ + if (dtd->scaffIndexSize > SIZE_MAX / 2 / sizeof(int)) { + return XML_ERROR_NO_MEMORY; + } + assert(dtd->scaffIndexSize > 0); + const size_t new_size = dtd->scaffIndexSize * 2; + int *const new_scaff_index + = REALLOC(parser, dtd->scaffIndex, new_size * sizeof(int)); + if (new_scaff_index == NULL) { + return XML_ERROR_NO_MEMORY; + } + dtd->scaffIndex = new_scaff_index; + dtd->scaffIndexSize = new_size; + } dtd->scaffIndex[dtd->scaffLevel] = myindex; dtd->scaffLevel++; dtd->scaffold[myindex].type = XML_CTYPE_SEQ; @@ -5535,7 +6363,9 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, /* cannot report skipped entities in declarations */ if ((role == XML_ROLE_PARAM_ENTITY_REF) && parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, name, 1); + afterHandler(parser); handleDefault = XML_FALSE; } break; @@ -5544,27 +6374,29 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, return XML_ERROR_RECURSIVE_ENTITY_REF; if (entity->textPtr) { enum XML_Error result; - XML_Bool betweenDecl - = (role == XML_ROLE_PARAM_ENTITY_REF ? XML_TRUE : XML_FALSE); - result = processInternalEntity(parser, entity, betweenDecl); + bool betweenDecl = (role == XML_ROLE_PARAM_ENTITY_REF); + result = processEntity(parser, entity, betweenDecl, ENTITY_INTERNAL); if (result != XML_ERROR_NONE) return result; handleDefault = XML_FALSE; break; } if (parser->m_externalEntityRefHandler) { - dtd->paramEntityRead = XML_FALSE; - entity->open = XML_TRUE; + dtd->paramEntityRead = false; + entity->open = true; entityTrackingOnOpen(parser, entity, __LINE__); - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, 0, entity->base, - entity->systemId, entity->publicId)) { + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, 0, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) { entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; return XML_ERROR_EXTERNAL_ENTITY_HANDLING; } entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; handleDefault = XML_FALSE; if (! dtd->paramEntityRead) { dtd->keepProcessing = dtd->standalone; @@ -5576,9 +6408,13 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } } #endif /* XML_DTD */ - if (! dtd->standalone && parser->m_notStandaloneHandler - && ! parser->m_notStandaloneHandler(parser->m_handlerArg)) - return XML_ERROR_NOT_STANDALONE; + if (! dtd->standalone && parser->m_notStandaloneHandler) { + beforeHandler(parser); + const int status = parser->m_notStandaloneHandler(parser->m_handlerArg); + afterHandler(parser); + if (! status) + return XML_ERROR_NOT_STANDALONE; + } break; /* Element declaration stuff */ @@ -5590,7 +6426,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, return XML_ERROR_NO_MEMORY; dtd->scaffLevel = 0; dtd->scaffCount = 0; - dtd->in_eldecl = XML_TRUE; + dtd->in_eldecl = true; handleDefault = XML_FALSE; } break; @@ -5599,8 +6435,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, case XML_ROLE_CONTENT_EMPTY: if (dtd->in_eldecl) { if (parser->m_elementDeclHandler) { - XML_Content *content - = (XML_Content *)MALLOC(parser, sizeof(XML_Content)); + // NOTE: We are avoiding MALLOC(..) here to so that + // applications that are not using XML_FreeContentModel but + // plain free(..) or .free_fcn() to free the content model's + // memory are safe. + XML_Content *content = parser->m_mem.malloc_fcn(sizeof(XML_Content)); if (! content) return XML_ERROR_NO_MEMORY; content->quant = XML_CQUANT_NONE; @@ -5610,11 +6449,13 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, content->type = ((role == XML_ROLE_CONTENT_ANY) ? XML_CTYPE_ANY : XML_CTYPE_EMPTY); *eventEndPP = s; + beforeHandler(parser); parser->m_elementDeclHandler( parser->m_handlerArg, parser->m_declElementType->name, content); + afterHandler(parser); handleDefault = XML_FALSE; } - dtd->in_eldecl = XML_FALSE; + dtd->in_eldecl = false; } break; @@ -5655,9 +6496,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, return XML_ERROR_NO_MEMORY; name = el->name; dtd->scaffold[myindex].name = name; - nameLen = 0; - for (; name[nameLen++];) - ; + nameLen = xcslen(name) + /*null terminator*/ 1; /* Detect and prevent integer overflow */ if (nameLen > UINT_MAX - dtd->contentStringLen) { @@ -5693,10 +6532,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (! model) return XML_ERROR_NO_MEMORY; *eventEndPP = s; + beforeHandler(parser); parser->m_elementDeclHandler( parser->m_handlerArg, parser->m_declElementType->name, model); + afterHandler(parser); } - dtd->in_eldecl = XML_FALSE; + dtd->in_eldecl = false; dtd->contentStringLen = 0; } } @@ -5751,6 +6592,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, return XML_ERROR_NONE; case XML_FINISHED: return XML_ERROR_ABORTED; + case XML_PARSING: + if (parser->m_reenter) { + *nextPtr = next; + return XML_ERROR_NONE; + } + EXPAT_FALLTHROUGH; default: s = next; tok = XmlPrologTok(enc, s, end, &next); @@ -5759,7 +6606,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, /* not reached */ } -static enum XML_Error PTRCALL +static enum XML_Error epilogProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { parser->m_processor = epilogProcessor; @@ -5818,84 +6665,90 @@ epilogProcessor(XML_Parser parser, const char *s, const char *end, default: return XML_ERROR_JUNK_AFTER_DOC_ELEMENT; } - parser->m_eventPtr = s = next; switch (parser->m_parsingStatus.parsing) { case XML_SUSPENDED: + parser->m_eventPtr = next; *nextPtr = next; return XML_ERROR_NONE; case XML_FINISHED: + parser->m_eventPtr = next; return XML_ERROR_ABORTED; + case XML_PARSING: + if (parser->m_reenter) { + return XML_ERROR_UNEXPECTED_STATE; // LCOV_EXCL_LINE + } + EXPAT_FALLTHROUGH; default:; + parser->m_eventPtr = s = next; } } } static enum XML_Error -processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl) { - const char *textStart, *textEnd; - const char *next; - enum XML_Error result; - OPEN_INTERNAL_ENTITY *openEntity; +processEntity(XML_Parser parser, ENTITY *entity, bool betweenDecl, + enum EntityType type) { + OPEN_INTERNAL_ENTITY *openEntity, **openEntityList; + OPEN_INTERNAL_ENTITY **const freeEntityList = &parser->m_freeEntities; + switch (type) { + case ENTITY_INTERNAL: + parser->m_processor = internalEntityProcessor; + openEntityList = &parser->m_openInternalEntities; + break; + case ENTITY_ATTRIBUTE: + openEntityList = &parser->m_openAttributeEntities; + break; + case ENTITY_VALUE: + openEntityList = &parser->m_openValueEntities; + break; + /* default case serves merely as a safety net in case of a + * wrong entityType. Therefore we exclude the following lines + * from the test coverage. + * + * LCOV_EXCL_START + */ + default: + // Should not reach here + assert(0); + /* LCOV_EXCL_STOP */ + } - if (parser->m_freeInternalEntities) { - openEntity = parser->m_freeInternalEntities; - parser->m_freeInternalEntities = openEntity->next; + if (*freeEntityList) { + openEntity = *freeEntityList; + *freeEntityList = openEntity->next; } else { - openEntity - = (OPEN_INTERNAL_ENTITY *)MALLOC(parser, sizeof(OPEN_INTERNAL_ENTITY)); + openEntity = MALLOC(parser, sizeof(OPEN_INTERNAL_ENTITY)); if (! openEntity) return XML_ERROR_NO_MEMORY; } - entity->open = XML_TRUE; + entity->open = true; + entity->hasMore = true; #if XML_GE == 1 entityTrackingOnOpen(parser, entity, __LINE__); #endif entity->processed = 0; - openEntity->next = parser->m_openInternalEntities; - parser->m_openInternalEntities = openEntity; + openEntity->next = *openEntityList; + *openEntityList = openEntity; openEntity->entity = entity; + openEntity->type = type; openEntity->startTagLevel = parser->m_tagLevel; openEntity->betweenDecl = betweenDecl; openEntity->internalEventPtr = NULL; openEntity->internalEventEndPtr = NULL; - textStart = (const char *)entity->textPtr; - textEnd = (const char *)(entity->textPtr + entity->textLen); - /* Set a safe default value in case 'next' does not get set */ - next = textStart; - - if (entity->is_param) { - int tok - = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next); - result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, - tok, next, &next, XML_FALSE, XML_FALSE, - XML_ACCOUNT_ENTITY_EXPANSION); - } else { - result = doContent(parser, parser->m_tagLevel, parser->m_internalEncoding, - textStart, textEnd, &next, XML_FALSE, - XML_ACCOUNT_ENTITY_EXPANSION); - } - if (result == XML_ERROR_NONE) { - if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) { - entity->processed = (int)(next - textStart); - parser->m_processor = internalEntityProcessor; - } else if (parser->m_openInternalEntities->entity == entity) { -#if XML_GE == 1 - entityTrackingOnClose(parser, entity, __LINE__); -#endif /* XML_GE == 1 */ - entity->open = XML_FALSE; - parser->m_openInternalEntities = openEntity->next; - /* put openEntity back in list of free instances */ - openEntity->next = parser->m_freeInternalEntities; - parser->m_freeInternalEntities = openEntity; - } + // Only internal entities make use of the reenter flag + // therefore no need to set it for other entity types + if (type == ENTITY_INTERNAL) { + triggerReenter(parser); } - return result; + return XML_ERROR_NONE; } -static enum XML_Error PTRCALL +static enum XML_Error internalEntityProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { + UNUSED_P(s); + UNUSED_P(end); + UNUSED_P(nextPtr); ENTITY *entity; const char *textStart, *textEnd; const char *next; @@ -5905,71 +6758,74 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, return XML_ERROR_UNEXPECTED_STATE; entity = openEntity->entity; - textStart = ((const char *)entity->textPtr) + entity->processed; - textEnd = (const char *)(entity->textPtr + entity->textLen); - /* Set a safe default value in case 'next' does not get set */ - next = textStart; - - if (entity->is_param) { - int tok - = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next); - result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, - tok, next, &next, XML_FALSE, XML_TRUE, - XML_ACCOUNT_ENTITY_EXPANSION); - } else { - result = doContent(parser, openEntity->startTagLevel, - parser->m_internalEncoding, textStart, textEnd, &next, - XML_FALSE, XML_ACCOUNT_ENTITY_EXPANSION); - } - if (result != XML_ERROR_NONE) - return result; + // This will return early + if (entity->hasMore) { + textStart = ((const char *)entity->textPtr) + entity->processed; + textEnd = (const char *)(entity->textPtr + entity->textLen); + /* Set a safe default value in case 'next' does not get set */ + next = textStart; + + if (entity->is_param) { + int tok + = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next); + result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, + tok, next, &next, XML_FALSE, XML_FALSE, + XML_ACCOUNT_ENTITY_EXPANSION); + } else { + result = doContent(parser, openEntity->startTagLevel, + parser->m_internalEncoding, textStart, textEnd, &next, + XML_FALSE, XML_ACCOUNT_ENTITY_EXPANSION); + } + + if (result != XML_ERROR_NONE) + return result; + // Check if entity is complete, if not, mark down how much of it is + // processed + if (textEnd != next + && (parser->m_parsingStatus.parsing == XML_SUSPENDED + || (parser->m_parsingStatus.parsing == XML_PARSING + && parser->m_reenter))) { + entity->processed = (int)(next - (const char *)entity->textPtr); + return result; + } - if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) { - entity->processed = (int)(next - (const char *)entity->textPtr); + // Entity is complete. We cannot close it here since we need to first + // process its possible inner entities (which are added to the + // m_openInternalEntities during doProlog or doContent calls above) + entity->hasMore = false; + if (! entity->is_param + && (openEntity->startTagLevel != parser->m_tagLevel)) { + return XML_ERROR_ASYNC_ENTITY; + } + triggerReenter(parser); return result; - } + } // End of entity processing, "if" block will return here + // Remove fully processed openEntity from open entity list. #if XML_GE == 1 entityTrackingOnClose(parser, entity, __LINE__); #endif - entity->open = XML_FALSE; - parser->m_openInternalEntities = openEntity->next; + // openEntity is m_openInternalEntities' head, as we set it at the start of + // this function and we skipped doProlog and doContent calls with hasMore set + // to false. This means we can directly remove the head of + // m_openInternalEntities + assert(parser->m_openInternalEntities == openEntity); + entity->open = false; + parser->m_openInternalEntities = parser->m_openInternalEntities->next; + /* put openEntity back in list of free instances */ - openEntity->next = parser->m_freeInternalEntities; - parser->m_freeInternalEntities = openEntity; - - // If there are more open entities we want to stop right here and have the - // upcoming call to XML_ResumeParser continue with entity content, or it would - // be ignored altogether. - if (parser->m_openInternalEntities != NULL - && parser->m_parsingStatus.parsing == XML_SUSPENDED) { - return XML_ERROR_NONE; - } + openEntity->next = parser->m_freeEntities; + parser->m_freeEntities = openEntity; - if (entity->is_param) { - int tok; - parser->m_processor = prologProcessor; - tok = XmlPrologTok(parser->m_encoding, s, end, &next); - return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE, - XML_ACCOUNT_DIRECT); - } else { - parser->m_processor = contentProcessor; - /* see externalEntityContentProcessor vs contentProcessor */ - result = doContent(parser, parser->m_parentParser ? 1 : 0, - parser->m_encoding, s, end, nextPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer, - XML_ACCOUNT_DIRECT); - if (result == XML_ERROR_NONE) { - if (! storeRawNames(parser)) - return XML_ERROR_NO_MEMORY; - } - return result; + if (parser->m_openInternalEntities == NULL) { + parser->m_processor = entity->is_param ? prologProcessor : contentProcessor; } + triggerReenter(parser); + return XML_ERROR_NONE; } -static enum XML_Error PTRCALL +static enum XML_Error errorProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { UNUSED_P(s); @@ -5982,8 +6838,71 @@ static enum XML_Error storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, const char *ptr, const char *end, STRING_POOL *pool, enum XML_Account account) { - enum XML_Error result - = appendAttributeValue(parser, enc, isCdata, ptr, end, pool, account); + const char *next = ptr; + enum XML_Error result = XML_ERROR_NONE; + + while (1) { + if (! parser->m_openAttributeEntities) { + result = appendAttributeValue(parser, enc, isCdata, next, end, pool, + account, &next); + } else { + OPEN_INTERNAL_ENTITY *const openEntity = parser->m_openAttributeEntities; + if (! openEntity) + return XML_ERROR_UNEXPECTED_STATE; + + ENTITY *const entity = openEntity->entity; + const char *const textStart + = ((const char *)entity->textPtr) + entity->processed; + const char *const textEnd + = (const char *)(entity->textPtr + entity->textLen); + /* Set a safe default value in case 'next' does not get set */ + const char *nextInEntity = textStart; + if (entity->hasMore) { + result = appendAttributeValue( + parser, parser->m_internalEncoding, isCdata, textStart, textEnd, + pool, XML_ACCOUNT_ENTITY_EXPANSION, &nextInEntity); + if (result != XML_ERROR_NONE) + break; + // Check if entity is complete, if not, mark down how much of it is + // processed. A XML_SUSPENDED check here is not required as + // appendAttributeValue will never suspend the parser. + if (nextInEntity < textEnd) { + entity->processed + = (int)(nextInEntity - (const char *)entity->textPtr); + continue; + } + assert(nextInEntity == textEnd); + + // Entity is complete. We cannot close it here since we need to first + // process its possible inner entities (which are added to the + // m_openAttributeEntities during appendAttributeValue) + entity->hasMore = false; + continue; + } // End of entity processing, "if" block skips the rest + + // Remove fully processed openEntity from open entity list. +#if XML_GE == 1 + entityTrackingOnClose(parser, entity, __LINE__); +#endif + // openEntity is m_openAttributeEntities' head, since we set it at the + // start of this function and because we skipped appendAttributeValue call + // with hasMore set to false. This means we can directly remove the head + // of m_openAttributeEntities + assert(parser->m_openAttributeEntities == openEntity); + entity->open = false; + parser->m_openAttributeEntities = parser->m_openAttributeEntities->next; + + /* put openEntity back in list of free instances */ + openEntity->next = parser->m_freeEntities; + parser->m_freeEntities = openEntity; + } + + // Break if an error occurred or there is nothing left to process + if (result || (parser->m_openAttributeEntities == NULL && end == next)) { + break; + } + } + if (result) return result; if (! isCdata && poolLength(pool) && poolLastChar(pool) == 0x20) @@ -5996,7 +6915,7 @@ storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, static enum XML_Error appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, const char *ptr, const char *end, STRING_POOL *pool, - enum XML_Account account) { + enum XML_Account account, const char **nextPtr) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ #ifndef XML_DTD UNUSED_P(account); @@ -6014,6 +6933,9 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, #endif switch (tok) { case XML_TOK_NONE: + if (nextPtr) { + *nextPtr = next; + } return XML_ERROR_NONE; case XML_TOK_INVALID: if (enc == parser->m_encoding) @@ -6025,7 +6947,6 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, return XML_ERROR_INVALID_TOKEN; case XML_TOK_CHAR_REF: { XML_Char buf[XML_ENCODE_MAX]; - int i; int n = XmlCharRefNumber(enc, ptr); if (n < 0) { if (enc == parser->m_encoding) @@ -6045,10 +6966,9 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, * XmlEncode() is never passed a value it might return an * error for. */ - for (i = 0; i < n; i++) { - if (! poolAppendChar(pool, buf[i])) - return XML_ERROR_NO_MEMORY; - } + + if (! poolAppendChars(pool, buf, n)) + return XML_ERROR_NO_MEMORY; } break; case XML_TOK_DATA_CHARS: if (! poolAppend(pool, enc, ptr, next)) @@ -6056,7 +6976,7 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, break; case XML_TOK_TRAILING_CR: next = ptr + enc->minBytesPerChar; - /* fall through */ + EXPAT_FALLTHROUGH; case XML_TOK_ATTRIBUTE_VALUE_S: case XML_TOK_DATA_NEWLINE: if (! isCdata && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20)) @@ -6067,7 +6987,7 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, case XML_TOK_ENTITY_REF: { const XML_Char *name; ENTITY *entity; - char checkEntityDecl; + bool checkEntityDecl; XML_Char ch = (XML_Char)XmlPredefinedEntityName( enc, ptr + enc->minBytesPerChar, next - enc->minBytesPerChar); if (ch) { @@ -6110,8 +7030,11 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, } else if (! entity) { /* Cannot report skipped entity here - see comments on parser->m_skippedEntityHandler. - if (parser->m_skippedEntityHandler) + if (parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0); + afterHandler(parser); + } */ /* Cannot call the default handler because this would be out of sync with the call to the startElementHandler. @@ -6154,21 +7077,11 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, return XML_ERROR_ATTRIBUTE_EXTERNAL_ENTITY_REF; } else { enum XML_Error result; - const XML_Char *textEnd = entity->textPtr + entity->textLen; - entity->open = XML_TRUE; -#if XML_GE == 1 - entityTrackingOnOpen(parser, entity, __LINE__); -#endif - result = appendAttributeValue(parser, parser->m_internalEncoding, - isCdata, (const char *)entity->textPtr, - (const char *)textEnd, pool, - XML_ACCOUNT_ENTITY_EXPANSION); -#if XML_GE == 1 - entityTrackingOnClose(parser, entity, __LINE__); -#endif - entity->open = XML_FALSE; - if (result) - return result; + result = processEntity(parser, entity, XML_FALSE, ENTITY_ATTRIBUTE); + if ((result == XML_ERROR_NONE) && (nextPtr != NULL)) { + *nextPtr = next; + } + return result; } } break; default: @@ -6197,7 +7110,7 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, static enum XML_Error storeEntityValue(XML_Parser parser, const ENCODING *enc, const char *entityTextPtr, const char *entityTextEnd, - enum XML_Account account) { + enum XML_Account account, const char **nextPtr) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ STRING_POOL *pool = &(dtd->entityValuePool); enum XML_Error result = XML_ERROR_NONE; @@ -6215,8 +7128,16 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, return XML_ERROR_NO_MEMORY; } + const char *next = entityTextPtr; + + /* Nothing to tokenize. */ + if (entityTextPtr >= entityTextEnd) { + result = XML_ERROR_NONE; + goto endEntityValue; + } + for (;;) { - const char *next + next = entityTextPtr; /* XmlEntityValueTok doesn't always set the last arg */ int tok = XmlEntityValueTok(enc, entityTextPtr, entityTextEnd, &next); @@ -6246,8 +7167,11 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, /* not a well-formedness error - see XML 1.0: WFC Entity Declared */ /* cannot report skipped entity here - see comments on parser->m_skippedEntityHandler - if (parser->m_skippedEntityHandler) + if (parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0); + afterHandler(parser); + } */ dtd->keepProcessing = dtd->standalone; goto endEntityValue; @@ -6260,34 +7184,29 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, } if (entity->systemId) { if (parser->m_externalEntityRefHandler) { - dtd->paramEntityRead = XML_FALSE; - entity->open = XML_TRUE; + dtd->paramEntityRead = false; + entity->open = true; entityTrackingOnOpen(parser, entity, __LINE__); - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, 0, entity->base, - entity->systemId, entity->publicId)) { + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, 0, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) { entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; result = XML_ERROR_EXTERNAL_ENTITY_HANDLING; goto endEntityValue; } entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; if (! dtd->paramEntityRead) dtd->keepProcessing = dtd->standalone; } else dtd->keepProcessing = dtd->standalone; } else { - entity->open = XML_TRUE; - entityTrackingOnOpen(parser, entity, __LINE__); - result = storeEntityValue( - parser, parser->m_internalEncoding, (const char *)entity->textPtr, - (const char *)(entity->textPtr + entity->textLen), - XML_ACCOUNT_ENTITY_EXPANSION); - entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; - if (result) - goto endEntityValue; + result = processEntity(parser, entity, XML_FALSE, ENTITY_VALUE); + goto endEntityValue; } break; } @@ -6309,17 +7228,15 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, break; case XML_TOK_TRAILING_CR: next = entityTextPtr + enc->minBytesPerChar; - /* fall through */ + EXPAT_FALLTHROUGH; case XML_TOK_DATA_NEWLINE: - if (pool->end == pool->ptr && ! poolGrow(pool)) { + if (! poolAppendChar(pool, 0xA)) { result = XML_ERROR_NO_MEMORY; goto endEntityValue; } - *(pool->ptr)++ = 0xA; break; case XML_TOK_CHAR_REF: { XML_Char buf[XML_ENCODE_MAX]; - int i; int n = XmlCharRefNumber(enc, entityTextPtr); if (n < 0) { if (enc == parser->m_encoding) @@ -6337,12 +7254,9 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, * XmlEncode() is never passed a value it might return an * error for. */ - for (i = 0; i < n; i++) { - if (pool->end == pool->ptr && ! poolGrow(pool)) { - result = XML_ERROR_NO_MEMORY; - goto endEntityValue; - } - *(pool->ptr)++ = buf[i]; + if (! poolAppendChars(pool, buf, n)) { + result = XML_ERROR_NO_MEMORY; + goto endEntityValue; } } break; case XML_TOK_PARTIAL: @@ -6375,6 +7289,81 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, # ifdef XML_DTD parser->m_prologState.inEntityValue = oldInEntityValue; # endif /* XML_DTD */ + // If 'nextPtr' is given, it should be updated during the processing + if (nextPtr != NULL) { + *nextPtr = next; + } + return result; +} + +static enum XML_Error +callStoreEntityValue(XML_Parser parser, const ENCODING *enc, + const char *entityTextPtr, const char *entityTextEnd, + enum XML_Account account) { + const char *next = entityTextPtr; + enum XML_Error result = XML_ERROR_NONE; + while (1) { + if (! parser->m_openValueEntities) { + result + = storeEntityValue(parser, enc, next, entityTextEnd, account, &next); + } else { + OPEN_INTERNAL_ENTITY *const openEntity = parser->m_openValueEntities; + if (! openEntity) + return XML_ERROR_UNEXPECTED_STATE; + + ENTITY *const entity = openEntity->entity; + const char *const textStart + = ((const char *)entity->textPtr) + entity->processed; + const char *const textEnd + = (const char *)(entity->textPtr + entity->textLen); + /* Set a safe default value in case 'next' does not get set */ + const char *nextInEntity = textStart; + if (entity->hasMore) { + result = storeEntityValue(parser, parser->m_internalEncoding, textStart, + textEnd, XML_ACCOUNT_ENTITY_EXPANSION, + &nextInEntity); + if (result != XML_ERROR_NONE) + break; + // Check if entity is complete, if not, mark down how much of it is + // processed. A XML_SUSPENDED check here is not required as + // appendAttributeValue will never suspend the parser. + if (textEnd != nextInEntity) { + entity->processed + = (int)(nextInEntity - (const char *)entity->textPtr); + continue; + } + + // Entity is complete. We cannot close it here since we need to first + // process its possible inner entities (which are added to the + // m_openValueEntities during storeEntityValue) + entity->hasMore = false; + continue; + } // End of entity processing, "if" block skips the rest + + // Remove fully processed openEntity from open entity list. +# if XML_GE == 1 + entityTrackingOnClose(parser, entity, __LINE__); +# endif + // openEntity is m_openValueEntities' head, since we set it at the + // start of this function and because we skipped storeEntityValue call + // with hasMore set to false. This means we can directly remove the head + // of m_openValueEntities + assert(parser->m_openValueEntities == openEntity); + entity->open = false; + parser->m_openValueEntities = parser->m_openValueEntities->next; + + /* put openEntity back in list of free instances */ + openEntity->next = parser->m_freeEntities; + parser->m_freeEntities = openEntity; + } + + // Break if an error occurred or there is nothing left to process + if (result + || (parser->m_openValueEntities == NULL && entityTextEnd == next)) { + break; + } + } + return result; } @@ -6395,6 +7384,11 @@ storeSelfEntityValue(XML_Parser parser, ENTITY *entity) { return XML_ERROR_NO_MEMORY; } + /* Detect and prevent signed integer overflow */ + if ((size_t)poolLength(pool) > (size_t)INT_MAX) { + poolDiscard(pool); + return XML_ERROR_NO_MEMORY; + } entity->textPtr = poolStart(pool); entity->textLen = (int)(poolLength(pool)); poolFinish(pool); @@ -6404,7 +7398,7 @@ storeSelfEntityValue(XML_Parser parser, ENTITY *entity) { #endif /* XML_GE == 0 */ -static void FASTCALL +static void normalizeLines(XML_Char *s) { XML_Char *p; for (;; s++) { @@ -6447,7 +7441,9 @@ reportProcessingInstruction(XML_Parser parser, const ENCODING *enc, if (! data) return 0; normalizeLines(data); + beforeHandler(parser); parser->m_processingInstructionHandler(parser->m_handlerArg, target, data); + afterHandler(parser); poolClear(&parser->m_tempPool); return 1; } @@ -6467,7 +7463,9 @@ reportComment(XML_Parser parser, const ENCODING *enc, const char *start, if (! data) return 0; normalizeLines(data); + beforeHandler(parser); parser->m_commentHandler(parser->m_handlerArg, data); + afterHandler(parser); poolClear(&parser->m_tempPool); return 1; } @@ -6508,15 +7506,20 @@ reportDefault(XML_Parser parser, const ENCODING *enc, const char *s, convert_res = XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd); *eventEndPP = s; + beforeHandler(parser); parser->m_defaultHandler(parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); + afterHandler(parser); *eventPP = s; } while ((convert_res != XML_CONVERT_COMPLETED) && (convert_res != XML_CONVERT_INPUT_INCOMPLETE)); - } else + } else { + beforeHandler(parser); parser->m_defaultHandler( parser->m_handlerArg, (const XML_Char *)s, (int)((const XML_Char *)end - (const XML_Char *)s)); + afterHandler(parser); + } } static int @@ -6526,56 +7529,61 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, if (value || isId) { /* The handling of default attributes gets messed up if we have a default which duplicates a non-default. */ - int i; - for (i = 0; i < type->nDefaultAtts; i++) - if (attId == type->defaultAtts[i].id) - return 1; + NAMED *const nameFound + = lookup(parser, &(type->defaultAttForName), attId->name, 0); + if (nameFound) + return 1; if (isId && ! type->idAtt && ! attId->xmlns) type->idAtt = attId; } if (type->nDefaultAtts == type->allocDefaultAtts) { - if (type->allocDefaultAtts == 0) { - type->allocDefaultAtts = 8; - type->defaultAtts = (DEFAULT_ATTRIBUTE *)MALLOC( - parser, type->allocDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); - if (! type->defaultAtts) { - type->allocDefaultAtts = 0; - return 0; - } - } else { - DEFAULT_ATTRIBUTE *temp; - - /* Detect and prevent integer overflow */ - if (type->allocDefaultAtts > INT_MAX / 2) { - return 0; - } - - int count = type->allocDefaultAtts * 2; + /* Detect and prevent integer overflow */ + if (type->allocDefaultAtts > SIZE_MAX / 2) { + return 0; + } - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)count > (size_t)(-1) / sizeof(DEFAULT_ATTRIBUTE)) { - return 0; - } -#endif + size_t count = type->allocDefaultAtts * 2; + if (count == 0) { + count = 8; + } - temp = (DEFAULT_ATTRIBUTE *)REALLOC(parser, type->defaultAtts, - (count * sizeof(DEFAULT_ATTRIBUTE))); - if (temp == NULL) - return 0; - type->allocDefaultAtts = count; - type->defaultAtts = temp; + /* Detect and prevent integer overflow. */ + if (count > SIZE_MAX / sizeof(DEFAULT_ATTRIBUTE)) { + return 0; } + + DEFAULT_ATTRIBUTE *const temp = REALLOC( + parser, type->defaultAtts, (count * sizeof(DEFAULT_ATTRIBUTE))); + if (temp == NULL) + return 0; + type->allocDefaultAtts = count; + type->defaultAtts = temp; } att = type->defaultAtts + type->nDefaultAtts; att->id = attId; att->value = value; att->isCdata = isCdata; if (! isCdata) - attId->maybeTokenized = XML_TRUE; + attId->maybeTokenized = true; + + NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute + = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup( + parser, &(type->defaultAttForName), attId->name, + sizeof(NAME_AND_DEFAULT_ATTRIBUTE)); + if (! nameAndDefaultAttribute) + return 0; + + assert(nameAndDefaultAttribute->name == attId->name); + + // NOTE: The XML 1.0r4 spec says: + // "When more than one definition is provided for the same attribute of a + // given element type, the first declaration is binding and later + // declarations are ignored." + if (! nameAndDefaultAttribute->initialized) { + nameAndDefaultAttribute->attIndex = type->nDefaultAtts; + nameAndDefaultAttribute->initialized = true; + } + type->nDefaultAtts += 1; return 1; } @@ -6641,17 +7649,18 @@ getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start, else id->prefix = (PPREFIX *)lookup(parser, &dtd->prefixes, name + 6, sizeof(PPREFIX)); - id->xmlns = XML_TRUE; + id->xmlns = true; } else { int i; for (i = 0; name[i]; i++) { + /* Detect and prevent signed integer overflow */ + if (i == INT_MAX) { + return NULL; + } /* attributes without prefix are *not* in the default namespace */ if (name[i] == XML_T(ASCII_COLON)) { - int j; - for (j = 0; j < i; j++) { - if (! poolAppendChar(&dtd->pool, name[j])) - return NULL; - } + if (! poolAppendChars(&dtd->pool, name, i)) + return NULL; if (! poolAppendChar(&dtd->pool, XML_T('\0'))) return NULL; id->prefix = (PPREFIX *)lookup(parser, &dtd->prefixes, @@ -6679,46 +7688,39 @@ getContext(XML_Parser parser) { XML_Bool needSep = XML_FALSE; if (dtd->defaultPrefix.binding) { - int i; - int len; if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS))) return NULL; - len = dtd->defaultPrefix.binding->uriLen; + size_t len = dtd->defaultPrefix.binding->uriLen; if (parser->m_namespaceSeparator) len--; - for (i = 0; i < len; i++) { - if (! poolAppendChar(&parser->m_tempPool, - dtd->defaultPrefix.binding->uri[i])) { - /* Because of memory caching, I don't believe this line can be - * executed. - * - * This is part of a loop copying the default prefix binding - * URI into the parser's temporary string pool. Previously, - * that URI was copied into the same string pool, with a - * terminating NUL character, as part of setContext(). When - * the pool was cleared, that leaves a block definitely big - * enough to hold the URI on the free block list of the pool. - * The URI copy in getContext() therefore cannot run out of - * memory. - * - * If the pool is used between the setContext() and - * getContext() calls, the worst it can do is leave a bigger - * block on the front of the free list. Given that this is - * all somewhat inobvious and program logic can be changed, we - * don't delete the line but we do exclude it from the test - * coverage statistics. - */ - return NULL; /* LCOV_EXCL_LINE */ - } + if (! poolAppendChars(&parser->m_tempPool, dtd->defaultPrefix.binding->uri, + len)) { + /* Because of memory caching, I don't believe this line can be + * executed. + * + * This is part of a loop copying the default prefix binding + * URI into the parser's temporary string pool. Previously, + * that URI was copied into the same string pool, with a + * terminating NUL character, as part of setContext(). When + * the pool was cleared, that leaves a block definitely big + * enough to hold the URI on the free block list of the pool. + * The URI copy in getContext() therefore cannot run out of + * memory. + * + * If the pool is used between the setContext() and + * getContext() calls, the worst it can do is leave a bigger + * block on the front of the free list. Given that this is + * all somewhat inobvious and program logic can be changed, we + * don't delete the line but we do exclude it from the test + * coverage statistics. + */ + return NULL; /* LCOV_EXCL_LINE */ } needSep = XML_TRUE; } hashTableIterInit(&iter, &(dtd->prefixes)); for (;;) { - int i; - int len; - const XML_Char *s; PPREFIX *prefix = (PPREFIX *)hashTableIterNext(&iter); if (! prefix) break; @@ -6733,23 +7735,21 @@ getContext(XML_Parser parser) { } if (needSep && ! poolAppendChar(&parser->m_tempPool, CONTEXT_SEP)) return NULL; - for (s = prefix->name; *s; s++) - if (! poolAppendChar(&parser->m_tempPool, *s)) - return NULL; + if (! poolAppendChars(&parser->m_tempPool, prefix->name, + xcslen(prefix->name))) + return NULL; if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS))) return NULL; - len = prefix->binding->uriLen; + size_t len = prefix->binding->uriLen; if (parser->m_namespaceSeparator) len--; - for (i = 0; i < len; i++) - if (! poolAppendChar(&parser->m_tempPool, prefix->binding->uri[i])) - return NULL; + if (! poolAppendChars(&parser->m_tempPool, prefix->binding->uri, len)) + return NULL; needSep = XML_TRUE; } hashTableIterInit(&iter, &(dtd->generalEntities)); for (;;) { - const XML_Char *s; ENTITY *e = (ENTITY *)hashTableIterNext(&iter); if (! e) break; @@ -6757,9 +7757,8 @@ getContext(XML_Parser parser) { continue; if (needSep && ! poolAppendChar(&parser->m_tempPool, CONTEXT_SEP)) return NULL; - for (s = e->name; *s; s++) - if (! poolAppendChar(&parser->m_tempPool, *s)) - return 0; + if (! poolAppendChars(&parser->m_tempPool, e->name, xcslen(e->name))) + return NULL; needSep = XML_TRUE; } @@ -6785,7 +7784,7 @@ setContext(XML_Parser parser, const XML_Char *context) { e = (ENTITY *)lookup(parser, &dtd->generalEntities, poolStart(&parser->m_tempPool), 0); if (e) - e->open = XML_TRUE; + e->open = true; if (*s != XML_T('\0')) s++; context = s; @@ -6797,16 +7796,24 @@ setContext(XML_Parser parser, const XML_Char *context) { else { if (! poolAppendChar(&parser->m_tempPool, XML_T('\0'))) return XML_FALSE; - prefix - = (PPREFIX *)lookup(parser, &dtd->prefixes, - poolStart(&parser->m_tempPool), sizeof(PPREFIX)); - if (! prefix) + const XML_Char *const prefixName = poolCopyStringNoFinish( + &dtd->pool, poolStart(&parser->m_tempPool)); + if (! prefixName) { return XML_FALSE; - if (prefix->name == poolStart(&parser->m_tempPool)) { - prefix->name = poolCopyString(&dtd->pool, prefix->name); - if (! prefix->name) - return XML_FALSE; } + + prefix = (PPREFIX *)lookup(parser, &dtd->prefixes, prefixName, + sizeof(PPREFIX)); + + const bool prefixNameUsed = prefix && prefix->name == prefixName; + if (prefixNameUsed) + poolFinish(&dtd->pool); + else + poolDiscard(&dtd->pool); + + if (! prefix) + return XML_FALSE; + poolDiscard(&parser->m_tempPool); } for (context = s + 1; *context != CONTEXT_SEP && *context != XML_T('\0'); @@ -6832,7 +7839,7 @@ setContext(XML_Parser parser, const XML_Char *context) { return XML_TRUE; } -static void FASTCALL +static void normalizePublicId(XML_Char *publicId) { XML_Char *p = publicId; XML_Char *s; @@ -6854,25 +7861,26 @@ normalizePublicId(XML_Char *publicId) { } static DTD * -dtdCreate(const XML_Memory_Handling_Suite *ms) { - DTD *p = ms->malloc_fcn(sizeof(DTD)); +dtdCreate(XML_Parser parser) { + DTD *p = MALLOC(parser, sizeof(DTD)); if (p == NULL) return p; - poolInit(&(p->pool), ms); - poolInit(&(p->entityValuePool), ms); - hashTableInit(&(p->generalEntities), ms); - hashTableInit(&(p->elementTypes), ms); - hashTableInit(&(p->attributeIds), ms); - hashTableInit(&(p->prefixes), ms); + poolInit(&(p->pool), parser); + poolInit(&(p->entityValuePool), parser); + hashTableInit(&(p->generalEntities), parser); + hashTableInit(&(p->elementTypes), parser); + hashTableInit(&(p->attributeIds), parser); + hashTableInit(&(p->prefixes), parser); #ifdef XML_DTD - p->paramEntityRead = XML_FALSE; - hashTableInit(&(p->paramEntities), ms); + p->paramEntityRead = false; + hashTableInit(&(p->paramEntities), parser); #endif /* XML_DTD */ p->defaultPrefix.name = NULL; p->defaultPrefix.binding = NULL; - p->in_eldecl = XML_FALSE; + p->in_eldecl = false; p->scaffIndex = NULL; + p->scaffIndexSize = 0; p->scaffold = NULL; p->scaffLevel = 0; p->scaffSize = 0; @@ -6886,19 +7894,19 @@ dtdCreate(const XML_Memory_Handling_Suite *ms) { } static void -dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms) { +dtdReset(DTD *p, XML_Parser parser) { HASH_TABLE_ITER iter; hashTableIterInit(&iter, &(p->elementTypes)); for (;;) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; - if (e->allocDefaultAtts != 0) - ms->free_fcn(e->defaultAtts); + hashTableDestroy(&(e->defaultAttForName)); + FREE(parser, e->defaultAtts); } hashTableClear(&(p->generalEntities)); #ifdef XML_DTD - p->paramEntityRead = XML_FALSE; + p->paramEntityRead = false; hashTableClear(&(p->paramEntities)); #endif /* XML_DTD */ hashTableClear(&(p->elementTypes)); @@ -6909,11 +7917,12 @@ dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms) { p->defaultPrefix.name = NULL; p->defaultPrefix.binding = NULL; - p->in_eldecl = XML_FALSE; + p->in_eldecl = false; - ms->free_fcn(p->scaffIndex); + FREE(parser, p->scaffIndex); p->scaffIndex = NULL; - ms->free_fcn(p->scaffold); + p->scaffIndexSize = 0; + FREE(parser, p->scaffold); p->scaffold = NULL; p->scaffLevel = 0; @@ -6927,15 +7936,15 @@ dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms) { } static void -dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms) { +dtdDestroy(DTD *p, XML_Bool isDocEntity, XML_Parser parser) { HASH_TABLE_ITER iter; hashTableIterInit(&iter, &(p->elementTypes)); for (;;) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; - if (e->allocDefaultAtts != 0) - ms->free_fcn(e->defaultAtts); + hashTableDestroy(&(e->defaultAttForName)); + FREE(parser, e->defaultAtts); } hashTableDestroy(&(p->generalEntities)); #ifdef XML_DTD @@ -6947,10 +7956,10 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms) { poolDestroy(&(p->pool)); poolDestroy(&(p->entityValuePool)); if (isDocEntity) { - ms->free_fcn(p->scaffIndex); - ms->free_fcn(p->scaffold); + FREE(parser, p->scaffIndex); + FREE(parser, p->scaffold); } - ms->free_fcn(p); + FREE(parser, p); } /* Do a deep copy of the DTD. Return 0 for out of memory, non-zero otherwise. @@ -6958,7 +7967,7 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms) { */ static int dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, - const XML_Memory_Handling_Suite *ms) { + XML_Parser parser) { HASH_TABLE_ITER iter; /* Copy the prefix table. */ @@ -7014,7 +8023,6 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, hashTableIterInit(&iter, &(oldDtd->elementTypes)); for (;;) { - int i; ELEMENT_TYPE *newE; const XML_Char *name; const ELEMENT_TYPE *oldE = (ELEMENT_TYPE *)hashTableIterNext(&iter); @@ -7027,19 +8035,17 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, sizeof(ELEMENT_TYPE)); if (! newE) return 0; + + if (! newE->defaultAttForName.parser) + hashTableInit(&(newE->defaultAttForName), parser); + if (oldE->nDefaultAtts) { - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((size_t)oldE->nDefaultAtts - > ((size_t)(-1) / sizeof(DEFAULT_ATTRIBUTE))) { + /* Detect and prevent integer overflow. */ + if (oldE->nDefaultAtts > SIZE_MAX / sizeof(DEFAULT_ATTRIBUTE)) { return 0; } -#endif newE->defaultAtts - = ms->malloc_fcn(oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); + = MALLOC(parser, oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); if (! newE->defaultAtts) { return 0; } @@ -7051,9 +8057,10 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, if (oldE->prefix) newE->prefix = (PPREFIX *)lookup(oldParser, &(newDtd->prefixes), oldE->prefix->name, 0); - for (i = 0; i < newE->nDefaultAtts; i++) { + for (size_t i = 0; i < newE->nDefaultAtts; i++) { + const XML_Char *const attributeName = oldE->defaultAtts[i].id->name; newE->defaultAtts[i].id = (ATTRIBUTE_ID *)lookup( - oldParser, &(newDtd->attributeIds), oldE->defaultAtts[i].id->name, 0); + oldParser, &(newDtd->attributeIds), attributeName, 0); newE->defaultAtts[i].isCdata = oldE->defaultAtts[i].isCdata; if (oldE->defaultAtts[i].value) { newE->defaultAtts[i].value @@ -7062,6 +8069,23 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, return 0; } else newE->defaultAtts[i].value = NULL; + + NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute + = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup( + parser, &(newE->defaultAttForName), attributeName, + sizeof(NAME_AND_DEFAULT_ATTRIBUTE)); + if (! nameAndDefaultAttribute) { + return 0; + } + + // NOTE: The XML 1.0r4 spec says: + // "When more than one definition is provided for the same attribute of a + // given element type, the first declaration is binding and later + // declarations are ignored." + if (! nameAndDefaultAttribute->initialized) { + nameAndDefaultAttribute->attIndex = i; + nameAndDefaultAttribute->initialized = true; + } } } @@ -7088,6 +8112,7 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, newDtd->scaffSize = oldDtd->scaffSize; newDtd->scaffLevel = oldDtd->scaffLevel; newDtd->scaffIndex = oldDtd->scaffIndex; + newDtd->scaffIndexSize = oldDtd->scaffIndexSize; return 1; } /* End dtdCopy */ @@ -7157,41 +8182,72 @@ copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, #define INIT_POWER 6 -static XML_Bool FASTCALL -keyeq(KEY s1, KEY s2) { - for (; *s1 == *s2; s1++, s2++) - if (*s1 == 0) - return XML_TRUE; - return XML_FALSE; +// Compares two strings `s1` and `s2` whereas: +// - `s2` is zero-terminated but +// - `s1` is made up of exactly (not just up to) `s1len` non-zero characters. +XML_NONTESTING_STATIC XML_Bool +keyeq(KEY s1, size_t s1len, KEY s2) { +#ifdef XML_UNICODE +# ifdef XML_UNICODE_WCHAR_T + return (wcsncmp(s1, s2, s1len) == 0 && s2[s1len] == L'\0') ? XML_TRUE + : XML_FALSE; +# else + for (; s1len > 0 && *s1 == *s2; s1len--, s1++, s2++) + ; /* no loop body! */ + return ((s1len == 0) && (*s2 == 0)) ? XML_TRUE : XML_FALSE; +# endif +#else + return (strncmp(s1, s2, s1len) == 0 && s2[s1len] == '\0') ? XML_TRUE + : XML_FALSE; +#endif } -static size_t +XML_NONTESTING_STATIC size_t keylen(KEY s) { - size_t len = 0; - for (; *s; s++, len++) - ; - return len; + return xcslen(s); } static void copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key) { - key->k[0] = 0; - key->k[1] = get_hash_secret_salt(parser); + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(! rootParser->m_parentParser); + + *key = rootParser->m_hash_secret_salt_128; } -static unsigned long FASTCALL -hash(XML_Parser parser, KEY s) { +static unsigned long +hash(XML_Parser parser, KEY s, size_t keyLen) { struct siphash state; struct sipkey key; (void)sip24_valid; copy_salt_to_sipkey(parser, &key); sip24_init(&state, &key); - sip24_update(&state, s, keylen(s) * sizeof(XML_Char)); + sip24_update(&state, s, keyLen * sizeof(XML_Char)); return (unsigned long)sip24_final(&state); } -static NAMED * -lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { +// Function `lookupWithLength` can be used to either… +// +// a) check whether an element with key `name` exists in the given hash table +// (read-only mode where `createSize == 0`) or +// +// b) check whether an element with key `name` exists in the given hash table +// *and* insert it if missing (i.e. read-write mode where `createSize != 0`. +// +// When inserting, a block of `createSize` number of bytes will be allocated +// and set to zero, and the resulting block of memory will be considered +// to start with a `NAMED` structure, and `->name = name;` is performed. +// The fact that all other bytes in the structure are initially zero can +// be used to tell cases "existed and found" and "newly inserted" apart +// with the structure returned. +// +// NOTE: Read-only lookup does not need zero-terminated keys but +// read-write mode does, because keys can be re-hashed later and the +// hash table does not store key length information. +// +XML_NONTESTING_STATIC NAMED * +lookupWithLength(XML_Parser parser, HASH_TABLE *table, KEY name, size_t nameLen, + size_t createSize) { size_t i; if (table->size == 0) { size_t tsize; @@ -7201,20 +8257,20 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { /* table->size is a power of 2 */ table->size = (size_t)1 << INIT_POWER; tsize = table->size * sizeof(NAMED *); - table->v = table->mem->malloc_fcn(tsize); + table->v = MALLOC(table->parser, tsize); if (! table->v) { table->size = 0; return NULL; } memset(table->v, 0, tsize); - i = hash(parser, name) & ((unsigned long)table->size - 1); + i = hash(parser, name, nameLen) & ((unsigned long)table->size - 1); } else { - unsigned long h = hash(parser, name); + unsigned long h = hash(parser, name, nameLen); unsigned long mask = (unsigned long)table->size - 1; unsigned char step = 0; i = h & mask; while (table->v[i]) { - if (keyeq(name, table->v[i]->name)) + if (keyeq(name, nameLen, table->v[i]->name)) return table->v[i]; if (! step) step = PROBE_STEP(h, mask, table->power); @@ -7236,18 +8292,19 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { unsigned long newMask = (unsigned long)newSize - 1; /* Detect and prevent integer overflow */ - if (newSize > (size_t)(-1) / sizeof(NAMED *)) { + if (newSize > SIZE_MAX / sizeof(NAMED *)) { return NULL; } size_t tsize = newSize * sizeof(NAMED *); - NAMED **newV = table->mem->malloc_fcn(tsize); + NAMED **newV = MALLOC(table->parser, tsize); if (! newV) return NULL; memset(newV, 0, tsize); for (i = 0; i < table->size; i++) if (table->v[i]) { - unsigned long newHash = hash(parser, table->v[i]->name); + KEY const key = table->v[i]->name; + unsigned long newHash = hash(parser, key, keylen(key)); size_t j = newHash & newMask; step = 0; while (newV[j]) { @@ -7257,7 +8314,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { } newV[j] = table->v[i]; } - table->mem->free_fcn(table->v); + FREE(table->parser, table->v); table->v = newV; table->power = newPower; table->size = newSize; @@ -7270,49 +8327,70 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { } } } - table->v[i] = table->mem->malloc_fcn(createSize); + assert(createSize >= sizeof(NAMED)); + table->v[i] = MALLOC(table->parser, createSize); if (! table->v[i]) return NULL; memset(table->v[i], 0, createSize); - table->v[i]->name = name; + table->v[i]->name = name; // NOTE: This requires and assumes zero termination! (table->used)++; return table->v[i]; } -static void FASTCALL +// Function `lookup` can be used to either… +// +// a) check whether an element with key `name` exists in the given hash table +// (read-only mode where `createSize == 0`) or +// +// b) check whether an element with key `name` exists in the given hash table +// *and* insert it if missing (i.e. read-write mode where `createSize != 0`. +// +// When inserting, a block of `createSize` number of bytes will be allocated +// and set to zero, and the resulting block of memory will be considered +// to start with a `NAMED` structure, and `->name = name;` is performed. +// The fact that all other bytes in the structure are initially zero can +// be used to tell cases "existed and found" and "newly inserted" apart +// with the structure returned. +// +XML_NONTESTING_STATIC NAMED * +lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { + return lookupWithLength(parser, table, name, keylen(name), createSize); +} + +XML_NONTESTING_STATIC void hashTableClear(HASH_TABLE *table) { size_t i; for (i = 0; i < table->size; i++) { - table->mem->free_fcn(table->v[i]); + FREE(table->parser, table->v[i]); table->v[i] = NULL; } table->used = 0; } -static void FASTCALL +XML_NONTESTING_STATIC void hashTableDestroy(HASH_TABLE *table) { size_t i; for (i = 0; i < table->size; i++) - table->mem->free_fcn(table->v[i]); - table->mem->free_fcn(table->v); + FREE(table->parser, table->v[i]); + FREE(table->parser, table->v); } -static void FASTCALL -hashTableInit(HASH_TABLE *p, const XML_Memory_Handling_Suite *ms) { +XML_NONTESTING_STATIC void +hashTableInit(HASH_TABLE *p, XML_Parser parser) { p->power = 0; p->size = 0; p->used = 0; p->v = NULL; - p->mem = ms; + p->parser = parser; } -static void FASTCALL +XML_NONTESTING_STATIC void hashTableIterInit(HASH_TABLE_ITER *iter, const HASH_TABLE *table) { iter->p = table->v; iter->end = iter->p ? iter->p + table->size : NULL; } -static NAMED *FASTCALL +XML_NONTESTING_STATIC NAMED * hashTableIterNext(HASH_TABLE_ITER *iter) { while (iter->p != iter->end) { NAMED *tem = *(iter->p)++; @@ -7322,17 +8400,17 @@ hashTableIterNext(HASH_TABLE_ITER *iter) { return NULL; } -static void FASTCALL -poolInit(STRING_POOL *pool, const XML_Memory_Handling_Suite *ms) { +static void +poolInit(STRING_POOL *pool, XML_Parser parser) { pool->blocks = NULL; pool->freeBlocks = NULL; pool->start = NULL; pool->ptr = NULL; pool->end = NULL; - pool->mem = ms; + pool->parser = parser; } -static void FASTCALL +static void poolClear(STRING_POOL *pool) { if (! pool->freeBlocks) pool->freeBlocks = pool->blocks; @@ -7351,18 +8429,18 @@ poolClear(STRING_POOL *pool) { pool->end = NULL; } -static void FASTCALL +static void poolDestroy(STRING_POOL *pool) { BLOCK *p = pool->blocks; while (p) { BLOCK *tem = p->next; - pool->mem->free_fcn(p); + FREE(pool->parser, p); p = tem; } p = pool->freeBlocks; while (p) { BLOCK *tem = p->next; - pool->mem->free_fcn(p); + FREE(pool->parser, p); p = tem; } } @@ -7384,17 +8462,32 @@ poolAppend(STRING_POOL *pool, const ENCODING *enc, const char *ptr, return pool->start; } -static const XML_Char *FASTCALL +static const XML_Char * poolCopyString(STRING_POOL *pool, const XML_Char *s) { - do { - if (! poolAppendChar(pool, *s)) - return NULL; - } while (*s++); + if (! poolAppendChars(pool, s, xcslen(s) + /*null terminator*/ 1)) + return NULL; s = pool->start; poolFinish(pool); return s; } +// A version of `poolCopyString` that does not call `poolFinish` +// and reverts any partial advancement upon failure. +static const XML_Char * +poolCopyStringNoFinish(STRING_POOL *pool, const XML_Char *s) { + const XML_Char *const original = s; + do { + if (! poolAppendChar(pool, *s)) { + // Revert any previously successful advancement + const ptrdiff_t advancedBy = s - original; + if (advancedBy > 0) + pool->ptr -= advancedBy; + return NULL; + } + } while (*s++); + return pool->start; +} + static const XML_Char * poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) { if (! pool->ptr && ! poolGrow(pool)) { @@ -7411,22 +8504,17 @@ poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) { */ return NULL; /* LCOV_EXCL_LINE */ } - for (; n > 0; --n, s++) { - if (! poolAppendChar(pool, *s)) - return NULL; - } + if (n > 0 && ! poolAppendChars(pool, s, n)) + return NULL; s = pool->start; poolFinish(pool); return s; } -static const XML_Char *FASTCALL +static const XML_Char * poolAppendString(STRING_POOL *pool, const XML_Char *s) { - while (*s) { - if (! poolAppendChar(pool, *s)) - return NULL; - s++; - } + if (! poolAppendChars(pool, s, xcslen(s))) + return NULL; return pool->start; } @@ -7435,9 +8523,8 @@ poolStoreString(STRING_POOL *pool, const ENCODING *enc, const char *ptr, const char *end) { if (! poolAppend(pool, enc, ptr, end)) return NULL; - if (pool->ptr == pool->end && ! poolGrow(pool)) + if (! poolAppendChar(pool, 0)) return NULL; - *(pool->ptr)++ = 0; return pool->start; } @@ -7469,10 +8556,10 @@ poolBytesToAllocateFor(int blockSize) { } } -static XML_Bool FASTCALL +static XML_Bool poolGrow(STRING_POOL *pool) { if (pool->freeBlocks) { - if (pool->start == 0) { + if (pool->start == NULL) { pool->blocks = pool->freeBlocks; pool->freeBlocks = pool->freeBlocks->next; pool->blocks->next = NULL; @@ -7488,7 +8575,7 @@ poolGrow(STRING_POOL *pool) { pool->freeBlocks = tem; memcpy(pool->blocks->s, pool->start, (pool->end - pool->start) * sizeof(XML_Char)); - pool->ptr = pool->blocks->s + (pool->ptr - pool->start); + pool->ptr = pool->blocks->s + EXPAT_SAFE_PTR_DIFF(pool->ptr, pool->start); pool->start = pool->blocks->s; pool->end = pool->start + pool->blocks->size; return XML_TRUE; @@ -7501,7 +8588,8 @@ poolGrow(STRING_POOL *pool) { /* NOTE: Needs to be calculated prior to calling `realloc` to avoid dangling pointers: */ - const ptrdiff_t offsetInsideBlock = pool->ptr - pool->start; + const ptrdiff_t offsetInsideBlock + = EXPAT_SAFE_PTR_DIFF(pool->ptr, pool->start); if (blockSize < 0) { /* This condition traps a situation where either more than @@ -7517,8 +8605,7 @@ poolGrow(STRING_POOL *pool) { if (bytesToAllocate == 0) return XML_FALSE; - temp = (BLOCK *)pool->mem->realloc_fcn(pool->blocks, - (unsigned)bytesToAllocate); + temp = REALLOC(pool->parser, pool->blocks, bytesToAllocate); if (temp == NULL) return XML_FALSE; pool->blocks = temp; @@ -7558,43 +8645,58 @@ poolGrow(STRING_POOL *pool) { if (bytesToAllocate == 0) return XML_FALSE; - tem = pool->mem->malloc_fcn(bytesToAllocate); + tem = MALLOC(pool->parser, bytesToAllocate); if (! tem) return XML_FALSE; tem->size = blockSize; tem->next = pool->blocks; pool->blocks = tem; if (pool->ptr != pool->start) - memcpy(tem->s, pool->start, (pool->ptr - pool->start) * sizeof(XML_Char)); - pool->ptr = tem->s + (pool->ptr - pool->start); + memcpy(tem->s, pool->start, + EXPAT_SAFE_PTR_DIFF(pool->ptr, pool->start) * sizeof(XML_Char)); + pool->ptr = tem->s + EXPAT_SAFE_PTR_DIFF(pool->ptr, pool->start); pool->start = tem->s; pool->end = tem->s + blockSize; } return XML_TRUE; } -static int FASTCALL +static bool +poolGrowUntil(STRING_POOL *pool, size_t needed) { + for (;;) { + const size_t available = pool->end - pool->ptr; + if (available >= needed) { + return true; + } + if (! poolGrow(pool)) { + return false; + } + } +} + +static int nextScaffoldPart(XML_Parser parser) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ CONTENT_SCAFFOLD *me; int next; if (! dtd->scaffIndex) { - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if (parser->m_groupSize > ((size_t)(-1) / sizeof(int))) { + /* Detect and prevent integer overflow. */ + if (parser->m_groupSize > SIZE_MAX / sizeof(int)) { return -1; } -#endif - dtd->scaffIndex = (int *)MALLOC(parser, parser->m_groupSize * sizeof(int)); + dtd->scaffIndex = MALLOC(parser, parser->m_groupSize * sizeof(int)); if (! dtd->scaffIndex) return -1; + dtd->scaffIndexSize = parser->m_groupSize; dtd->scaffIndex[0] = 0; } + // Will casting to int be safe further down? + if (dtd->scaffCount > INT_MAX) { + return -1; + } + if (dtd->scaffCount >= dtd->scaffSize) { CONTENT_SCAFFOLD *temp; if (dtd->scaffold) { @@ -7607,26 +8709,25 @@ nextScaffoldPart(XML_Parser parser) { * from -Wtype-limits on platforms where * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ #if UINT_MAX >= SIZE_MAX - if (dtd->scaffSize > (size_t)(-1) / 2u / sizeof(CONTENT_SCAFFOLD)) { + if (dtd->scaffSize > SIZE_MAX / 2u / sizeof(CONTENT_SCAFFOLD)) { return -1; } #endif - temp = (CONTENT_SCAFFOLD *)REALLOC( - parser, dtd->scaffold, dtd->scaffSize * 2 * sizeof(CONTENT_SCAFFOLD)); + temp = REALLOC(parser, dtd->scaffold, + dtd->scaffSize * 2 * sizeof(CONTENT_SCAFFOLD)); if (temp == NULL) return -1; dtd->scaffSize *= 2; } else { - temp = (CONTENT_SCAFFOLD *)MALLOC(parser, INIT_SCAFFOLD_ELEMENTS - * sizeof(CONTENT_SCAFFOLD)); + temp = MALLOC(parser, INIT_SCAFFOLD_ELEMENTS * sizeof(CONTENT_SCAFFOLD)); if (temp == NULL) return -1; dtd->scaffSize = INIT_SCAFFOLD_ELEMENTS; } dtd->scaffold = temp; } - next = dtd->scaffCount++; + next = (int)dtd->scaffCount++; me = &dtd->scaffold[next]; if (dtd->scaffLevel) { CONTENT_SCAFFOLD *parent @@ -7658,22 +8759,25 @@ build_model(XML_Parser parser) { * from -Wtype-limits on platforms where * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ #if UINT_MAX >= SIZE_MAX - if (dtd->scaffCount > (size_t)(-1) / sizeof(XML_Content)) { + if (dtd->scaffCount > SIZE_MAX / sizeof(XML_Content)) { return NULL; } - if (dtd->contentStringLen > (size_t)(-1) / sizeof(XML_Char)) { + if (dtd->contentStringLen > SIZE_MAX / sizeof(XML_Char)) { return NULL; } #endif if (dtd->scaffCount * sizeof(XML_Content) - > (size_t)(-1) - dtd->contentStringLen * sizeof(XML_Char)) { + > SIZE_MAX - dtd->contentStringLen * sizeof(XML_Char)) { return NULL; } const size_t allocsize = (dtd->scaffCount * sizeof(XML_Content) + (dtd->contentStringLen * sizeof(XML_Char))); - ret = (XML_Content *)MALLOC(parser, allocsize); + // NOTE: We are avoiding MALLOC(..) here to so that + // applications that are not using XML_FreeContentModel but plain + // free(..) or .free_fcn() to free the content model's memory are safe. + ret = parser->m_mem.malloc_fcn(allocsize); if (! ret) return NULL; @@ -7745,12 +8849,21 @@ build_model(XML_Parser parser) { const XML_Char *src; dest->name = str; src = dtd->scaffold[src_node].name; - for (;;) { - *str++ = *src; - if (! *src) - break; - src++; + + const size_t nameLen = xcslen(src) + /* null terminator*/ 1; + + // Detect and prevent integer overflow + if (nameLen > SIZE_MAX / sizeof(XML_Char)) { + // NOTE: We are avoiding FREE(..) here because the model + // is not being allocated with MALLOC(..) but with plain + // .malloc_fcn(..). + parser->m_mem.free_fcn(ret); + return NULL; } + + memcpy(str, src, nameLen * sizeof(XML_Char)); + str += nameLen; + dest->numchildren = 0; dest->children = NULL; } else { @@ -7783,6 +8896,8 @@ getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, sizeof(ELEMENT_TYPE)); if (! ret) return NULL; + if (! ret->defaultAttForName.parser) + hashTableInit(&(ret->defaultAttForName), getRootParserOf(parser, NULL)); if (ret->name != name) poolDiscard(&dtd->pool); else { @@ -7794,23 +8909,25 @@ getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, } static XML_Char * -copyString(const XML_Char *s, const XML_Memory_Handling_Suite *memsuite) { - size_t charsRequired = 0; - XML_Char *result; - +copyString(const XML_Char *s, XML_Parser parser) { /* First determine how long the string is */ - while (s[charsRequired] != 0) { - charsRequired++; - } - /* Include the terminator */ - charsRequired++; + const size_t charsRequired = xcslen(s) + /*null terminator*/ 1; + + /* Detect and prevent integer overflow */ + if (charsRequired > SIZE_MAX / sizeof(XML_Char)) + return NULL; + + const size_t bytesRequired = charsRequired * sizeof(XML_Char); /* Now allocate space for the copy */ - result = memsuite->malloc_fcn(charsRequired * sizeof(XML_Char)); + XML_Char *const result = MALLOC(parser, bytesRequired); + if (result == NULL) return NULL; + /* Copy the original into place */ - memcpy(result, s, charsRequired * sizeof(XML_Char)); + memcpy(result, s, bytesRequired); + return result; } @@ -7825,10 +8942,10 @@ accountingGetCurrentAmplification(XML_Parser rootParser) { + rootParser->m_accounting.countBytesIndirect; const float amplificationFactor = rootParser->m_accounting.countBytesDirect - ? (countBytesOutput + ? ((float)countBytesOutput / (float)(rootParser->m_accounting.countBytesDirect)) - : ((lenOfShortestInclude - + rootParser->m_accounting.countBytesIndirect) + : ((float)(lenOfShortestInclude + + rootParser->m_accounting.countBytesIndirect) / (float)lenOfShortestInclude); assert(! rootParser->m_parentParser); return amplificationFactor; @@ -7977,15 +9094,25 @@ entityTrackingReportStats(XML_Parser rootParser, ENTITY *entity, const char *const entityName = entity->name; # endif + const bool limitingWanted = rootParser->m_entity_stats.debugLevel < 2; + const int maxLimitedDepth = 10; // somewhat arbitrary + const int candidateIndentDepth + = (int)rootParser->m_entity_stats.currentDepth - 1; + const bool limitingNeeded + = limitingWanted && (candidateIndentDepth > maxLimitedDepth); + const char *const ellipisOrEmpty = limitingNeeded ? " [..] " : ""; + const int indentDepth + = limitingNeeded ? (maxLimitedDepth - /* make space for ellipis */ 2) + : candidateIndentDepth; + fprintf( stderr, - "expat: Entities(%p): Count %9u, depth %2u/%2u %*s%s%s; %s length %d (xmlparse.c:%d)\n", + "expat: Entities(%p): Count %9u, depth %2u/%2u %*s%s%s%s; %s length %d (xmlparse.c:%d)\n", (void *)rootParser, rootParser->m_entity_stats.countEverOpened, rootParser->m_entity_stats.currentDepth, - rootParser->m_entity_stats.maximumDepthSeen, - (rootParser->m_entity_stats.currentDepth - 1) * 2, "", - entity->is_param ? "%" : "&", entityName, action, entity->textLen, - sourceLine); + rootParser->m_entity_stats.maximumDepthSeen, indentDepth * 2, "", + ellipisOrEmpty, entity->is_param ? "%" : "&", entityName, action, + entity->textLen, sourceLine); } static void @@ -8012,6 +9139,8 @@ entityTrackingOnClose(XML_Parser originParser, ENTITY *entity, int sourceLine) { rootParser->m_entity_stats.currentDepth--; } +#endif /* XML_GE == 1 */ + static XML_Parser getRootParserOf(XML_Parser parser, unsigned int *outLevelDiff) { XML_Parser rootParser = parser; @@ -8027,6 +9156,8 @@ getRootParserOf(XML_Parser parser, unsigned int *outLevelDiff) { return rootParser; } +#if XML_GE == 1 + const char * unsignedCharToPrintable(unsigned char c) { switch (c) { @@ -8542,11 +9673,13 @@ unsignedCharToPrintable(unsigned char c) { return "\\xFE"; case 255: return "\\xFF"; + // LCOV_EXCL_START default: assert(0); /* never gets here */ return "dead code"; } assert(0); /* never gets here */ + // LCOV_EXCL_STOP } #endif /* XML_GE == 1 */ diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c index 2c48bf40867953..dbf4a2d1b8f708 100644 --- a/Modules/expat/xmlrole.c +++ b/Modules/expat/xmlrole.c @@ -12,10 +12,11 @@ Copyright (c) 2002-2006 Karl Waclawek Copyright (c) 2002-2003 Fred L. Drake, Jr. Copyright (c) 2005-2009 Steven Solie - Copyright (c) 2016-2023 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2017 Rhodri James Copyright (c) 2019 David Loffredo Copyright (c) 2021 Donghee Na + Copyright (c) 2025 Alfonso Gregory Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -36,6 +37,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #include "expat_config.h" @@ -46,7 +49,6 @@ # include "winconfig.h" #endif -#include "expat_external.h" #include "internal.h" #include "xmlrole.h" #include "ascii.h" @@ -119,9 +121,8 @@ static const char KW_SYSTEM[] # define setTopLevel(state) ((state)->handler = internalSubset) #endif /* not XML_DTD */ -typedef int PTRCALL PROLOG_HANDLER(PROLOG_STATE *state, int tok, - const char *ptr, const char *end, - const ENCODING *enc); +typedef int PROLOG_HANDLER(PROLOG_STATE *state, int tok, const char *ptr, + const char *end, const ENCODING *enc); static PROLOG_HANDLER prolog0, prolog1, prolog2, doctype0, doctype1, doctype2, doctype3, doctype4, doctype5, internalSubset, entity0, entity1, entity2, @@ -135,9 +136,9 @@ static PROLOG_HANDLER prolog0, prolog1, prolog2, doctype0, doctype1, doctype2, #endif /* XML_DTD */ declClose, error; -static int FASTCALL common(PROLOG_STATE *state, int tok); +static int common(PROLOG_STATE *state, int tok); -static int PTRCALL +static int prolog0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -168,7 +169,7 @@ prolog0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int prolog1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -200,7 +201,7 @@ prolog1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int prolog2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -220,7 +221,7 @@ prolog2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int doctype0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -237,7 +238,7 @@ doctype0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int doctype1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -263,7 +264,7 @@ doctype1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int doctype2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -279,7 +280,7 @@ doctype2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int doctype3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -295,7 +296,7 @@ doctype3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int doctype4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -314,7 +315,7 @@ doctype4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int doctype5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -330,7 +331,7 @@ doctype5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int internalSubset(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -375,7 +376,7 @@ internalSubset(PROLOG_STATE *state, int tok, const char *ptr, const char *end, #ifdef XML_DTD -static int PTRCALL +static int externalSubset0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { state->handler = externalSubset1; @@ -384,7 +385,7 @@ externalSubset0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return externalSubset1(state, tok, ptr, end, enc); } -static int PTRCALL +static int externalSubset1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -412,7 +413,7 @@ externalSubset1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, #endif /* XML_DTD */ -static int PTRCALL +static int entity0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -431,7 +432,7 @@ entity0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -447,7 +448,7 @@ entity1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -471,7 +472,7 @@ entity2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -487,7 +488,7 @@ entity3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -503,7 +504,7 @@ entity4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -522,7 +523,7 @@ entity5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity6(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -539,7 +540,7 @@ entity6(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity7(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -563,7 +564,7 @@ entity7(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity8(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -579,7 +580,7 @@ entity8(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity9(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -595,7 +596,7 @@ entity9(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity10(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -611,7 +612,7 @@ entity10(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int notation0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -627,7 +628,7 @@ notation0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int notation1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -647,7 +648,7 @@ notation1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int notation2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -663,7 +664,7 @@ notation2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int notation3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -680,7 +681,7 @@ notation3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int notation4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -700,7 +701,7 @@ notation4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -717,7 +718,7 @@ attlist0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -737,7 +738,7 @@ attlist1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -767,7 +768,7 @@ attlist2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -785,7 +786,7 @@ attlist3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -804,7 +805,7 @@ attlist4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -820,7 +821,7 @@ attlist5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist6(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -836,7 +837,7 @@ attlist6(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist7(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -856,7 +857,7 @@ attlist7(PROLOG_STATE *state, int tok, const char *ptr, const char *end, } /* default value */ -static int PTRCALL +static int attlist8(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -886,7 +887,7 @@ attlist8(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist9(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -902,7 +903,7 @@ attlist9(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -919,7 +920,7 @@ element0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -945,7 +946,7 @@ element1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -979,7 +980,7 @@ element2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1003,7 +1004,7 @@ element3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1020,7 +1021,7 @@ element4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1040,7 +1041,7 @@ element5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element6(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1069,7 +1070,7 @@ element6(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element7(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1118,7 +1119,7 @@ element7(PROLOG_STATE *state, int tok, const char *ptr, const char *end, #ifdef XML_DTD -static int PTRCALL +static int condSect0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -1138,7 +1139,7 @@ condSect0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int condSect1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1155,7 +1156,7 @@ condSect1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int condSect2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1173,7 +1174,7 @@ condSect2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, #endif /* XML_DTD */ -static int PTRCALL +static int declClose(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1209,7 +1210,7 @@ declClose(PROLOG_STATE *state, int tok, const char *ptr, const char *end, * * LCOV_EXCL_START */ -static int PTRCALL +static int error(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(state); @@ -1221,7 +1222,7 @@ error(PROLOG_STATE *state, int tok, const char *ptr, const char *end, } /* LCOV_EXCL_STOP */ -static int FASTCALL +static int common(PROLOG_STATE *state, int tok) { #ifdef XML_DTD if (! state->documentEntity && tok == XML_TOK_PARAM_ENTITY_REF) diff --git a/Modules/expat/xmlrole.h b/Modules/expat/xmlrole.h index a7904274c91d4e..f4aba4079f5cdf 100644 --- a/Modules/expat/xmlrole.h +++ b/Modules/expat/xmlrole.h @@ -10,7 +10,7 @@ Copyright (c) 2000 Clark Cooper Copyright (c) 2002 Karl Waclawek Copyright (c) 2002 Fred L. Drake, Jr. - Copyright (c) 2017-2024 Sebastian Pipping + Copyright (c) 2017-2026 Sebastian Pipping Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -31,22 +31,14 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifndef XmlRole_INCLUDED -#define XmlRole_INCLUDED 1 - -#ifdef __VMS -/* 0 1 2 3 0 1 2 3 - 1234567890123456789012345678901 1234567890123456789012345678901 */ -# define XmlPrologStateInitExternalEntity XmlPrologStateInitExternalEnt -#endif +# define XmlRole_INCLUDED 1 -#include "xmltok.h" - -#ifdef __cplusplus -extern "C" { -#endif +# include "xmltok.h" enum { XML_ROLE_ERROR = -1, @@ -107,36 +99,32 @@ enum { XML_ROLE_CONTENT_ELEMENT_PLUS, XML_ROLE_PI, XML_ROLE_COMMENT, -#ifdef XML_DTD +# ifdef XML_DTD XML_ROLE_TEXT_DECL, XML_ROLE_IGNORE_SECT, XML_ROLE_INNER_PARAM_ENTITY_REF, -#endif /* XML_DTD */ +# endif /* XML_DTD */ XML_ROLE_PARAM_ENTITY_REF }; typedef struct prolog_state { - int(PTRCALL *handler)(struct prolog_state *state, int tok, const char *ptr, - const char *end, const ENCODING *enc); + int (*handler)(struct prolog_state *state, int tok, const char *ptr, + const char *end, const ENCODING *enc); unsigned level; int role_none; -#ifdef XML_DTD +# ifdef XML_DTD unsigned includeLevel; int documentEntity; int inEntityValue; -#endif /* XML_DTD */ +# endif /* XML_DTD */ } PROLOG_STATE; void XmlPrologStateInit(PROLOG_STATE *state); -#ifdef XML_DTD +# ifdef XML_DTD void XmlPrologStateInitExternalEntity(PROLOG_STATE *state); -#endif /* XML_DTD */ - -#define XmlTokenRole(state, tok, ptr, end, enc) \ - (((state)->handler)(state, tok, ptr, end, enc)) +# endif /* XML_DTD */ -#ifdef __cplusplus -} -#endif +# define XmlTokenRole(state, tok, ptr, end, enc) \ + (((state)->handler)(state, tok, ptr, end, enc)) #endif /* not XmlRole_INCLUDED */ diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index 29a66d72ceea5e..2f837acc4cbf72 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -12,7 +12,7 @@ Copyright (c) 2002 Greg Stein Copyright (c) 2002-2016 Karl Waclawek Copyright (c) 2005-2009 Steven Solie - Copyright (c) 2016-2024 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2016 Pascal Cuoq Copyright (c) 2016 Don Lewis Copyright (c) 2017 Rhodri James @@ -24,6 +24,10 @@ Copyright (c) 2022 Martin Ettl Copyright (c) 2022 Sean McBride Copyright (c) 2023 Hanno Böck + Copyright (c) 2025 Alfonso Gregory + Copyright (c) 2026 Nick Begg + Copyright (c) 2026 Kartik Kenchi + Copyright (c) 2026 Afonso Januário Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -44,6 +48,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #include "expat_config.h" @@ -56,8 +62,8 @@ # include "winconfig.h" #endif -#include "expat_external.h" #include "internal.h" +#include "fallthrough.h" #include "xmltok.h" #include "nametab.h" @@ -131,20 +137,20 @@ : ((p)[1] & 0x80) == 0 \ || ((*p) == 0xF4 ? (p)[1] > 0x8F : ((p)[1] & 0xC0) == 0xC0))) -static int PTRFASTCALL +static int isNever(const ENCODING *enc, const char *p) { UNUSED_P(enc); UNUSED_P(p); return 0; } -static int PTRFASTCALL +static int utf8_isName2(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_GET_NAMING2(namePages, (const unsigned char *)p); } -static int PTRFASTCALL +static int utf8_isName3(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_GET_NAMING3(namePages, (const unsigned char *)p); @@ -152,13 +158,13 @@ utf8_isName3(const ENCODING *enc, const char *p) { #define utf8_isName4 isNever -static int PTRFASTCALL +static int utf8_isNmstrt2(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_GET_NAMING2(nmstrtPages, (const unsigned char *)p); } -static int PTRFASTCALL +static int utf8_isNmstrt3(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_GET_NAMING3(nmstrtPages, (const unsigned char *)p); @@ -166,19 +172,19 @@ utf8_isNmstrt3(const ENCODING *enc, const char *p) { #define utf8_isNmstrt4 isNever -static int PTRFASTCALL +static int utf8_isInvalid2(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_INVALID2((const unsigned char *)p); } -static int PTRFASTCALL +static int utf8_isInvalid3(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_INVALID3((const unsigned char *)p); } -static int PTRFASTCALL +static int utf8_isInvalid4(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_INVALID4((const unsigned char *)p); @@ -187,36 +193,20 @@ utf8_isInvalid4(const ENCODING *enc, const char *p) { struct normal_encoding { ENCODING enc; unsigned char type[256]; -#ifdef XML_MIN_SIZE - int(PTRFASTCALL *byteType)(const ENCODING *, const char *); - int(PTRFASTCALL *isNameMin)(const ENCODING *, const char *); - int(PTRFASTCALL *isNmstrtMin)(const ENCODING *, const char *); - int(PTRFASTCALL *byteToAscii)(const ENCODING *, const char *); - int(PTRCALL *charMatches)(const ENCODING *, const char *, int); -#endif /* XML_MIN_SIZE */ - int(PTRFASTCALL *isName2)(const ENCODING *, const char *); - int(PTRFASTCALL *isName3)(const ENCODING *, const char *); - int(PTRFASTCALL *isName4)(const ENCODING *, const char *); - int(PTRFASTCALL *isNmstrt2)(const ENCODING *, const char *); - int(PTRFASTCALL *isNmstrt3)(const ENCODING *, const char *); - int(PTRFASTCALL *isNmstrt4)(const ENCODING *, const char *); - int(PTRFASTCALL *isInvalid2)(const ENCODING *, const char *); - int(PTRFASTCALL *isInvalid3)(const ENCODING *, const char *); - int(PTRFASTCALL *isInvalid4)(const ENCODING *, const char *); + int (*isName2)(const ENCODING *, const char *); + int (*isName3)(const ENCODING *, const char *); + int (*isName4)(const ENCODING *, const char *); + int (*isNmstrt2)(const ENCODING *, const char *); + int (*isNmstrt3)(const ENCODING *, const char *); + int (*isNmstrt4)(const ENCODING *, const char *); + int (*isInvalid2)(const ENCODING *, const char *); + int (*isInvalid3)(const ENCODING *, const char *); + int (*isInvalid4)(const ENCODING *, const char *); }; #define AS_NORMAL_ENCODING(enc) ((const struct normal_encoding *)(enc)) -#ifdef XML_MIN_SIZE - -# define STANDARD_VTABLE(E) \ - E##byteType, E##isNameMin, E##isNmstrtMin, E##byteToAscii, E##charMatches, - -#else - -# define STANDARD_VTABLE(E) /* as nothing */ - -#endif +#define STANDARD_VTABLE(E) /* as nothing */ #define NORMAL_VTABLE(E) \ E##isName2, E##isName3, E##isName4, E##isNmstrt2, E##isNmstrt3, \ @@ -227,80 +217,33 @@ struct normal_encoding { /* isNmstrt2 */ NULL, /* isNmstrt3 */ NULL, /* isNmstrt4 */ NULL, \ /* isInvalid2 */ NULL, /* isInvalid3 */ NULL, /* isInvalid4 */ NULL -static int FASTCALL checkCharRefNumber(int result); +#define UTF16_NULL_VTABLE(E) NULL_VTABLE + +static int checkCharRefNumber(int result); #include "xmltok_impl.h" #include "ascii.h" -#ifdef XML_MIN_SIZE -# define sb_isNameMin isNever -# define sb_isNmstrtMin isNever -#endif - -#ifdef XML_MIN_SIZE -# define MINBPC(enc) ((enc)->minBytesPerChar) -#else /* minimum bytes per character */ -# define MINBPC(enc) 1 -#endif +#define MINBPC(enc) 1 #define SB_BYTE_TYPE(enc, p) \ (((const struct normal_encoding *)(enc))->type[(unsigned char)*(p)]) -#ifdef XML_MIN_SIZE -static int PTRFASTCALL -sb_byteType(const ENCODING *enc, const char *p) { - return SB_BYTE_TYPE(enc, p); -} -# define BYTE_TYPE(enc, p) (AS_NORMAL_ENCODING(enc)->byteType(enc, p)) -#else -# define BYTE_TYPE(enc, p) SB_BYTE_TYPE(enc, p) -#endif +#define BYTE_TYPE(enc, p) SB_BYTE_TYPE(enc, p) -#ifdef XML_MIN_SIZE -# define BYTE_TO_ASCII(enc, p) (AS_NORMAL_ENCODING(enc)->byteToAscii(enc, p)) -static int PTRFASTCALL -sb_byteToAscii(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return *p; -} -#else -# define BYTE_TO_ASCII(enc, p) (*(p)) -#endif +#define BYTE_TO_ASCII(enc, p) (*(p)) #define IS_NAME_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isName##n(enc, p)) #define IS_NMSTRT_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isNmstrt##n(enc, p)) -#ifdef XML_MIN_SIZE -# define IS_INVALID_CHAR(enc, p, n) \ - (AS_NORMAL_ENCODING(enc)->isInvalid##n \ - && AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p)) -#else -# define IS_INVALID_CHAR(enc, p, n) \ - (AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p)) -#endif +#define IS_INVALID_CHAR(enc, p, n) \ + (AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p)) -#ifdef XML_MIN_SIZE -# define IS_NAME_CHAR_MINBPC(enc, p) \ - (AS_NORMAL_ENCODING(enc)->isNameMin(enc, p)) -# define IS_NMSTRT_CHAR_MINBPC(enc, p) \ - (AS_NORMAL_ENCODING(enc)->isNmstrtMin(enc, p)) -#else -# define IS_NAME_CHAR_MINBPC(enc, p) (0) -# define IS_NMSTRT_CHAR_MINBPC(enc, p) (0) -#endif +#define IS_NAME_CHAR_MINBPC(enc, p) (0) +#define IS_NMSTRT_CHAR_MINBPC(enc, p) (0) -#ifdef XML_MIN_SIZE -# define CHAR_MATCHES(enc, p, c) \ - (AS_NORMAL_ENCODING(enc)->charMatches(enc, p, c)) -static int PTRCALL -sb_charMatches(const ENCODING *enc, const char *p, int c) { - UNUSED_P(enc); - return *p == c; -} -#else /* c is an ASCII character */ -# define CHAR_MATCHES(enc, p, c) (*(p) == (c)) -#endif +#define CHAR_MATCHES(enc, p, c) (*(p) == (c)) #define PREFIX(ident) normal_##ident #define XML_TOK_IMPL_C @@ -363,7 +306,7 @@ _INTERNAL_trim_to_complete_utf8_characters(const char *from, *fromLimRef = fromLim; } -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result utf8_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, char **toP, const char *toLim) { bool input_incomplete = false; @@ -402,7 +345,7 @@ utf8_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, return XML_CONVERT_COMPLETED; } -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result utf8_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim, unsigned short **toP, const unsigned short *toLim) { enum XML_Convert_Result res = XML_CONVERT_COMPLETED; @@ -500,7 +443,7 @@ static const struct normal_encoding internal_utf8_encoding }, STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)}; -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result latin1_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, char **toP, const char *toLim) { UNUSED_P(enc); @@ -523,7 +466,7 @@ latin1_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, } } -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result latin1_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim, unsigned short **toP, const unsigned short *toLim) { UNUSED_P(enc); @@ -558,7 +501,7 @@ static const struct normal_encoding latin1_encoding }, STANDARD_VTABLE(sb_) NULL_VTABLE}; -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result ascii_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, char **toP, const char *toLim) { UNUSED_P(enc); @@ -593,7 +536,7 @@ static const struct normal_encoding ascii_encoding }, STANDARD_VTABLE(sb_) NULL_VTABLE}; -static int PTRFASTCALL +static int unicode_byte_type(char hi, char lo) { switch ((unsigned char)hi) { /* 0xD800-0xDBFF first 16-bit code unit or high surrogate (W1) */ @@ -620,7 +563,7 @@ unicode_byte_type(char hi, char lo) { } #define DEFINE_UTF16_TO_UTF8(E) \ - static enum XML_Convert_Result PTRCALL E##toUtf8( \ + static enum XML_Convert_Result E##toUtf8( \ const ENCODING *enc, const char **fromP, const char *fromLim, \ char **toP, const char *toLim) { \ const char *from = *fromP; \ @@ -641,7 +584,7 @@ unicode_byte_type(char hi, char lo) { *(*toP)++ = lo; \ break; \ } \ - /* fall through */ \ + EXPAT_FALLTHROUGH; \ case 0x1: \ case 0x2: \ case 0x3: \ @@ -697,15 +640,16 @@ unicode_byte_type(char hi, char lo) { } #define DEFINE_UTF16_TO_UTF16(E) \ - static enum XML_Convert_Result PTRCALL E##toUtf16( \ + static enum XML_Convert_Result E##toUtf16( \ const ENCODING *enc, const char **fromP, const char *fromLim, \ unsigned short **toP, const unsigned short *toLim) { \ enum XML_Convert_Result res = XML_CONVERT_COMPLETED; \ UNUSED_P(enc); \ fromLim = *fromP + (((fromLim - *fromP) >> 1) << 1); /* shrink to even */ \ - /* Avoid copying first half only of surrogate */ \ + /* Avoid copying the first half (2 bytes) of surrogate pairs (4 bytes) */ \ if (fromLim - *fromP > ((toLim - *toP) << 1) \ - && (GET_HI(fromLim - 2) & 0xF8) == 0xD8) { \ + && /* are the last two bytes a high surrogate (0xD800-0xDBFF)? */ \ + (GET_HI(fromLim - 2) & 0xFC) == 0xD8) { \ fromLim -= 2; \ res = XML_CONVERT_INPUT_INCOMPLETE; \ } \ @@ -743,70 +687,38 @@ DEFINE_UTF16_TO_UTF16(big2_) UCS2_GET_NAMING(namePages, (unsigned char)p[1], (unsigned char)p[0]) #define LITTLE2_IS_NMSTRT_CHAR_MINBPC(p) \ UCS2_GET_NAMING(nmstrtPages, (unsigned char)p[1], (unsigned char)p[0]) +/* A 4-byte UTF-16 character is a surrogate pair; byteType only reports BT_LEAD4 + for a high surrogate, so the pair is invalid unless the second unit is a low + surrogate (U+DC00..U+DFFF, i.e. high byte 0xDC..0xDF). */ +#define LITTLE2_IS_INVALID_CHAR(p, n) \ + ((n) == 4 && ((unsigned char)(p)[3] & 0xFC) != 0xDC) -#ifdef XML_MIN_SIZE - -static int PTRFASTCALL -little2_byteType(const ENCODING *enc, const char *p) { - return LITTLE2_BYTE_TYPE(enc, p); -} - -static int PTRFASTCALL -little2_byteToAscii(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return LITTLE2_BYTE_TO_ASCII(p); -} - -static int PTRCALL -little2_charMatches(const ENCODING *enc, const char *p, int c) { - UNUSED_P(enc); - return LITTLE2_CHAR_MATCHES(p, c); -} - -static int PTRFASTCALL -little2_isNameMin(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return LITTLE2_IS_NAME_CHAR_MINBPC(p); -} - -static int PTRFASTCALL -little2_isNmstrtMin(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return LITTLE2_IS_NMSTRT_CHAR_MINBPC(p); -} - -# undef VTABLE -# define VTABLE VTABLE1, little2_toUtf8, little2_toUtf16 +#undef PREFIX +#define PREFIX(ident) little2_##ident +#define MINBPC(enc) 2 +/* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */ +#define BYTE_TYPE(enc, p) LITTLE2_BYTE_TYPE(enc, p) +#define BYTE_TO_ASCII(enc, p) LITTLE2_BYTE_TO_ASCII(p) +#define CHAR_MATCHES(enc, p, c) LITTLE2_CHAR_MATCHES(p, c) +#define IS_NAME_CHAR(enc, p, n) 0 +#define IS_NAME_CHAR_MINBPC(enc, p) LITTLE2_IS_NAME_CHAR_MINBPC(p) +#define IS_NMSTRT_CHAR(enc, p, n) (0) +#define IS_NMSTRT_CHAR_MINBPC(enc, p) LITTLE2_IS_NMSTRT_CHAR_MINBPC(p) +#define IS_INVALID_CHAR(enc, p, n) LITTLE2_IS_INVALID_CHAR(p, n) -#else /* not XML_MIN_SIZE */ +#define XML_TOK_IMPL_C +#include "xmltok_impl.c" +#undef XML_TOK_IMPL_C -# undef PREFIX -# define PREFIX(ident) little2_##ident -# define MINBPC(enc) 2 -/* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */ -# define BYTE_TYPE(enc, p) LITTLE2_BYTE_TYPE(enc, p) -# define BYTE_TO_ASCII(enc, p) LITTLE2_BYTE_TO_ASCII(p) -# define CHAR_MATCHES(enc, p, c) LITTLE2_CHAR_MATCHES(p, c) -# define IS_NAME_CHAR(enc, p, n) 0 -# define IS_NAME_CHAR_MINBPC(enc, p) LITTLE2_IS_NAME_CHAR_MINBPC(p) -# define IS_NMSTRT_CHAR(enc, p, n) (0) -# define IS_NMSTRT_CHAR_MINBPC(enc, p) LITTLE2_IS_NMSTRT_CHAR_MINBPC(p) - -# define XML_TOK_IMPL_C -# include "xmltok_impl.c" -# undef XML_TOK_IMPL_C - -# undef MINBPC -# undef BYTE_TYPE -# undef BYTE_TO_ASCII -# undef CHAR_MATCHES -# undef IS_NAME_CHAR -# undef IS_NAME_CHAR_MINBPC -# undef IS_NMSTRT_CHAR -# undef IS_NMSTRT_CHAR_MINBPC -# undef IS_INVALID_CHAR - -#endif /* not XML_MIN_SIZE */ +#undef MINBPC +#undef BYTE_TYPE +#undef BYTE_TO_ASCII +#undef CHAR_MATCHES +#undef IS_NAME_CHAR +#undef IS_NAME_CHAR_MINBPC +#undef IS_NMSTRT_CHAR +#undef IS_NMSTRT_CHAR_MINBPC +#undef IS_INVALID_CHAR #ifdef XML_NS @@ -822,7 +734,7 @@ static const struct normal_encoding little2_encoding_ns # include "asciitab.h" # include "latin1tab.h" }, - STANDARD_VTABLE(little2_) NULL_VTABLE}; + STANDARD_VTABLE(little2_) UTF16_NULL_VTABLE(little2_)}; #endif @@ -840,7 +752,7 @@ static const struct normal_encoding little2_encoding #undef BT_COLON #include "latin1tab.h" }, - STANDARD_VTABLE(little2_) NULL_VTABLE}; + STANDARD_VTABLE(little2_) UTF16_NULL_VTABLE(little2_)}; #if BYTEORDER != 4321 @@ -852,7 +764,7 @@ static const struct normal_encoding internal_little2_encoding_ns # include "iasciitab.h" # include "latin1tab.h" }, - STANDARD_VTABLE(little2_) NULL_VTABLE}; + STANDARD_VTABLE(little2_) UTF16_NULL_VTABLE(little2_)}; # endif @@ -864,7 +776,7 @@ static const struct normal_encoding internal_little2_encoding # undef BT_COLON # include "latin1tab.h" }, - STANDARD_VTABLE(little2_) NULL_VTABLE}; + STANDARD_VTABLE(little2_) UTF16_NULL_VTABLE(little2_)}; #endif @@ -876,70 +788,38 @@ static const struct normal_encoding internal_little2_encoding UCS2_GET_NAMING(namePages, (unsigned char)p[0], (unsigned char)p[1]) #define BIG2_IS_NMSTRT_CHAR_MINBPC(p) \ UCS2_GET_NAMING(nmstrtPages, (unsigned char)p[0], (unsigned char)p[1]) +/* A 4-byte UTF-16 character is a surrogate pair; byteType only reports BT_LEAD4 + for a high surrogate, so the pair is invalid unless the second unit is a low + surrogate (U+DC00..U+DFFF, i.e. high byte 0xDC..0xDF). */ +#define BIG2_IS_INVALID_CHAR(p, n) \ + ((n) == 4 && ((unsigned char)(p)[2] & 0xFC) != 0xDC) -#ifdef XML_MIN_SIZE - -static int PTRFASTCALL -big2_byteType(const ENCODING *enc, const char *p) { - return BIG2_BYTE_TYPE(enc, p); -} - -static int PTRFASTCALL -big2_byteToAscii(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return BIG2_BYTE_TO_ASCII(p); -} - -static int PTRCALL -big2_charMatches(const ENCODING *enc, const char *p, int c) { - UNUSED_P(enc); - return BIG2_CHAR_MATCHES(p, c); -} - -static int PTRFASTCALL -big2_isNameMin(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return BIG2_IS_NAME_CHAR_MINBPC(p); -} - -static int PTRFASTCALL -big2_isNmstrtMin(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return BIG2_IS_NMSTRT_CHAR_MINBPC(p); -} - -# undef VTABLE -# define VTABLE VTABLE1, big2_toUtf8, big2_toUtf16 +#undef PREFIX +#define PREFIX(ident) big2_##ident +#define MINBPC(enc) 2 +/* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */ +#define BYTE_TYPE(enc, p) BIG2_BYTE_TYPE(enc, p) +#define BYTE_TO_ASCII(enc, p) BIG2_BYTE_TO_ASCII(p) +#define CHAR_MATCHES(enc, p, c) BIG2_CHAR_MATCHES(p, c) +#define IS_NAME_CHAR(enc, p, n) 0 +#define IS_NAME_CHAR_MINBPC(enc, p) BIG2_IS_NAME_CHAR_MINBPC(p) +#define IS_NMSTRT_CHAR(enc, p, n) (0) +#define IS_NMSTRT_CHAR_MINBPC(enc, p) BIG2_IS_NMSTRT_CHAR_MINBPC(p) +#define IS_INVALID_CHAR(enc, p, n) BIG2_IS_INVALID_CHAR(p, n) -#else /* not XML_MIN_SIZE */ +#define XML_TOK_IMPL_C +#include "xmltok_impl.c" +#undef XML_TOK_IMPL_C -# undef PREFIX -# define PREFIX(ident) big2_##ident -# define MINBPC(enc) 2 -/* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */ -# define BYTE_TYPE(enc, p) BIG2_BYTE_TYPE(enc, p) -# define BYTE_TO_ASCII(enc, p) BIG2_BYTE_TO_ASCII(p) -# define CHAR_MATCHES(enc, p, c) BIG2_CHAR_MATCHES(p, c) -# define IS_NAME_CHAR(enc, p, n) 0 -# define IS_NAME_CHAR_MINBPC(enc, p) BIG2_IS_NAME_CHAR_MINBPC(p) -# define IS_NMSTRT_CHAR(enc, p, n) (0) -# define IS_NMSTRT_CHAR_MINBPC(enc, p) BIG2_IS_NMSTRT_CHAR_MINBPC(p) - -# define XML_TOK_IMPL_C -# include "xmltok_impl.c" -# undef XML_TOK_IMPL_C - -# undef MINBPC -# undef BYTE_TYPE -# undef BYTE_TO_ASCII -# undef CHAR_MATCHES -# undef IS_NAME_CHAR -# undef IS_NAME_CHAR_MINBPC -# undef IS_NMSTRT_CHAR -# undef IS_NMSTRT_CHAR_MINBPC -# undef IS_INVALID_CHAR - -#endif /* not XML_MIN_SIZE */ +#undef MINBPC +#undef BYTE_TYPE +#undef BYTE_TO_ASCII +#undef CHAR_MATCHES +#undef IS_NAME_CHAR +#undef IS_NAME_CHAR_MINBPC +#undef IS_NMSTRT_CHAR +#undef IS_NMSTRT_CHAR_MINBPC +#undef IS_INVALID_CHAR #ifdef XML_NS @@ -955,7 +835,7 @@ static const struct normal_encoding big2_encoding_ns # include "asciitab.h" # include "latin1tab.h" }, - STANDARD_VTABLE(big2_) NULL_VTABLE}; + STANDARD_VTABLE(big2_) UTF16_NULL_VTABLE(big2_)}; #endif @@ -973,7 +853,7 @@ static const struct normal_encoding big2_encoding #undef BT_COLON #include "latin1tab.h" }, - STANDARD_VTABLE(big2_) NULL_VTABLE}; + STANDARD_VTABLE(big2_) UTF16_NULL_VTABLE(big2_)}; #if BYTEORDER != 1234 @@ -985,7 +865,7 @@ static const struct normal_encoding internal_big2_encoding_ns # include "iasciitab.h" # include "latin1tab.h" }, - STANDARD_VTABLE(big2_) NULL_VTABLE}; + STANDARD_VTABLE(big2_) UTF16_NULL_VTABLE(big2_)}; # endif @@ -997,13 +877,13 @@ static const struct normal_encoding internal_big2_encoding # undef BT_COLON # include "latin1tab.h" }, - STANDARD_VTABLE(big2_) NULL_VTABLE}; + STANDARD_VTABLE(big2_) UTF16_NULL_VTABLE(big2_)}; #endif #undef PREFIX -static int FASTCALL +static int streqci(const char *s1, const char *s2) { for (;;) { char c1 = *s1++; @@ -1024,7 +904,7 @@ streqci(const char *s1, const char *s2) { return 1; } -static void PTRCALL +static void initUpdatePosition(const ENCODING *enc, const char *ptr, const char *end, POSITION *pos) { UNUSED_P(enc); @@ -1042,7 +922,7 @@ toAscii(const ENCODING *enc, const char *ptr, const char *end) { return buf[0]; } -static int FASTCALL +static int isSpace(int c) { switch (c) { case 0x20: @@ -1148,6 +1028,34 @@ static const char KW_yes[] = {ASCII_y, ASCII_e, ASCII_s, '\0'}; static const char KW_no[] = {ASCII_n, ASCII_o, '\0'}; +static const char KW_1_dot[] = {ASCII_1, ASCII_PERIOD, '\0'}; + +/* Checks a version pseudo-attribute value against the VersionNum production. + XML 1.0 Fourth Edition only allows the literal "1.0", but the Fifth + Edition relaxed this to "1." followed by one or more digits, since Expat + only implements 1.0 itself but plans to track the Fifth Edition's laxer + grammar here so that "1.1" and similar aren't rejected only to have that + rejection reverted later. Returns true for a value matching "1.[0-9]+". + val/valEnd bound the value itself; valEnd is the upper bound used when + decoding the individual characters between them. */ +static bool +checkXmlDeclVersionNum(const ENCODING *enc, const char *val, + const char *valEnd) { + if (valEnd - val < 2 * enc->minBytesPerChar + || ! XmlNameMatchesAscii(enc, val, val + 2 * enc->minBytesPerChar, + KW_1_dot)) + return false; + val += 2 * enc->minBytesPerChar; + if (val == valEnd) + return false; + for (; val != valEnd; val += enc->minBytesPerChar) { + int c = toAscii(enc, val, valEnd); + if (c < ASCII_0 || c > ASCII_9) + return false; + } + return true; +} + static int doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *, const char *, const char *), @@ -1175,6 +1083,22 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *, const char *, *versionPtr = val; if (versionEndPtr) *versionEndPtr = ptr; + /* The version number must not be empty; VersionNum requires at least + one character. The encoding and standalone pseudo-attributes below + already reject an empty value, so keep version consistent. */ + if (val == ptr - enc->minBytesPerChar) { + *badPtr = val; + return 0; + } + /* Expat implements XML 1.0 only, so any version outside the "1.0"/"1.x" + family is rejected. Following the Fifth Edition's VersionNum + production (rather than the Fourth Edition's exact "1.0") avoids + rejecting "1.1" now only to have to revert that once Expat tracks + the newer edition. */ + if (! checkXmlDeclVersionNum(enc, val, ptr - enc->minBytesPerChar)) { + *badPtr = val; + return 0; + } if (! parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) { *badPtr = ptr; return 0; @@ -1229,7 +1153,7 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *, const char *, return 1; } -static int FASTCALL +static int checkCharRefNumber(int result) { switch (result >> 8) { case 0xD8: @@ -1253,7 +1177,7 @@ checkCharRefNumber(int result) { return result; } -int FASTCALL +int XmlUtf8Encode(int c, char *buf) { enum { /* minN is minimum legal resulting value for N byte sequence */ @@ -1289,7 +1213,7 @@ XmlUtf8Encode(int c, char *buf) { return 0; /* LCOV_EXCL_LINE: this case too is eliminated before calling */ } -int FASTCALL +int XmlUtf16Encode(int charNum, unsigned short *buf) { if (charNum < 0) return 0; @@ -1321,7 +1245,7 @@ XmlSizeOfUnknownEncoding(void) { return sizeof(struct unknown_encoding); } -static int PTRFASTCALL +static int unknown_isName(const ENCODING *enc, const char *p) { const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc); int c = uenc->convert(uenc->userData, p); @@ -1330,7 +1254,7 @@ unknown_isName(const ENCODING *enc, const char *p) { return UCS2_GET_NAMING(namePages, c >> 8, c & 0xFF); } -static int PTRFASTCALL +static int unknown_isNmstrt(const ENCODING *enc, const char *p) { const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc); int c = uenc->convert(uenc->userData, p); @@ -1339,14 +1263,14 @@ unknown_isNmstrt(const ENCODING *enc, const char *p) { return UCS2_GET_NAMING(nmstrtPages, c >> 8, c & 0xFF); } -static int PTRFASTCALL +static int unknown_isInvalid(const ENCODING *enc, const char *p) { const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc); int c = uenc->convert(uenc->userData, p); return (c & ~0xFFFF) || checkCharRefNumber(c) < 0; } -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result unknown_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, char **toP, const char *toLim) { const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc); @@ -1376,7 +1300,7 @@ unknown_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, } } -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result unknown_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim, unsigned short **toP, const unsigned short *toLim) { const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc); @@ -1398,7 +1322,7 @@ unknown_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim, } ENCODING * -XmlInitUnknownEncoding(void *mem, int *table, CONVERTER convert, +XmlInitUnknownEncoding(void *mem, const int *table, CONVERTER convert, void *userData) { int i; struct unknown_encoding *e = (struct unknown_encoding *)mem; @@ -1500,7 +1424,7 @@ static const char KW_UTF_16LE[] = {ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, ASCII_L, ASCII_E, '\0'}; -static int FASTCALL +static int getEncodingIndex(const char *name) { static const char *const encodingNames[] = { KW_ISO_8859_1, KW_US_ASCII, KW_UTF_8, KW_UTF_16, KW_UTF_16BE, KW_UTF_16LE, @@ -1557,7 +1481,7 @@ initScan(const ENCODING *const *encodingTable, const INIT_ENCODING *enc, case 0xEF: /* possibly first byte of UTF-8 BOM */ if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC && state == XML_CONTENT_STATE) break; - /* fall through */ + EXPAT_FALLTHROUGH; case 0x00: case 0x3C: return XML_TOK_PARTIAL; @@ -1661,7 +1585,7 @@ initScan(const ENCODING *const *encodingTable, const INIT_ENCODING *enc, # undef ns ENCODING * -XmlInitUnknownEncodingNS(void *mem, int *table, CONVERTER convert, +XmlInitUnknownEncodingNS(void *mem, const int *table, CONVERTER convert, void *userData) { ENCODING *enc = XmlInitUnknownEncoding(mem, table, convert, userData); if (enc) diff --git a/Modules/expat/xmltok.h b/Modules/expat/xmltok.h index c51fce1ec1518b..bf641af6d4f77f 100644 --- a/Modules/expat/xmltok.h +++ b/Modules/expat/xmltok.h @@ -10,7 +10,7 @@ Copyright (c) 2000 Clark Cooper Copyright (c) 2002 Fred L. Drake, Jr. Copyright (c) 2002-2005 Karl Waclawek - Copyright (c) 2016-2024 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2017 Rhodri James Licensed under the MIT license: @@ -32,121 +32,121 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifndef XmlTok_INCLUDED -#define XmlTok_INCLUDED 1 +# define XmlTok_INCLUDED 1 -#ifdef __cplusplus -extern "C" { -#endif +# include // uint64_t /* The following token may be returned by XmlContentTok */ -#define XML_TOK_TRAILING_RSQB \ - -5 /* ] or ]] at the end of the scan; might be \ - start of illegal ]]> sequence */ +# define XML_TOK_TRAILING_RSQB \ + -5 /* ] or ]] at the end of the scan; might be \ + start of illegal ]]> sequence */ /* The following tokens may be returned by both XmlPrologTok and XmlContentTok. */ -#define XML_TOK_NONE -4 /* The string to be scanned is empty */ -#define XML_TOK_TRAILING_CR \ - -3 /* A CR at the end of the scan; \ - might be part of CRLF sequence */ -#define XML_TOK_PARTIAL_CHAR -2 /* only part of a multibyte sequence */ -#define XML_TOK_PARTIAL -1 /* only part of a token */ -#define XML_TOK_INVALID 0 +# define XML_TOK_NONE -4 /* The string to be scanned is empty */ +# define XML_TOK_TRAILING_CR \ + -3 /* A CR at the end of the scan; \ + might be part of CRLF sequence */ +# define XML_TOK_PARTIAL_CHAR -2 /* only part of a multibyte sequence */ +# define XML_TOK_PARTIAL -1 /* only part of a token */ +# define XML_TOK_INVALID 0 /* The following tokens are returned by XmlContentTok; some are also returned by XmlAttributeValueTok, XmlEntityTok, XmlCdataSectionTok. */ -#define XML_TOK_START_TAG_WITH_ATTS 1 -#define XML_TOK_START_TAG_NO_ATTS 2 -#define XML_TOK_EMPTY_ELEMENT_WITH_ATTS 3 /* empty element tag */ -#define XML_TOK_EMPTY_ELEMENT_NO_ATTS 4 -#define XML_TOK_END_TAG 5 -#define XML_TOK_DATA_CHARS 6 -#define XML_TOK_DATA_NEWLINE 7 -#define XML_TOK_CDATA_SECT_OPEN 8 -#define XML_TOK_ENTITY_REF 9 -#define XML_TOK_CHAR_REF 10 /* numeric character reference */ +# define XML_TOK_START_TAG_WITH_ATTS 1 +# define XML_TOK_START_TAG_NO_ATTS 2 +# define XML_TOK_EMPTY_ELEMENT_WITH_ATTS 3 /* empty element tag */ +# define XML_TOK_EMPTY_ELEMENT_NO_ATTS 4 +# define XML_TOK_END_TAG 5 +# define XML_TOK_DATA_CHARS 6 +# define XML_TOK_DATA_NEWLINE 7 +# define XML_TOK_CDATA_SECT_OPEN 8 +# define XML_TOK_ENTITY_REF 9 +# define XML_TOK_CHAR_REF 10 /* numeric character reference */ /* The following tokens may be returned by both XmlPrologTok and XmlContentTok. */ -#define XML_TOK_PI 11 /* processing instruction */ -#define XML_TOK_XML_DECL 12 /* XML decl or text decl */ -#define XML_TOK_COMMENT 13 -#define XML_TOK_BOM 14 /* Byte order mark */ +# define XML_TOK_PI 11 /* processing instruction */ +# define XML_TOK_XML_DECL 12 /* XML decl or text decl */ +# define XML_TOK_COMMENT 13 +# define XML_TOK_BOM 14 /* Byte order mark */ /* The following tokens are returned only by XmlPrologTok */ -#define XML_TOK_PROLOG_S 15 -#define XML_TOK_DECL_OPEN 16 /* */ -#define XML_TOK_NAME 18 -#define XML_TOK_NMTOKEN 19 -#define XML_TOK_POUND_NAME 20 /* #name */ -#define XML_TOK_OR 21 /* | */ -#define XML_TOK_PERCENT 22 -#define XML_TOK_OPEN_PAREN 23 -#define XML_TOK_CLOSE_PAREN 24 -#define XML_TOK_OPEN_BRACKET 25 -#define XML_TOK_CLOSE_BRACKET 26 -#define XML_TOK_LITERAL 27 -#define XML_TOK_PARAM_ENTITY_REF 28 -#define XML_TOK_INSTANCE_START 29 +# define XML_TOK_PROLOG_S 15 +# define XML_TOK_DECL_OPEN 16 /* */ +# define XML_TOK_NAME 18 +# define XML_TOK_NMTOKEN 19 +# define XML_TOK_POUND_NAME 20 /* #name */ +# define XML_TOK_OR 21 /* | */ +# define XML_TOK_PERCENT 22 +# define XML_TOK_OPEN_PAREN 23 +# define XML_TOK_CLOSE_PAREN 24 +# define XML_TOK_OPEN_BRACKET 25 +# define XML_TOK_CLOSE_BRACKET 26 +# define XML_TOK_LITERAL 27 +# define XML_TOK_PARAM_ENTITY_REF 28 +# define XML_TOK_INSTANCE_START 29 /* The following occur only in element type declarations */ -#define XML_TOK_NAME_QUESTION 30 /* name? */ -#define XML_TOK_NAME_ASTERISK 31 /* name* */ -#define XML_TOK_NAME_PLUS 32 /* name+ */ -#define XML_TOK_COND_SECT_OPEN 33 /* */ -#define XML_TOK_CLOSE_PAREN_QUESTION 35 /* )? */ -#define XML_TOK_CLOSE_PAREN_ASTERISK 36 /* )* */ -#define XML_TOK_CLOSE_PAREN_PLUS 37 /* )+ */ -#define XML_TOK_COMMA 38 +# define XML_TOK_NAME_QUESTION 30 /* name? */ +# define XML_TOK_NAME_ASTERISK 31 /* name* */ +# define XML_TOK_NAME_PLUS 32 /* name+ */ +# define XML_TOK_COND_SECT_OPEN 33 /* */ +# define XML_TOK_CLOSE_PAREN_QUESTION 35 /* )? */ +# define XML_TOK_CLOSE_PAREN_ASTERISK 36 /* )* */ +# define XML_TOK_CLOSE_PAREN_PLUS 37 /* )+ */ +# define XML_TOK_COMMA 38 /* The following token is returned only by XmlAttributeValueTok */ -#define XML_TOK_ATTRIBUTE_VALUE_S 39 +# define XML_TOK_ATTRIBUTE_VALUE_S 39 /* The following token is returned only by XmlCdataSectionTok */ -#define XML_TOK_CDATA_SECT_CLOSE 40 +# define XML_TOK_CDATA_SECT_CLOSE 40 /* With namespace processing this is returned by XmlPrologTok for a name with a colon. */ -#define XML_TOK_PREFIXED_NAME 41 +# define XML_TOK_PREFIXED_NAME 41 -#ifdef XML_DTD -# define XML_TOK_IGNORE_SECT 42 -#endif /* XML_DTD */ +# ifdef XML_DTD +# define XML_TOK_IGNORE_SECT 42 +# endif /* XML_DTD */ -#ifdef XML_DTD -# define XML_N_STATES 4 -#else /* not XML_DTD */ -# define XML_N_STATES 3 -#endif /* not XML_DTD */ +# ifdef XML_DTD +# define XML_N_STATES 4 +# else /* not XML_DTD */ +# define XML_N_STATES 3 +# endif /* not XML_DTD */ -#define XML_PROLOG_STATE 0 -#define XML_CONTENT_STATE 1 -#define XML_CDATA_SECTION_STATE 2 -#ifdef XML_DTD -# define XML_IGNORE_SECTION_STATE 3 -#endif /* XML_DTD */ +# define XML_PROLOG_STATE 0 +# define XML_CONTENT_STATE 1 +# define XML_CDATA_SECTION_STATE 2 +# ifdef XML_DTD +# define XML_IGNORE_SECTION_STATE 3 +# endif /* XML_DTD */ -#define XML_N_LITERAL_TYPES 2 -#define XML_ATTRIBUTE_VALUE_LITERAL 0 -#define XML_ENTITY_VALUE_LITERAL 1 +# define XML_N_LITERAL_TYPES 2 +# define XML_ATTRIBUTE_VALUE_LITERAL 0 +# define XML_ENTITY_VALUE_LITERAL 1 /* The size of the buffer passed to XmlUtf8Encode must be at least this. */ -#define XML_UTF8_ENCODE_MAX 4 +# define XML_UTF8_ENCODE_MAX 4 /* The size of the buffer passed to XmlUtf16Encode must be at least this. */ -#define XML_UTF16_ENCODE_MAX 2 +# define XML_UTF16_ENCODE_MAX 2 typedef struct position { /* first line and first column are 0 not 1 */ - XML_Size lineNumber; - XML_Size columnNumber; + uint64_t lineNumber; + uint64_t columnNumber; } POSITION; typedef struct { @@ -159,41 +159,40 @@ typedef struct { struct encoding; typedef struct encoding ENCODING; -typedef int(PTRCALL *SCANNER)(const ENCODING *, const char *, const char *, - const char **); +typedef int (*SCANNER)(const ENCODING *, const char *, const char *, + const char **); enum XML_Convert_Result { XML_CONVERT_COMPLETED = 0, XML_CONVERT_INPUT_INCOMPLETE = 1, - XML_CONVERT_OUTPUT_EXHAUSTED - = 2 /* and therefore potentially input remaining as well */ + XML_CONVERT_OUTPUT_EXHAUSTED = 2 /* and therefore potentially input remaining + as well */ }; struct encoding { SCANNER scanners[XML_N_STATES]; SCANNER literalScanners[XML_N_LITERAL_TYPES]; - int(PTRCALL *nameMatchesAscii)(const ENCODING *, const char *, const char *, - const char *); - int(PTRFASTCALL *nameLength)(const ENCODING *, const char *); - const char *(PTRFASTCALL *skipS)(const ENCODING *, const char *); - int(PTRCALL *getAtts)(const ENCODING *enc, const char *ptr, int attsMax, - ATTRIBUTE *atts); - int(PTRFASTCALL *charRefNumber)(const ENCODING *enc, const char *ptr); - int(PTRCALL *predefinedEntityName)(const ENCODING *, const char *, - const char *); - void(PTRCALL *updatePosition)(const ENCODING *, const char *ptr, - const char *end, POSITION *); - int(PTRCALL *isPublicId)(const ENCODING *enc, const char *ptr, - const char *end, const char **badPtr); - enum XML_Convert_Result(PTRCALL *utf8Convert)(const ENCODING *enc, - const char **fromP, - const char *fromLim, char **toP, - const char *toLim); - enum XML_Convert_Result(PTRCALL *utf16Convert)(const ENCODING *enc, - const char **fromP, - const char *fromLim, - unsigned short **toP, - const unsigned short *toLim); + int (*nameMatchesAscii)(const ENCODING *, const char *, const char *, + const char *); + int (*nameLength)(const ENCODING *, const char *); + const char *(*skipS)(const ENCODING *, const char *); + int (*getAtts)(const ENCODING *enc, const char *ptr, int attsMax, + ATTRIBUTE *atts); + int (*charRefNumber)(const ENCODING *enc, const char *ptr); + int (*predefinedEntityName)(const ENCODING *, const char *, const char *); + void (*updatePosition)(const ENCODING *, const char *ptr, const char *end, + POSITION *); + int (*isPublicId)(const ENCODING *enc, const char *ptr, const char *end, + const char **badPtr); + enum XML_Convert_Result (*utf8Convert)(const ENCODING *enc, + const char **fromP, + const char *fromLim, char **toP, + const char *toLim); + enum XML_Convert_Result (*utf16Convert)(const ENCODING *enc, + const char **fromP, + const char *fromLim, + unsigned short **toP, + const unsigned short *toLim); int minBytesPerChar; char isUtf8; char isUtf16; @@ -220,63 +219,63 @@ struct encoding { the prolog outside literals, comments and processing instructions. */ -#define XmlTok(enc, state, ptr, end, nextTokPtr) \ - (((enc)->scanners[state])(enc, ptr, end, nextTokPtr)) +# define XmlTok(enc, state, ptr, end, nextTokPtr) \ + (((enc)->scanners[state])(enc, ptr, end, nextTokPtr)) -#define XmlPrologTok(enc, ptr, end, nextTokPtr) \ - XmlTok(enc, XML_PROLOG_STATE, ptr, end, nextTokPtr) +# define XmlPrologTok(enc, ptr, end, nextTokPtr) \ + XmlTok(enc, XML_PROLOG_STATE, ptr, end, nextTokPtr) -#define XmlContentTok(enc, ptr, end, nextTokPtr) \ - XmlTok(enc, XML_CONTENT_STATE, ptr, end, nextTokPtr) +# define XmlContentTok(enc, ptr, end, nextTokPtr) \ + XmlTok(enc, XML_CONTENT_STATE, ptr, end, nextTokPtr) -#define XmlCdataSectionTok(enc, ptr, end, nextTokPtr) \ - XmlTok(enc, XML_CDATA_SECTION_STATE, ptr, end, nextTokPtr) +# define XmlCdataSectionTok(enc, ptr, end, nextTokPtr) \ + XmlTok(enc, XML_CDATA_SECTION_STATE, ptr, end, nextTokPtr) -#ifdef XML_DTD +# ifdef XML_DTD -# define XmlIgnoreSectionTok(enc, ptr, end, nextTokPtr) \ - XmlTok(enc, XML_IGNORE_SECTION_STATE, ptr, end, nextTokPtr) +# define XmlIgnoreSectionTok(enc, ptr, end, nextTokPtr) \ + XmlTok(enc, XML_IGNORE_SECTION_STATE, ptr, end, nextTokPtr) -#endif /* XML_DTD */ +# endif /* XML_DTD */ /* This is used for performing a 2nd-level tokenization on the content of a literal that has already been returned by XmlTok. */ -#define XmlLiteralTok(enc, literalType, ptr, end, nextTokPtr) \ - (((enc)->literalScanners[literalType])(enc, ptr, end, nextTokPtr)) +# define XmlLiteralTok(enc, literalType, ptr, end, nextTokPtr) \ + (((enc)->literalScanners[literalType])(enc, ptr, end, nextTokPtr)) -#define XmlAttributeValueTok(enc, ptr, end, nextTokPtr) \ - XmlLiteralTok(enc, XML_ATTRIBUTE_VALUE_LITERAL, ptr, end, nextTokPtr) +# define XmlAttributeValueTok(enc, ptr, end, nextTokPtr) \ + XmlLiteralTok(enc, XML_ATTRIBUTE_VALUE_LITERAL, ptr, end, nextTokPtr) -#define XmlEntityValueTok(enc, ptr, end, nextTokPtr) \ - XmlLiteralTok(enc, XML_ENTITY_VALUE_LITERAL, ptr, end, nextTokPtr) +# define XmlEntityValueTok(enc, ptr, end, nextTokPtr) \ + XmlLiteralTok(enc, XML_ENTITY_VALUE_LITERAL, ptr, end, nextTokPtr) -#define XmlNameMatchesAscii(enc, ptr1, end1, ptr2) \ - (((enc)->nameMatchesAscii)(enc, ptr1, end1, ptr2)) +# define XmlNameMatchesAscii(enc, ptr1, end1, ptr2) \ + (((enc)->nameMatchesAscii)(enc, ptr1, end1, ptr2)) -#define XmlNameLength(enc, ptr) (((enc)->nameLength)(enc, ptr)) +# define XmlNameLength(enc, ptr) (((enc)->nameLength)(enc, ptr)) -#define XmlSkipS(enc, ptr) (((enc)->skipS)(enc, ptr)) +# define XmlSkipS(enc, ptr) (((enc)->skipS)(enc, ptr)) -#define XmlGetAttributes(enc, ptr, attsMax, atts) \ - (((enc)->getAtts)(enc, ptr, attsMax, atts)) +# define XmlGetAttributes(enc, ptr, attsMax, atts) \ + (((enc)->getAtts)(enc, ptr, attsMax, atts)) -#define XmlCharRefNumber(enc, ptr) (((enc)->charRefNumber)(enc, ptr)) +# define XmlCharRefNumber(enc, ptr) (((enc)->charRefNumber)(enc, ptr)) -#define XmlPredefinedEntityName(enc, ptr, end) \ - (((enc)->predefinedEntityName)(enc, ptr, end)) +# define XmlPredefinedEntityName(enc, ptr, end) \ + (((enc)->predefinedEntityName)(enc, ptr, end)) -#define XmlUpdatePosition(enc, ptr, end, pos) \ - (((enc)->updatePosition)(enc, ptr, end, pos)) +# define XmlUpdatePosition(enc, ptr, end, pos) \ + (((enc)->updatePosition)(enc, ptr, end, pos)) -#define XmlIsPublicId(enc, ptr, end, badPtr) \ - (((enc)->isPublicId)(enc, ptr, end, badPtr)) +# define XmlIsPublicId(enc, ptr, end, badPtr) \ + (((enc)->isPublicId)(enc, ptr, end, badPtr)) -#define XmlUtf8Convert(enc, fromP, fromLim, toP, toLim) \ - (((enc)->utf8Convert)(enc, fromP, fromLim, toP, toLim)) +# define XmlUtf8Convert(enc, fromP, fromLim, toP, toLim) \ + (((enc)->utf8Convert)(enc, fromP, fromLim, toP, toLim)) -#define XmlUtf16Convert(enc, fromP, fromLim, toP, toLim) \ - (((enc)->utf16Convert)(enc, fromP, fromLim, toP, toLim)) +# define XmlUtf16Convert(enc, fromP, fromLim, toP, toLim) \ + (((enc)->utf16Convert)(enc, fromP, fromLim, toP, toLim)) typedef struct { ENCODING initEnc; @@ -293,13 +292,13 @@ int XmlInitEncoding(INIT_ENCODING *p, const ENCODING **encPtr, const char *name); const ENCODING *XmlGetUtf8InternalEncoding(void); const ENCODING *XmlGetUtf16InternalEncoding(void); -int FASTCALL XmlUtf8Encode(int charNumber, char *buf); -int FASTCALL XmlUtf16Encode(int charNumber, unsigned short *buf); +int XmlUtf8Encode(int charNumber, char *buf); +int XmlUtf16Encode(int charNumber, unsigned short *buf); int XmlSizeOfUnknownEncoding(void); typedef int(XMLCALL *CONVERTER)(void *userData, const char *p); -ENCODING *XmlInitUnknownEncoding(void *mem, int *table, CONVERTER convert, +ENCODING *XmlInitUnknownEncoding(void *mem, const int *table, CONVERTER convert, void *userData); int XmlParseXmlDeclNS(int isGeneralTextEntity, const ENCODING *enc, @@ -312,10 +311,7 @@ int XmlInitEncodingNS(INIT_ENCODING *p, const ENCODING **encPtr, const char *name); const ENCODING *XmlGetUtf8InternalEncodingNS(void); const ENCODING *XmlGetUtf16InternalEncodingNS(void); -ENCODING *XmlInitUnknownEncodingNS(void *mem, int *table, CONVERTER convert, - void *userData); -#ifdef __cplusplus -} -#endif +ENCODING *XmlInitUnknownEncodingNS(void *mem, const int *table, + CONVERTER convert, void *userData); #endif /* not XmlTok_INCLUDED */ diff --git a/Modules/expat/xmltok_impl.c b/Modules/expat/xmltok_impl.c index 239a2d06c4512c..7900e94701e276 100644 --- a/Modules/expat/xmltok_impl.c +++ b/Modules/expat/xmltok_impl.c @@ -1,4 +1,4 @@ -/* This file is included (from xmltok.c, 1-3 times depending on XML_MIN_SIZE)! +/* This file is included (from xmltok.c, 3 times)! __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -10,13 +10,15 @@ Copyright (c) 2000 Clark Cooper Copyright (c) 2002 Fred L. Drake, Jr. Copyright (c) 2002-2016 Karl Waclawek - Copyright (c) 2016-2022 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2017 Rhodri James Copyright (c) 2018 Benjamin Peterson Copyright (c) 2018 Anton Maklakov Copyright (c) 2019 David Loffredo Copyright (c) 2020 Boris Kolpackov Copyright (c) 2022 Martin Ettl + Copyright (c) 2026 Nick Begg + Copyright (c) 2026 Kartik Kenchi Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -37,14 +39,12 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifdef XML_TOK_IMPL_C -# ifndef IS_INVALID_CHAR // i.e. for UTF-16 and XML_MIN_SIZE not defined -# define IS_INVALID_CHAR(enc, ptr, n) (0) -# endif - # define INVALID_LEAD_CASE(n, ptr, nextTokPtr) \ case BT_LEAD##n: \ if (end - ptr < n) \ @@ -83,7 +83,7 @@ *nextTokPtr = ptr; \ return XML_TOK_INVALID; \ } \ - /* fall through */ \ + EXPAT_FALLTHROUGH; \ case BT_NMSTRT: \ case BT_HEX: \ case BT_DIGIT: \ @@ -112,7 +112,7 @@ *nextTokPtr = ptr; \ return XML_TOK_INVALID; \ } \ - /* fall through */ \ + EXPAT_FALLTHROUGH; \ case BT_NMSTRT: \ case BT_HEX: \ ptr += MINBPC(enc); \ @@ -141,7 +141,7 @@ /* ptr points to character following "= end) @@ -427,7 +427,7 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr, const char *end, /* ptr points to character following "= end) @@ -898,7 +898,7 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_INVALID; } } - /* fall through */ + EXPAT_FALLTHROUGH; case BT_AMP: case BT_LT: case BT_NONXML: @@ -919,7 +919,7 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end, /* ptr points to character following "%" */ -static int PTRCALL +static int PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { REQUIRE_CHAR(enc, ptr, end); @@ -949,7 +949,7 @@ PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_PARTIAL; } -static int PTRCALL +static int PREFIX(scanPoundName)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { REQUIRE_CHAR(enc, ptr, end); @@ -979,7 +979,7 @@ PREFIX(scanPoundName)(const ENCODING *enc, const char *ptr, const char *end, return -XML_TOK_POUND_NAME; } -static int PTRCALL +static int PREFIX(scanLit)(int open, const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { while (HAS_CHAR(enc, ptr, end)) { @@ -1013,7 +1013,7 @@ PREFIX(scanLit)(int open, const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_PARTIAL; } -static int PTRCALL +static int PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { int tok; @@ -1059,7 +1059,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, /* indicate that this might be part of a CR/LF pair */ return -XML_TOK_PROLOG_S; } - /* fall through */ + EXPAT_FALLTHROUGH; case BT_S: case BT_LF: for (;;) { @@ -1074,7 +1074,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, /* don't split CR/LF pair */ if (ptr + MINBPC(enc) != end) break; - /* fall through */ + EXPAT_FALLTHROUGH; default: *nextTokPtr = ptr; return XML_TOK_PROLOG_S; @@ -1189,7 +1189,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, tok = XML_TOK_NMTOKEN; break; } - /* fall through */ + EXPAT_FALLTHROUGH; default: *nextTokPtr = ptr; return XML_TOK_INVALID; @@ -1257,7 +1257,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, return -tok; } -static int PTRCALL +static int PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { const char *start; @@ -1326,7 +1326,7 @@ PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_DATA_CHARS; } -static int PTRCALL +static int PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { const char *start; @@ -1393,7 +1393,7 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr, const char *end, # ifdef XML_DTD -static int PTRCALL +static int PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { int level = 0; @@ -1445,7 +1445,7 @@ PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr, const char *end, # endif /* XML_DTD */ -static int PTRCALL +static int PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end, const char **badPtr) { ptr += MINBPC(enc); @@ -1484,7 +1484,7 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end, case BT_NMSTRT: if (! (BYTE_TO_ASCII(enc, ptr) & ~0x7f)) break; - /* fall through */ + EXPAT_FALLTHROUGH; default: switch (BYTE_TO_ASCII(enc, ptr)) { case 0x24: /* $ */ @@ -1505,7 +1505,7 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end, first attsMax attributes are stored in atts. */ -static int PTRCALL +static int PREFIX(getAtts)(const ENCODING *enc, const char *ptr, int attsMax, ATTRIBUTE *atts) { enum { other, inName, inValue } state = inName; @@ -1598,7 +1598,7 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr, int attsMax, /* not reached */ } -static int PTRFASTCALL +static int PREFIX(charRefNumber)(const ENCODING *enc, const char *ptr) { int result = 0; /* skip &# */ @@ -1656,7 +1656,7 @@ PREFIX(charRefNumber)(const ENCODING *enc, const char *ptr) { return checkCharRefNumber(result); } -static int PTRCALL +static int PREFIX(predefinedEntityName)(const ENCODING *enc, const char *ptr, const char *end) { UNUSED_P(enc); @@ -1710,7 +1710,7 @@ PREFIX(predefinedEntityName)(const ENCODING *enc, const char *ptr, return 0; } -static int PTRCALL +static int PREFIX(nameMatchesAscii)(const ENCODING *enc, const char *ptr1, const char *end1, const char *ptr2) { UNUSED_P(enc); @@ -1729,7 +1729,7 @@ PREFIX(nameMatchesAscii)(const ENCODING *enc, const char *ptr1, return ptr1 == end1; } -static int PTRFASTCALL +static int PREFIX(nameLength)(const ENCODING *enc, const char *ptr) { const char *start = ptr; for (;;) { @@ -1759,7 +1759,7 @@ PREFIX(nameLength)(const ENCODING *enc, const char *ptr) { } } -static const char *PTRFASTCALL +static const char * PREFIX(skipS)(const ENCODING *enc, const char *ptr) { for (;;) { switch (BYTE_TYPE(enc, ptr)) { @@ -1774,7 +1774,7 @@ PREFIX(skipS)(const ENCODING *enc, const char *ptr) { } } -static void PTRCALL +static void PREFIX(updatePosition)(const ENCODING *enc, const char *ptr, const char *end, POSITION *pos) { while (HAS_CHAR(enc, ptr, end)) { diff --git a/Modules/expat/xmltok_impl.h b/Modules/expat/xmltok_impl.h index 3469c4ae138c95..13a08899da3b1a 100644 --- a/Modules/expat/xmltok_impl.h +++ b/Modules/expat/xmltok_impl.h @@ -29,6 +29,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ enum { diff --git a/Modules/expat/xmltok_ns.c b/Modules/expat/xmltok_ns.c index fbdd3e3c7b7999..d9f69abb92ef39 100644 --- a/Modules/expat/xmltok_ns.c +++ b/Modules/expat/xmltok_ns.c @@ -11,7 +11,8 @@ Copyright (c) 2002 Greg Stein Copyright (c) 2002 Fred L. Drake, Jr. Copyright (c) 2002-2006 Karl Waclawek - Copyright (c) 2017-2021 Sebastian Pipping + Copyright (c) 2017-2026 Sebastian Pipping + Copyright (c) 2025 Alfonso Gregory Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -32,6 +33,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifdef XML_TOK_NS_C @@ -61,14 +64,14 @@ static const ENCODING *const NS(encodings)[] = { &ns(utf8_encoding).enc /* NO_ENC */ }; -static int PTRCALL +static int NS(initScanProlog)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { return initScan(NS(encodings), (const INIT_ENCODING *)enc, XML_PROLOG_STATE, ptr, end, nextTokPtr); } -static int PTRCALL +static int NS(initScanContent)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { return initScan(NS(encodings), (const INIT_ENCODING *)enc, XML_CONTENT_STATE, @@ -98,13 +101,13 @@ NS(findEncoding)(const ENCODING *enc, const char *ptr, const char *end) { int i; XmlUtf8Convert(enc, &ptr, end, &p, p + ENCODING_MAX - 1); if (ptr != end) - return 0; + return NULL; *p = 0; if (streqci(buf, KW_UTF_16) && enc->minBytesPerChar == 2) return enc; i = getEncodingIndex(buf); if (i == UNKNOWN_ENC) - return 0; + return NULL; return NS(encodings)[i]; } diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index e5d108228f62ef..4ad7de27c6206e 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -282,6 +282,10 @@ extern int lstat(const char *, struct stat *); #define pclose _pclose #endif /* _MSC_VER */ +#ifdef MS_WINDOWS +#include /* for ConvertStringSecurityDescriptorToSecurityDescriptorW() */ +#endif + #if defined(PYCC_VACPP) && defined(PYOS_OS2) #include #endif /* OS2 */ @@ -2564,7 +2568,52 @@ posix__getfullpathname(PyObject *self, PyObject *args) PyDoc_STRVAR(posix_mkdir__doc__, "mkdir(path [, mode=0777])\n\n\ -Create a directory."); +Create a directory.\n\ +\n\ +On Windows, a mode of 0700 is specifically handled to apply access control\n\ +to the new directory such that only the current user and administrators\n\ +have access. Other values of mode are ignored."); + +#ifdef MS_WINDOWS +/* gh-118486 (CVE-2024-4030): for mode 0700, build a security descriptor + whose protected discretionary ACL (D:P) only has inheritable (OICI) + full-control (FA) entries for SYSTEM (SY), Administrators (BA) and the + owner (OW). Returns 0 or a Windows error code. On success with mode + 0700, *pSecAttr points to secAttr and secAttr->lpSecurityDescriptor + must be released with LocalFree(). */ +static DWORD +win32_mkdir_security(int mode, SECURITY_ATTRIBUTES *secAttr, + SECURITY_ATTRIBUTES **pSecAttr) +{ + ULONG sdSize; + + ZeroMemory(secAttr, sizeof(*secAttr)); + secAttr->nLength = sizeof(*secAttr); + *pSecAttr = NULL; + if (mode != 0700) + return 0; + if (!ConvertStringSecurityDescriptorToSecurityDescriptorW( + L"D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)", + SDDL_REVISION_1, + &secAttr->lpSecurityDescriptor, + &sdSize)) + return GetLastError(); + *pSecAttr = secAttr; + return 0; +} + +/* Release the descriptor from win32_mkdir_security(). Returns 0 or a + Windows error code; uncommonly, LocalFree() returns non-NULL on error + and reports the error code through GetLastError(). */ +static DWORD +win32_mkdir_security_free(SECURITY_ATTRIBUTES *secAttr) +{ + if (secAttr->lpSecurityDescriptor && + LocalFree(secAttr->lpSecurityDescriptor)) + return GetLastError(); + return 0; +} +#endif /* MS_WINDOWS */ static PyObject * posix_mkdir(PyObject *self, PyObject *args) @@ -2575,10 +2624,21 @@ posix_mkdir(PyObject *self, PyObject *args) #ifdef MS_WINDOWS Py_UNICODE *wpath; + DWORD error; + SECURITY_ATTRIBUTES secAttr; + SECURITY_ATTRIBUTES *pSecAttr; + if (PyArg_ParseTuple(args, "u|i:mkdir", &wpath, &mode)) { + res = 0; Py_BEGIN_ALLOW_THREADS - res = CreateDirectoryW(wpath, NULL); + error = win32_mkdir_security(mode, &secAttr, &pSecAttr); + if (!error) { + res = CreateDirectoryW(wpath, pSecAttr); + error = win32_mkdir_security_free(&secAttr); + } Py_END_ALLOW_THREADS + if (error) + return PyErr_SetFromWindowsErr(error); if (!res) return win32_error_unicode("mkdir", wpath); Py_INCREF(Py_None); @@ -2590,9 +2650,19 @@ posix_mkdir(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "et|i:mkdir", Py_FileSystemDefaultEncoding, &path, &mode)) return NULL; + res = 0; Py_BEGIN_ALLOW_THREADS - res = CreateDirectoryA(path, NULL); + error = win32_mkdir_security(mode, &secAttr, &pSecAttr); + if (!error) { + res = CreateDirectoryA(path, pSecAttr); + error = win32_mkdir_security_free(&secAttr); + } Py_END_ALLOW_THREADS + if (error) { + PyErr_SetFromWindowsErr(error); + PyMem_Free(path); + return NULL; + } if (!res) { win32_error("mkdir", path); PyMem_Free(path); diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index a1a870d418e8b3..81a9ee26ef0fc4 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -8,6 +8,15 @@ #define XML_COMBINED_VERSION (10000*XML_MAJOR_VERSION+100*XML_MINOR_VERSION+XML_MICRO_VERSION) +#if XML_COMBINED_VERSION >= 20800 +/* Process-wide hash salt for XML_SetHashSalt16Bytes(), read from the OS + random source at import (gh-149018). 2.7 has no 16-byte expat secret + in _Py_HashSecret, and _Py_HashSecret is all zeros unless hash + randomization (-R) is enabled. */ +static uint8_t expat_hash_salt16[16]; +static int expat_hash_salt16_set = 0; +#endif + #ifndef PyDoc_STRVAR /* @@ -692,9 +701,14 @@ conv_content_model(XML_Content * const model, PyObject *(*conv_string)(const XML_Char *)) { PyObject *result = NULL; - PyObject *children = PyTuple_New(model->numchildren); + PyObject *children; int i; + if (Py_EnterRecursiveCall(" in conv_content_model")) { + return NULL; + } + + children = PyTuple_New(model->numchildren); if (children != NULL) { assert(model->numchildren < INT_MAX); for (i = 0; i < (int)model->numchildren; ++i) { @@ -702,7 +716,7 @@ conv_content_model(XML_Content * const model, conv_string); if (child == NULL) { Py_XDECREF(children); - return NULL; + goto done; } PyTuple_SET_ITEM(children, i, child); } @@ -710,6 +724,8 @@ conv_content_model(XML_Content * const model, model->type, model->quant, conv_string,model->name, children); } +done: + Py_LeaveRecursiveCall(); return result; } @@ -1318,7 +1334,14 @@ newxmlparseobject(char *encoding, char *namespace_separator, PyObject *intern) Py_DECREF(self); return NULL; } -#if XML_COMBINED_VERSION >= 20100 +#if XML_COMBINED_VERSION >= 20800 + /* Prefer 16 bytes of entropy, only expat >= 2.8.0. See gh-149018. */ + if (expat_hash_salt16_set) + XML_SetHashSalt16Bytes(self->itself, expat_hash_salt16); + else + XML_SetHashSalt(self->itself, + (unsigned long)_Py_HashSecret.prefix); +#elif XML_COMBINED_VERSION >= 20100 /* This feature was added upstream in libexpat 2.1.0. */ XML_SetHashSalt(self->itself, (unsigned long)_Py_HashSecret.prefix); @@ -1843,6 +1866,12 @@ MODULE_INITFUNC(void) if (errmod_name == NULL) return; +#if XML_COMBINED_VERSION >= 20800 + if (_PyOS_URandom(expat_hash_salt16, sizeof(expat_hash_salt16)) == 0) + expat_hash_salt16_set = 1; + else + PyErr_Clear(); +#endif modelmod_name = PyString_FromString(MODULE_NAME ".model"); if (modelmod_name == NULL) return; @@ -2045,6 +2074,11 @@ MODULE_INITFUNC(void) #else capi.SetHashSalt = NULL; #endif +#if XML_COMBINED_VERSION >= 20800 + capi.SetHashSalt16Bytes = XML_SetHashSalt16Bytes; +#else + capi.SetHashSalt16Bytes = NULL; +#endif /* export using capsule */ capi_object = PyCapsule_New(&capi, PyExpat_CAPSULE_NAME, NULL); diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index b145daf5c88b6d..69472cf2d95e9c 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -1836,47 +1836,6 @@ PyDoc_STRVAR(getblocking_doc, Returns True if socket is in blocking mode, or False if it\n\ is in non-blocking mode."); -static int -socket_parse_timeout(_PyTime_t *timeout, PyObject *timeout_obj) -{ -#ifdef MS_WINDOWS - struct timeval tv; -#endif -#ifndef HAVE_POLL - _PyTime_t ms; -#endif - int overflow = 0; - - if (timeout_obj == Py_None) { - *timeout = _PyTime_FromSeconds(-1); - return 0; - } - - if (_PyTime_FromSecondsObject(timeout, - timeout_obj, _PyTime_ROUND_TIMEOUT) < 0) - return -1; - - if (*timeout < 0) { - PyErr_SetString(PyExc_ValueError, "Timeout value out of range"); - return -1; - } - -#ifdef MS_WINDOWS - overflow |= (_PyTime_AsTimeval(*timeout, &tv, _PyTime_ROUND_TIMEOUT) < 0); -#endif -#ifndef HAVE_POLL - ms = _PyTime_AsMilliseconds(*timeout, _PyTime_ROUND_TIMEOUT); - overflow |= (ms > INT_MAX); -#endif - if (overflow) { - PyErr_SetString(PyExc_OverflowError, - "timeout doesn't fit into C timeval"); - return -1; - } - - return 0; -} - /* s.settimeout(timeout) method. Argument: None -- no timeout, blocking mode; same as setblocking(True) 0.0 -- non-blocking mode; same as setblocking(False) diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index df6ffe343c3f1c..aedf41f7d44f4d 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -493,6 +493,56 @@ get_decomp_record(PyObject *self, Py_UCS4 code, int *index, int *prefix, int *co #define NCount (VCount*TCount) #define SCount (LCount*NCount) +/* Small combining runs are usually cheaper with insertion sort. */ +#define CANONICAL_ORDERING_COUNTING_SORT_THRESHOLD 20 + +static void +canonical_ordering_sort_insertion(Py_UNICODE *start, Py_UNICODE *end) +{ + Py_UNICODE *i, *j; + + for (i = start + 1; i < end; i++) { + Py_UNICODE code = *i; + unsigned char combining = _getrecord_ex(code)->combining; + + j = i; + while (j > start) { + if (_getrecord_ex(j[-1])->combining <= combining) + break; + *j = j[-1]; + j--; + } + if (j != i) + *j = code; + } +} + +static void +canonical_ordering_sort_counting(Py_UNICODE *start, Py_UNICODE *end, + Py_UNICODE *sortbuf) +{ + Py_ssize_t counts[256]; + Py_ssize_t total = 0; + Py_UNICODE *i; + int c; + + memset(counts, 0, sizeof(counts)); + for (i = start; i < end; i++) + counts[_getrecord_ex(*i)->combining]++; + + for (c = 0; c < 256; c++) { + Py_ssize_t count = counts[c]; + counts[c] = total; + total += count; + } + + /* Reuse counts[] as the next output slot for each CCC; this is a + stable sort, as canonical ordering requires. */ + for (i = start; i < end; i++) + sortbuf[counts[_getrecord_ex(*i)->combining]++] = *i; + memcpy(start, sortbuf, (end - start) * sizeof(Py_UNICODE)); +} + static PyObject* nfd_nfkd(PyObject *self, PyObject *input, int k) { @@ -503,6 +553,8 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) Py_ssize_t space, isize; int index, prefix, count, stackptr; unsigned char prev, cur; + Py_UNICODE *sortbuf = NULL; + Py_ssize_t sortbuflen = 0; stackptr = 0; isize = PyUnicode_GET_SIZE(input); @@ -581,31 +633,58 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) /* Drop overallocation. Cannot fail. */ PyUnicode_Resize(&result, PyUnicode_GET_SIZE(result) - space); - /* Sort canonically. */ + /* Sort each consecutive combining-character run canonically. The + old in-place insertion sort was O(n**2) in the run length + (gh-149079, CVE-2026-3276). */ i = PyUnicode_AS_UNICODE(result); - prev = _getrecord_ex(*i)->combining; end = i + PyUnicode_GET_SIZE(result); - for (i++; i < end; i++) { - cur = _getrecord_ex(*i)->combining; - if (prev == 0 || cur == 0 || prev <= cur) { - prev = cur; + while (i < end) { + Py_UNICODE *run_start; + Py_ssize_t run_length; + int needs_sort = 0; + + prev = _getrecord_ex(*i)->combining; + if (prev == 0) { + i++; continue; } - /* Non-canonical order. Need to switch *i with previous. */ - o = i - 1; - while (1) { - Py_UNICODE tmp = o[1]; - o[1] = o[0]; - o[0] = tmp; - o--; - if (o < PyUnicode_AS_UNICODE(result)) - break; - prev = _getrecord_ex(*o)->combining; - if (prev == 0 || prev <= cur) + + run_start = i++; + while (i < end) { + cur = _getrecord_ex(*i)->combining; + if (cur == 0) break; + if (prev > cur) + needs_sort = 1; + prev = cur; + i++; } - prev = _getrecord_ex(*i)->combining; + if (!needs_sort) + continue; + + run_length = i - run_start; + if (run_length < CANONICAL_ORDERING_COUNTING_SORT_THRESHOLD) { + canonical_ordering_sort_insertion(run_start, i); + continue; + } + + if (run_length > sortbuflen) { + Py_UNICODE *new_sortbuf = NULL; + if ((size_t)run_length <= PY_SSIZE_T_MAX / sizeof(Py_UNICODE)) + new_sortbuf = (Py_UNICODE *)PyMem_Realloc( + sortbuf, run_length * sizeof(Py_UNICODE)); + if (new_sortbuf == NULL) { + PyErr_NoMemory(); + PyMem_Free(sortbuf); + Py_DECREF(result); + return NULL; + } + sortbuf = new_sortbuf; + sortbuflen = run_length; + } + canonical_ordering_sort_counting(run_start, i, sortbuf); } + PyMem_Free(sortbuf); return result; } diff --git a/PCbuild/_elementtree.vcxproj b/PCbuild/_elementtree.vcxproj index 511e26bb60dbd8..64586e173f8cb3 100644 --- a/PCbuild/_elementtree.vcxproj +++ b/PCbuild/_elementtree.vcxproj @@ -62,7 +62,7 @@ ..\Modules\expat;%(AdditionalIncludeDirectories) - _CRT_SECURE_NO_WARNINGS;USE_PYEXPAT_CAPI;XML_STATIC;%(PreprocessorDefinitions) + _CRT_SECURE_NO_WARNINGS;USE_PYEXPAT_CAPI;XML_POOR_ENTROPY;XML_STATIC;%(PreprocessorDefinitions) 0x1D100000 @@ -74,19 +74,23 @@ + + - + + + diff --git a/PCbuild/_elementtree.vcxproj.filters b/PCbuild/_elementtree.vcxproj.filters index 6acdf35846ab14..b8a02b047dd584 100644 --- a/PCbuild/_elementtree.vcxproj.filters +++ b/PCbuild/_elementtree.vcxproj.filters @@ -24,6 +24,12 @@ Header Files + + Header Files + + + Header Files + Header Files @@ -33,7 +39,7 @@ Header Files - + Header Files @@ -48,6 +54,9 @@ Header Files + + Header Files + Header Files @@ -59,6 +68,9 @@ Source Files + + Source Files + Source Files diff --git a/setup.py b/setup.py index 927572c949cbe7..ba2e81f415697e 100644 --- a/setup.py +++ b/setup.py @@ -1508,7 +1508,8 @@ class db_found(Exception): pass ('XML_POOR_ENTROPY', '1'), ] expat_lib = [] - expat_sources = ['expat/xmlparse.c', + expat_sources = ['expat/xcs.c', + 'expat/xmlparse.c', 'expat/xmlrole.c', 'expat/xmltok.c'] expat_depends = ['expat/ascii.h', @@ -1519,6 +1520,7 @@ class db_found(Exception): pass 'expat/internal.h', 'expat/latin1tab.h', 'expat/utf8tab.h', + 'expat/xcs.h', 'expat/xmlrole.h', 'expat/xmltok.h', 'expat/xmltok_impl.h'