diff --git a/Doc/conf.py b/Doc/conf.py index 178e52033b67293..2ab63d2e3f32680 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -15,7 +15,8 @@ extensions = ['sphinx.ext.coverage', 'sphinx.ext.doctest', 'pyspecific', 'c_annotations', 'escape4chm', - 'asdl_highlight', 'peg_highlight', 'glossary_search'] + 'asdl_highlight', 'peg_highlight', 'glossary_search', + 'sphinx.ext.extlinks'] doctest_global_setup = ''' try: @@ -234,3 +235,8 @@ # Relative filename of the data files refcount_file = 'data/refcounts.dat' stable_abi_file = 'data/stable_abi.dat' + +# sphinx.ext.extlinks config +extlinks = { + 'cve': ('https://www.cve.org/CVERecord?id=CVE-%s', 'CVE-%s'), +} diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index a8fdef14d7b2538..67b63eabb06978b 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -957,6 +957,11 @@ reused in custom filters: - :ref:`Refuse ` to extract files whose absolute path (after following symlinks) would end up outside the destination. This raises :class:`~tarfile.OutsideDestinationError`. + - :ref:`Refuse ` to extract links (hard or soft) + whose name refers to the destination directory itself. + This raises :class:`~tarfile.OutsideDestinationError`. + Note that such files are refused even on platforms that do not support + symbolic links. - Clear high mode bits (setuid, setgid, sticky) and group/other write bits (:attr:`~stat.S_IWGRP`|:attr:`~stat.S_IWOTH`). diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index f8081c4f982478c..6968f620f9cca89 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -438,6 +438,7 @@ def run(self): issue_re = re.compile('(?:[Ii]ssue #|bpo-)([0-9]+)', re.I) gh_issue_re = re.compile('(?:gh-issue-|gh-)([0-9]+)', re.I) +cve_re = re.compile('CVE-([0-9]{4}-[0-9]{4,})', re.I) whatsnew_re = re.compile(r"(?im)^what's new in (.*?)\??$") @@ -467,6 +468,7 @@ def run(self): content = issue_re.sub(r':issue:`\1`', content) # Fallback handling for the GitHub issue content = gh_issue_re.sub(r':gh:`\1`', content) + content = cve_re.sub(r':cve:`\1`', content) content = whatsnew_re.sub(r'\1', content) # remove first 3 lines as they are the main heading lines = ['.. default-role:: obj', ''] + content.splitlines()[3:] diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index f07b70a9ec9622c..28e9fd3b7203305 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -2428,3 +2428,69 @@ tarfile :func:`~tarfile.TarFile.errorlevel` is zero. (Contributed by Matt Prodani and Petr Viktorin in :gh:`112887` and CVE 2025-4435.) + + +Notable changes in 3.10.22.1 +============================ + +This is an ActiveState security release of the 3.10 line, which is past +its upstream end of life. + +It carries the security fixes from upstream Python 3.11.18 (released +2026-10-25) that apply to 3.10, on top of 3.10.22, plus three fixes that +upstream released only for newer lines (or has not released yet): the +:mod:`imaplib`, :mod:`poplib` and :mod:`http.client` items below. The full +list of changes is in the changelog; the notable ones are below. + +ssl +--- + +* The :mod:`ssl` module now rejects TLS certificate chains that contain one + of a set of compromised eMudhra intermediate CA certificates. The affected + certificates are identified by their public key, so cross-signed copies that + reuse the same key are blocked as well. It is a defense-in-depth backstop + that applies even when an application supplies its own root CA bundle, and + regardless of whether the platform trust store has removed trust. + (Contributed by Christian Heimes in :gh:`160040` for CVE 2026-115557.) + +ensurepip +--------- + +* The bundled ``pip`` is upgraded from 23.0.1 to 26.2.2. pip loads its + own vendored ``certifi`` CA bundle in addition to the system trust store, + and the bundle in pip 23.0.1 still trusts the compromised eMudhra emSign + root; pip 26.2.2 ships certifi 2026.10.19, which removes it. + (:gh:`159894`) +* The bundled ``setuptools`` is upgraded from 79.0.1 to 84.2.1, fixing + CVE 2026-59890 and GHSA-xmj3-9gf7-h52w. setuptools removed + ``pkg_resources`` in 82.0.0, so environments seeded by :mod:`ensurepip` or + :mod:`venv` no longer include it; install an older setuptools if an + application still imports it. + +tarfile +------- + +* :func:`tarfile.tar_filter` now rejects link members whose path resolves to + the destination directory itself, as :func:`tarfile.data_filter` already + did. (:gh:`160097`, CVE 2026-114583.) + +Other security fixes +-------------------- + +* :class:`http.cookiejar.DefaultCookiePolicy` no longer accepts cookies with + a negative ``Version`` attribute without checking their domain. + (:gh:`160300`) +* :mod:`mimetypes` no longer reads :data:`mimetypes.knownfiles` by default on + Windows, where those paths resolve to the root of the current drive. + (:gh:`159428`) +* The bundled libexpat is updated to 2.9.0. (:gh:`158876`) +* :mod:`email` header refolding now re-quotes text that came from an + encoded word. (:gh:`87720`) +* :mod:`imaplib` rejects NUL, CR and LF in command arguments, which could + be used to inject extra IMAP commands. (:gh:`143921`, CVE 2025-15366.) +* :mod:`poplib` rejects control characters in commands, which could be used + to inject extra POP3 commands. (:gh:`143923`, CVE 2025-15367.) +* :mod:`http.client` only accepts a ``Content-Length`` and chunk sizes that + match the HTTP grammar (digits, and hex digits for chunk sizes); signs, + underscores and ``0x`` prefixes are no longer accepted. + (:gh:`150751`, CVE 2026-103995.) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index d44eb4fa98c34d4..d3c764d83a366b0 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -23,7 +23,7 @@ #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.10.22" +#define PY_VERSION "3.10.22.1" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 2c05abeadea22b5..985ff2088e55d43 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -168,6 +168,14 @@ def comments(self): comments.extend(token.comments) return comments + def contains_token_type(self, *token_types): + if self.token_type in token_types: + return True + for t in self: + if t.contains_token_type(*token_types): + return True + return False + def fold(self, *, policy): return _refold_parse_tree(self, policy=policy) @@ -922,6 +930,9 @@ def pop_trailing_ws(self): def comments(self): return [] + def contains_token_type(self, *token_types): + return self.token_type in token_types + def __getnewargs__(self): return(str(self), self.token_type) @@ -2795,7 +2806,7 @@ def _refold_parse_tree(parse_tree, *, policy): continue tstr = str(part) if not want_encoding: - if part.token_type in ('ptext', 'vtext'): + if part.token_type == 'ptext' or part.contains_token_type('vtext'): # Encode if tstr contains special characters. want_encoding = not SPECIALSNL.isdisjoint(tstr) else: diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 94ddda442ade834..69f608e8e613aaa 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -11,8 +11,8 @@ __all__ = ["version", "bootstrap"] _PACKAGE_NAMES = ('setuptools', 'pip') -_SETUPTOOLS_VERSION = "79.0.1" -_PIP_VERSION = "23.0.1" +_SETUPTOOLS_VERSION = "84.2.1" +_PIP_VERSION = "26.2.2" _PROJECTS = [ ("setuptools", _SETUPTOOLS_VERSION, "py3"), ("pip", _PIP_VERSION, "py3"), diff --git a/Lib/ensurepip/_bundled/pip-23.0.1-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-23.0.1-py3-none-any.whl deleted file mode 100644 index a855dc40e8630d1..000000000000000 Binary files a/Lib/ensurepip/_bundled/pip-23.0.1-py3-none-any.whl and /dev/null differ diff --git a/Lib/ensurepip/_bundled/pip-26.2.2-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-26.2.2-py3-none-any.whl new file mode 100644 index 000000000000000..d884a0ad81170c1 Binary files /dev/null and b/Lib/ensurepip/_bundled/pip-26.2.2-py3-none-any.whl differ diff --git a/Lib/ensurepip/_bundled/setuptools-79.0.1-py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-79.0.1-py3-none-any.whl deleted file mode 100644 index 3eb896915897fb7..000000000000000 Binary files a/Lib/ensurepip/_bundled/setuptools-79.0.1-py3-none-any.whl and /dev/null differ diff --git a/Lib/ensurepip/_bundled/setuptools-84.2.1-py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-84.2.1-py3-none-any.whl new file mode 100644 index 000000000000000..f708be8904f2d5e Binary files /dev/null and b/Lib/ensurepip/_bundled/setuptools-84.2.1-py3-none-any.whl differ diff --git a/Lib/http/client.py b/Lib/http/client.py index c05d33b79cc15f5..8c7706a88b9266f 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -165,6 +165,11 @@ # to prevent http header injection. _contains_disallowed_method_pchar_re = re.compile('[\x00-\x1f]') +# RFC 9112: Content-Length = 1*DIGIT and chunk-size = 1*HEXDIG. +# int() is more permissive, so we match against the grammar before calling it. +_is_legal_content_length = re.compile(r'[0-9]+').fullmatch +_is_legal_chunk_size = re.compile(rb'[0-9a-fA-F]+').fullmatch + # We always set the Content-Length header for these methods because some # servers will otherwise respond with a 411 _METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'} @@ -370,14 +375,12 @@ def begin(self): # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked" self.length = None length = self.headers.get("content-length") - if length and not self.chunked: - try: - self.length = int(length) - except ValueError: - self.length = None - else: - if self.length < 0: # ignore nonsensical negative lengths - self.length = None + if length is not None: + # RFC 9112, section 5.1: optional whitespace (SP / HTAB) around + # the field value is not part of the value. + length = length.strip(' \t') + if length and not self.chunked and _is_legal_content_length(length): + self.length = int(length) else: self.length = None @@ -544,7 +547,10 @@ def _read_next_chunk_size(self): i = line.find(b";") if i >= 0: line = line[:i] # strip chunk-extensions + line = line.rstrip() try: + if not _is_legal_chunk_size(line): + raise ValueError("invalid chunk size") return int(line, 16) except ValueError: # close the connection as protocol synchronisation is diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index 0380d9f1096c080..e6692c6325a0797 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -973,6 +973,12 @@ def set_ok_version(self, cookie, request): _debug(" Set-Cookie2 without version attribute (%s=%s)", cookie.name, cookie.value) return False + if cookie.version < 0: + # Neither the Netscape (version 0) nor the RFC 2965 (version > 0) + # rules would apply, so the domain checks would all be skipped. + _debug(" negative version attribute (%s=%s)", + cookie.name, cookie.value) + return False if cookie.version > 0 and not self.rfc2965: _debug(" RFC 2965 cookies are switched off") return False @@ -1112,6 +1118,9 @@ def return_ok(self, cookie, request): return True def return_ok_version(self, cookie, request): + if cookie.version < 0: + _debug(" negative version attribute") + return False if cookie.version > 0 and not self.rfc2965: _debug(" RFC 2965 cookies are switched off") return False @@ -1505,6 +1514,8 @@ def _cookie_from_cookie_tuple(self, tup, request): version = int(version) except ValueError: return None # invalid version, ignore cookie + if version < 0: + return None # invalid version, ignore cookie secure = standard.get("secure", False) # (discard is also set if expires is Absent) discard = standard.get("discard", False) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 54122f985b9aca2..6c5d2dabbff536e 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -132,7 +132,9 @@ # We compile these in _mode_xxx. _Literal = br'.*{(?P\d+)}$' _Untagged_status = br'\* (?P\d+) (?P[A-Z-]+)( (?P.*))?' - +# Only NUL, CR and LF are unsafe (they cannot be represented even in +# a quoted string, and CR/LF would end the command line early). +_control_chars = re.compile(b'[\x00\r\n]') class IMAP4: @@ -994,6 +996,8 @@ def _command(self, name, *args): if arg is None: continue if isinstance(arg, str): arg = bytes(arg, self._encoding) + if _control_chars.search(arg): + raise ValueError("NUL, CR and LF not allowed in commands") data = data + b' ' + arg literal = self.literal diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index b72ce083a69d060..9b7df5eec59cc30 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -45,17 +45,23 @@ "suffix_map", "encodings_map", "types_map", "common_types" ] -knownfiles = [ - "/etc/mime.types", - "/etc/httpd/mime.types", # Mac OS X - "/etc/httpd/conf/mime.types", # Apache - "/etc/apache/mime.types", # Apache 1 - "/etc/apache2/mime.types", # Apache 2 - "/usr/local/etc/httpd/conf/mime.types", - "/usr/local/lib/netscape/mime.types", - "/usr/local/etc/httpd/conf/mime.types", # Apache 1.2 - "/usr/local/etc/mime.types", # Apache 1.3 - ] +if sys.platform == "win32": + # The default paths would resolve to the root of the current drive on Windows + # (e.g. C:\etc\mime.types), which unprivileged users can usually create. + # Only the registry is used by default. + knownfiles = [] +else: + knownfiles = [ + "/etc/mime.types", + "/etc/httpd/mime.types", # Mac OS X + "/etc/httpd/conf/mime.types", # Apache + "/etc/apache/mime.types", # Apache 1 + "/etc/apache2/mime.types", # Apache 2 + "/usr/local/etc/httpd/conf/mime.types", + "/usr/local/lib/netscape/mime.types", + "/usr/local/etc/httpd/conf/mime.types", # Apache 1.2 + "/usr/local/etc/mime.types", # Apache 1.3 + ] inited = False _db = None diff --git a/Lib/poplib.py b/Lib/poplib.py index 0f8587317c2bbc4..f563030f7f3c0e0 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -122,6 +122,8 @@ def _putline(self, line): def _putcmd(self, line): if self._debugging: print('*cmd*', repr(line)) line = bytes(line, self.encoding) + if re.search(b'[\x00-\x1F\x7F]', line): + raise ValueError('Control characters not allowed in commands') self._putline(line) diff --git a/Lib/tarfile.py b/Lib/tarfile.py index d27dbf88f74b158..41e91c664441782 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -788,6 +788,12 @@ def _get_filtered_attrs(member, dest_path, for_data=True): strict=os.path.ALLOW_MISSING) if os.path.commonpath([target_path, dest_path]) != dest_path: raise OutsideDestinationError(member, target_path) + # A link member named after the destination directory itself + # would replace it with a (sym)link, redirecting the destination + # for all subsequent members. + if ((member.islnk() or member.issym()) + and os.path.normpath(name) == os.curdir): + raise OutsideDestinationError(member, target_path) # Limit permissions (no high bits, and go-w) mode = member.mode if mode is not None: @@ -823,11 +829,6 @@ def _get_filtered_attrs(member, dest_path, for_data=True): if member.islnk() or member.issym(): if os.path.isabs(member.linkname): raise AbsoluteLinkError(member) - # A link member that resolves to the destination directory itself - # would replace it with a (sym)link, redirecting the destination - # for all subsequent members. - if target_path == dest_path: - raise OutsideDestinationError(member, target_path) normalized = os.path.normpath(member.linkname) if normalized != member.linkname: new_attrs['linkname'] = normalized diff --git a/Lib/test/distrusted.zip b/Lib/test/distrusted.zip new file mode 100644 index 000000000000000..83184329a5ab9e0 Binary files /dev/null and b/Lib/test/distrusted.zip differ diff --git a/Lib/test/make_ssl_certs.py b/Lib/test/make_ssl_certs.py index 94a35a64ab1abe3..9241347e363f387 100644 --- a/Lib/test/make_ssl_certs.py +++ b/Lib/test/make_ssl_certs.py @@ -5,6 +5,7 @@ import pprint import shutil import tempfile +import zipfile from subprocess import * startdate = "20180829142316Z" @@ -223,6 +224,107 @@ def print_cert(path): pprint.pprint(_ssl._test_decode_cert(path)) +# The distrusted test material (compromised eMudhra CA certs and a leaf signed +# by them) lives in a password-protected zip so it doesn't trip security +# scanners that cannot tell internal test fixtures from production secrets and +# flag any certificate or private key they find. It is encrypted, not merely +# zipped, because such scanners read inside plain archives. The password is +# fixed, public, and not a secret. +DISTRUSTED_ZIP = 'distrusted.zip' +DISTRUSTED_ZIP_DIR = 'distrusted' +DISTRUSTED_ZIP_CHOSEN_KNIGHT = 'Brave Sir Robin' + +# ASN.1 template for the intermediate's compromised key: secp384r1 scalar 4. +# openssl derives the public key when the SEC1 structure omits it. +distrusted_key_template = """ + asn1 = SEQUENCE:eckey + [eckey] + version = INTEGER:1 + priv = FORMAT:HEX,OCTETSTRING:{scalar} + params = EXPLICIT:0,OID:secp384r1 + """ + + +def make_distrusted_spki_cert(): + # leaf signed by the distrusted intermediate, chains to the distrusted root + print("creating distrusted-SPKI test cert") + # The standard library can read but not write encrypted zip files, so we + # need the Info-ZIP "zip" command to repackage the archive. + if shutil.which('zip') is None: + raise SystemExit( + f"the 'zip' command is required to write the encrypted {DISTRUSTED_ZIP} " + "(the zipfile module cannot create encrypted archives); " + "please install Info-ZIP's zip utility" + ) + + # the root and intermediate are real certs we cannot re-mint; carry them + # over from the existing archive + pw = DISTRUSTED_ZIP_CHOSEN_KNIGHT.encode() + with zipfile.ZipFile(DISTRUSTED_ZIP) as z: + root = z.read(DISTRUSTED_ZIP_DIR + '/distrusted_root.pem', pw) + intermediate = z.read(DISTRUSTED_ZIP_DIR + '/distrusted_intermediate.pem', pw) + + with tempfile.TemporaryDirectory() as workdir: + stage = os.path.join(workdir, DISTRUSTED_ZIP_DIR) + os.mkdir(stage) + req_file = os.path.join(workdir, 'req.cnf') + key_file = os.path.join(workdir, 'leaf.key') + csr_file = os.path.join(workdir, 'leaf.csr') + cert_file = os.path.join(workdir, 'leaf.pem') + cakey_conf = os.path.join(workdir, 'cakey.cnf') + cakey_der = os.path.join(workdir, 'cakey.der') + cakey_file = os.path.join(workdir, 'cakey.pem') + + inter_file = os.path.join(stage, 'distrusted_intermediate.pem') + with open(os.path.join(stage, 'distrusted_root.pem'), 'wb') as f: + f.write(root) + with open(inter_file, 'wb') as f: + f.write(intermediate) + + # recreate the intermediate's key (secp384r1 scalar 4) on the fly + with open(cakey_conf, 'w') as f: + f.write(distrusted_key_template.format(scalar='00' * 47 + '04')) + check_call(['openssl', 'asn1parse', '-genconf', cakey_conf, + '-out', cakey_der, '-noout']) + check_call(['openssl', 'pkey', '-in', cakey_der, + '-inform', 'DER', '-out', cakey_file]) + + req = req_template.format( + hostname='localhost', + extra_san='', + startdate=startdate, + enddate=enddate + ) + with open(req_file, 'w') as f: + f.write(req) + # ECC leaf to match the ECC intermediate + check_call(['openssl', 'req', '-new', '-nodes', '-days', '7000', + '-newkey', 'param:secp384r1.pem', '-keyout', key_file, + '-config', req_file, '-out', csr_file]) + # sign with the distrusted intermediate, not the test CA + check_call(['openssl', 'ca', '-config', req_file, + '-extensions', 'req_x509_extensions_full', + '-cert', inter_file, '-keyfile', cakey_file, + '-out', cert_file, '-outdir', 'cadir', + '-policy', 'policy_anything', '-batch', '-infiles', csr_file]) + with open(key_file) as f: + key = f.read() + with open(cert_file) as f: + cert = f.read() + # server sends leaf + intermediate, client trusts the root + with open(os.path.join(stage, 'distrusted_keycert.pem'), 'w') as f: + f.write(key) + f.write(cert) + f.write(intermediate.decode()) + + # repackage the encrypted archive (zipfile cannot write encryption) + zippath = os.path.join(here, DISTRUSTED_ZIP) + if os.path.exists(zippath): + os.remove(zippath) + check_call(['zip', '-q', '-X', '-P', DISTRUSTED_ZIP_CHOSEN_KNIGHT, '-r', + zippath, DISTRUSTED_ZIP_DIR], cwd=workdir) + + if __name__ == '__main__': os.chdir(here) cert, key = make_cert_key('localhost', ext='req_x509_extensions_simple') @@ -306,6 +408,8 @@ def print_cert(path): f.write(key) f.write(cert) + make_distrusted_spki_cert() + unmake_ca() print("update Lib/test/test_ssl.py and Lib/test/test_asyncio/utils.py") print_cert('keycert.pem') diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index 2eaaaaef675284b..cd82ed957ab54ce 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -3016,6 +3016,11 @@ def test_address_list_with_specials_in_encoded_word(self): 'A =?utf-8?q?v=C3=A9ry?= long name\n' ' containing =?utf-8?q?a=2C?= comma\n' ' \n'), + ('=?utf-8?Q?a=2C=20123456789012345678901234567890123456?=' + ' ', + '=?utf-8?q?a=2C?=\n' + ' 123456789012345678901234567890123456\n' + ' \n'), ] for (to, folded) in cases: with self.subTest(to=to): diff --git a/Lib/test/test_http_cookiejar.py b/Lib/test/test_http_cookiejar.py index 9450104d0b9a75a..37365e47b125829 100644 --- a/Lib/test/test_http_cookiejar.py +++ b/Lib/test/test_http_cookiejar.py @@ -854,10 +854,37 @@ def test_wrong_domain(self): # XXX far from complete c = CookieJar() - interact_2965(c, "http://www.nasty.com/", - 'foo=bar; domain=friendly.org; Version="1"') + interact_2965(c, "http://www.example.com/", + 'foo=bar; domain=python.org; Version="1"') self.assertEqual(len(c), 0) + def test_negative_version(self): + # A negative version is neither Netscape (0) nor RFC 2965 (> 0), so + # such cookies must be neither set nor returned. + for rfc2965 in False, True: + for interact in interact_netscape, interact_2965: + with self.subTest(rfc2965=rfc2965, interact=interact.__name__): + c = CookieJar(DefaultCookiePolicy(rfc2965=rfc2965)) + interact(c, "http://www.example.com/", + 'foo=bar; domain=python.org; Version=-1') + self.assertEqual(len(c), 0) + interact(c, "http://www.example.com/", 'foo=bar; Version=-1') + self.assertEqual(len(c), 0) + + # Such a header is discarded before the policy is consulted. + c = CookieJar() + req = urllib.request.Request("http://www.example.com/") + res = FakeResponse(["Set-Cookie: foo=bar; Version=-1"], + "http://www.example.com/") + self.assertEqual(c.make_cookies(res, req), []) + + # Not set from a response, for example loaded from an LWP cookie file. + c = CookieJar() + c.set_cookie(Cookie(-1, "foo", "bar", None, False, + ".python.org", True, True, "/", True, + False, None, True, None, None, {})) + self.assertEqual(interact_netscape(c, "http://www.python.org/"), "") + def test_strict_domain(self): # Cookies whose domain is a country-code tld like .co.uk should # not be set if CookiePolicy.strict_domain is true. @@ -1315,6 +1342,7 @@ def cookiejar_from_cookie_headers(headers): ["Set-Cookie: b=foo; max-age=oops"], # bad version ["Set-Cookie: b=foo; version=spam"], + ["Set-Cookie: b=foo; version=-1"], ["Set-Cookie:; Expires=%s" % future], ]: c = cookiejar_from_cookie_headers(headers) diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 19670c5f756ab57..4ed8c7b40a4acc0 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -1034,6 +1034,49 @@ def test_negative_content_length(self): self.assertEqual(resp.read(), b'Hello\r\n') self.assertTrue(resp.isclosed()) + def test_malformed_content_length(self): + # RFC 9112: Content-Length = 1*DIGIT. Values that int() accepts but + # the grammar forbids must not be used to frame the body. + for value in ('+5', '5_0', '5 0'): + with self.subTest(value=value): + sock = FakeSocket( + 'HTTP/1.1 200 OK\r\nContent-Length: %s\r\n\r\nHello\r\n' % value) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertIsNone(resp.length) + self.assertEqual(resp.read(), b'Hello\r\n') + resp.close() + + def test_content_length_with_whitespace(self): + # RFC 9112, section 5.1: optional whitespace (SP / HTAB) around + # the field value is not part of the value, so it must not + # prevent the Content-Length from being used to frame the body. + for value in ('7 ', '7\t', '7 \t ', ' 7 '): + with self.subTest(value=value): + sock = FakeSocket( + 'HTTP/1.1 200 OK\r\nContent-Length: %s\r\n\r\n' + 'Hello\r\nextra' % value) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.length, 7) + self.assertEqual(resp.read(), b'Hello\r\n') + self.assertTrue(resp.isclosed()) + resp.close() + + def test_malformed_chunk_size(self): + # RFC 9112: chunk-size = 1*HEXDIG. Reject sizes that int(_, 16) accepts + # but the grammar forbids (a sign, an "0x" prefix, underscores or + # leading whitespace). + start = 'HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n' + for size in ('-5', '+5', '0x5', '1_f', ' 5'): + with self.subTest(size=size): + sock = FakeSocket(start + '%s\r\nHELLO\r\n0\r\n\r\n' % size) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertRaises(client.IncompleteRead, resp.read) + self.assertTrue(resp.isclosed()) + resp.close() + def test_incomplete_read(self): sock = FakeSocket('HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nHello\r\n') resp = client.HTTPResponse(sock, method="GET") diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index f8177077439d3c9..e3c50cf85c8d4ad 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -538,6 +538,21 @@ def test_unselect(self): self.assertEqual(data[0], b'Returned to authenticated state. (Success)') self.assertEqual(client.state, 'AUTH') + def test_control_characters(self): + client, server = self._setup(SimpleIMAPHandler) + client.login('user', 'pass') + for c in '\0\r\n': + with self.assertRaises(ValueError): + client.select(f'a{c}b') + # Other control characters are valid in a quoted string and can + # occur in mailbox names returned by the server, so they must + # still be accepted. + for c in support.control_characters_c0(): + if c in '\0\r\n': + continue + typ, _ = client.select(f'a{c}b') + self.assertEqual(typ, 'OK') + class NewIMAPTests(NewIMAPTestsMixin, unittest.TestCase): imap_class = imaplib.IMAP4 diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index f31ea4664f4d373..219d59c8d1fdadb 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -51,6 +51,10 @@ def test_default_data(self): eq(self.db.guess_type("foo.tar.bz2"), ("application/x-tar", "bzip2")) eq(self.db.guess_type("foo.tar.xz"), ("application/x-tar", "xz")) + @unittest.skipIf(sys.platform == "win32", "non-Windows only") + def test_default_knownfiles(self): + self.assertIn("/etc/mime.types", knownfiles) + def test_data_urls(self): eq = self.assertEqual guess_type = self.db.guess_type @@ -256,6 +260,9 @@ def test_registry_parsing(self): eq(self.db.guess_type("image.jpg"), ("image/jpeg", None)) eq(self.db.guess_type("image.png"), ("image/png", None)) + def test_default_knownfiles(self): + self.assertEqual(knownfiles, []) + @unittest.skipIf(not hasattr(_winapi, "_mimetypes_read_windows_registry"), "read_windows_registry accelerator unavailable") def test_registry_accelerator(self): diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index 1220ca32ef82e8f..b66d1ec2c5464c3 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -15,6 +15,7 @@ from test.support import hashlib_helper from test.support import socket_helper from test.support import threading_helper +from test.support import control_characters_c0 import warnings with warnings.catch_warnings(): @@ -365,6 +366,13 @@ def test_quit(self): self.assertIsNone(self.client.sock) self.assertIsNone(self.client.file) + def test_control_characters(self): + for c0 in control_characters_c0(): + with self.assertRaises(ValueError): + self.client.user(f'user{c0}') + with self.assertRaises(ValueError): + self.client.pass_(f'{c0}pass') + @requires_ssl def test_stls_capa(self): capa = self.client.capa() diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index e9af4c4b084acce..30aa990557ffa72 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -14,8 +14,10 @@ import socket import select import struct +import tempfile import time import datetime +import zipfile import gc import http.client import os @@ -132,6 +134,19 @@ def data_file(*name): SIGNED_CERTFILE_ECC = data_file("keycertecc.pem") SIGNED_CERTFILE_ECC_HOSTNAME = 'localhost-ecc' +# gh-160040: a leaf signed by a compromised eMudhra intermediate whose public +# key is on the _ssl distrusted list. The chain to distrusted_root.pem is +# otherwise valid. +# The PEMs live in a password-protected zip so the compromised eMudhra certs +# don't trip security scanners that cannot tell internal test fixtures from +# production secrets and flag any certificate or private key they find. It is +# encrypted, not merely zipped, because such scanners read inside plain archives. +# The password is fixed, public, and not a secret; it only keeps the archive +# from storing the PEMs in the clear. +DISTRUSTED_SPKI_ZIP = data_file("distrusted.zip") +DISTRUSTED_SPKI_CHOSEN_KNIGHT = b"Brave Sir Robin" # not a secret, see comment +DISTRUSTED_SPKI_HOSTNAME = "localhost" + # Same certificate as pycacert.pem, but without extra text in file SIGNING_CA = data_file("capath", "ceff1710.0") # cert with all kinds of subject alt names @@ -2103,6 +2118,28 @@ def test_unwrap(self): c_in.write(s_out.read()) client.unwrap() + def test_distrusted_spki(self): + # gh-160040: the server chain is otherwise valid, but the eMudhra + # intermediate's public key is on the _ssl distrusted list, so + # verification rejects it. + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + # the contexts load the PEMs now, so the files are only needed here + with tempfile.TemporaryDirectory() as tmpdir: + with zipfile.ZipFile(DISTRUSTED_SPKI_ZIP) as z: + z.extractall(tmpdir, pwd=DISTRUSTED_SPKI_CHOSEN_KNIGHT) + certdir = os.path.join(tmpdir, "distrusted") + client_context.load_verify_locations( + os.path.join(certdir, "distrusted_root.pem")) + server_context.load_cert_chain( + os.path.join(certdir, "distrusted_keycert.pem")) + + with self.assertRaises(ssl.SSLCertVerificationError) as cm: + connected_bio_pair(client_context, server_context, + DISTRUSTED_SPKI_HOSTNAME) + # X509_V_ERR_CERT_REJECTED + self.assertEqual(cm.exception.verify_code, 28) + def test_sni_callback_context_released_and_callback_raises(self): # Variant of the test below without a HelloRetryRequest: the callback # switches the connection to another context, drops the last diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index dd95feb55d9272b..4779f9526365a69 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -3881,8 +3881,10 @@ def test_link_at_destination(self): t.linkname = '.' arc.tar_w.addfile(t) - with self.check_context(arc.open(), 'data'): - self.expect_exception(tarfile.OutsideDestinationError) + for filter in 'tar', 'data': + with self.subTest(name=name, filter=filter): + with self.check_context(arc.open(), filter): + self.expect_exception(tarfile.OutsideDestinationError) @os_helper.skip_unless_symlink def test_empty_name_symlink_chain(self): diff --git a/Misc/NEWS.d/next/Library/2026-06-02-13-10-00.gh-issue-150751.Hk2Vw9.rst b/Misc/NEWS.d/next/Library/2026-06-02-13-10-00.gh-issue-150751.Hk2Vw9.rst new file mode 100644 index 000000000000000..670d677aff79ff2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-02-13-10-00.gh-issue-150751.Hk2Vw9.rst @@ -0,0 +1,5 @@ +:mod:`http.client` now validates the ``Content-Length`` header and the +chunked ``chunk-size`` against the RFC 9112 grammar (``1*DIGIT`` and +``1*HEXDIG``) before parsing them, rejecting values such as ``+5``, ``5_0`` +or a ``0x``-prefixed or negative chunk size that :func:`int` would otherwise +accept. This avoids framing a response differently from a strict peer. diff --git a/Misc/NEWS.d/next/Library/2026-10-08-16-18-43.gh-issue-87720.Elqdl9.rst b/Misc/NEWS.d/next/Library/2026-10-08-16-18-43.gh-issue-87720.Elqdl9.rst new file mode 100644 index 000000000000000..a2df7654b31444b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-08-16-18-43.gh-issue-87720.Elqdl9.rst @@ -0,0 +1,3 @@ +Fixed error in :mod:`email` new policies where header folding failed to +detect the need to re-quote text obtained from encoded words in certain +cases. diff --git a/Misc/NEWS.d/next/Library/2026-10-10-22-14-22.gh-issue-150751.JInuGL.rst b/Misc/NEWS.d/next/Library/2026-10-10-22-14-22.gh-issue-150751.JInuGL.rst new file mode 100644 index 000000000000000..40c8d606aa67734 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-10-22-14-22.gh-issue-150751.JInuGL.rst @@ -0,0 +1,5 @@ +:mod:`http.client` now ignores optional whitespace around the +``Content-Length`` header value when validating it, as required by +:rfc:`9112`. Previously a value with trailing whitespace (for example +``Content-Length: 5`` followed by a space) was treated as missing and the +response body was read until the connection was closed. diff --git a/Misc/NEWS.d/next/Security/2026-01-16-11-41-06.gh-issue-143921.AeCOor.rst b/Misc/NEWS.d/next/Security/2026-01-16-11-41-06.gh-issue-143921.AeCOor.rst new file mode 100644 index 000000000000000..de62e020a95e6b2 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-01-16-11-41-06.gh-issue-143921.AeCOor.rst @@ -0,0 +1,2 @@ +Reject NUL, CR and LF characters in IMAP commands. Other control +characters are allowed and sent quoted. diff --git a/Misc/NEWS.d/next/Security/2026-01-16-11-43-47.gh-issue-143923.DuytMe.rst b/Misc/NEWS.d/next/Security/2026-01-16-11-43-47.gh-issue-143923.DuytMe.rst new file mode 100644 index 000000000000000..3cde4df3e0069f7 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-01-16-11-43-47.gh-issue-143923.DuytMe.rst @@ -0,0 +1 @@ +Reject control characters in POP3 commands. diff --git a/Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-157953.KxCFSN.rst b/Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-157953.KxCFSN.rst new file mode 100644 index 000000000000000..3220f4613a91af6 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-157953.KxCFSN.rst @@ -0,0 +1 @@ +Update bundled `libexpat `_ to version 2.9.0. diff --git a/Misc/NEWS.d/next/Security/2026-10-15-03-47-08.gh-issue-159428.XHRKwY.rst b/Misc/NEWS.d/next/Security/2026-10-15-03-47-08.gh-issue-159428.XHRKwY.rst new file mode 100644 index 000000000000000..d481ed2b74a20ea --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-10-15-03-47-08.gh-issue-159428.XHRKwY.rst @@ -0,0 +1,2 @@ +:mod:`mimetypes` no longer reads the files listed in :data:`mimetypes.knownfiles` +by default on Windows, and uses only the registry. diff --git a/Misc/NEWS.d/next/Security/2026-10-21-16-55-07.gh-issue-160040.9PjFjC.rst b/Misc/NEWS.d/next/Security/2026-10-21-16-55-07.gh-issue-160040.9PjFjC.rst new file mode 100644 index 000000000000000..7bcec7f5808cac6 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-10-21-16-55-07.gh-issue-160040.9PjFjC.rst @@ -0,0 +1,12 @@ +.. cve: CVE-2026-115557 + +:mod:`ssl` now rejects TLS certificate chains that contain any of a set of +compromised eMudhra intermediate CA keys. This is an extraordinary measure and +a last line of defense: removing trust is primarily the job of the platform root +store, and the built-in denylist only blocks connections while that update has +not yet reached the OS, a container image, or a bundled trust store in various +Python packages. Entries are added to the denylist only under a very high +threshold. +The denylist matches on the public key (the SHA-256 digest of the :rfc:`5280` +``SubjectPublicKeyInfo``), so it also covers the cross-signed certificates that +reuse one of the distrusted keys. diff --git a/Misc/NEWS.d/next/Security/2026-10-21-19-58-53.gh-issue-160097.-opfAL.rst b/Misc/NEWS.d/next/Security/2026-10-21-19-58-53.gh-issue-160097.-opfAL.rst new file mode 100644 index 000000000000000..b48c8f7213ee826 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-10-21-19-58-53.gh-issue-160097.-opfAL.rst @@ -0,0 +1,5 @@ +.. cve: CVE-2026-114583 + +:func:`tarfile.tar_filter` now rejects link members whose path resolves to +the destination directory itself, as :func:`tarfile.data_filter` already +did. diff --git a/Misc/NEWS.d/next/Security/2026-10-22-08-04-33.gh-issue-160300.q7Zk2B.rst b/Misc/NEWS.d/next/Security/2026-10-22-08-04-33.gh-issue-160300.q7Zk2B.rst new file mode 100644 index 000000000000000..3cadc6c2c4fd1fb --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-10-22-08-04-33.gh-issue-160300.q7Zk2B.rst @@ -0,0 +1,3 @@ +Fix :class:`http.cookiejar.DefaultCookiePolicy` accepting and returning +cookies with a negative ``Version`` attribute without checking their domain +against the request host. Such cookies are now ignored. diff --git a/Misc/NEWS.d/next/Security/2026-10-27-12-00-00.gh-issue-159894.AsPip1.rst b/Misc/NEWS.d/next/Security/2026-10-27-12-00-00.gh-issue-159894.AsPip1.rst new file mode 100644 index 000000000000000..a5e9d3fca6a55a2 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-10-27-12-00-00.gh-issue-159894.AsPip1.rst @@ -0,0 +1,3 @@ +Upgrade the bundled pip in :mod:`ensurepip` to 26.2.2, which no longer trusts +the compromised eMudhra emSign CA (the pip 23.0.1 it replaces ships a +vendored certifi that still trusts it). diff --git a/Misc/NEWS.d/next/Security/2026-10-27-21-00-00.gh-issue-135374.AsSet1.rst b/Misc/NEWS.d/next/Security/2026-10-27-21-00-00.gh-issue-135374.AsSet1.rst new file mode 100644 index 000000000000000..6835f2067f62b4d --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-10-27-21-00-00.gh-issue-135374.AsSet1.rst @@ -0,0 +1,4 @@ +Upgrade the bundled setuptools in :mod:`ensurepip` to 84.2.1, which fixes +CVE-2026-59890 and GHSA-xmj3-9gf7-h52w. setuptools 82.0.0 removed +``pkg_resources``, so it is no longer available in environments seeded by +:mod:`ensurepip` or :mod:`venv`. diff --git a/Modules/Setup b/Modules/Setup index 87c6a152f86eac0..fb4a844415bfb76 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -348,7 +348,7 @@ _symtable symtablemodule.c # Interface to the Expat XML parser # More information on Expat can be found at www.libexpat.org. # -#pyexpat expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DXML_POOR_ENTROPY -DUSE_PYEXPAT_CAPI +#pyexpat expat/xcs.c expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DXML_POOR_ENTROPY -DUSE_PYEXPAT_CAPI # Hye-Shik Chang's CJKCodecs diff --git a/Modules/_ssl.c b/Modules/_ssl.c index d58bb1565ccf1f8..2b409bbd1977885 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -54,6 +54,7 @@ /* Include OpenSSL header files */ #include "openssl/rsa.h" #include "openssl/crypto.h" +#include "openssl/evp.h" #include "openssl/x509.h" #include "openssl/x509v3.h" #include "openssl/pem.h" @@ -880,6 +881,7 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, int (*verify_cb)(int, X509_STORE_CTX *) = NULL; verify_cb = SSL_get_verify_callback(self->ssl); mode |= SSL_VERIFY_POST_HANDSHAKE; + /* Keep ctx verify callback (distrusted SPKI check) */ SSL_set_verify(self->ssl, mode, verify_cb); } } else { @@ -2985,11 +2987,70 @@ static PyType_Spec PySSLSocket_spec = { * _SSLContext objects */ +#include "_ssl/distrusted_spki.h" + +/* Returns 1 if the SHA-256 of the cert's SPKI is on the distrusted list, + 0 if not (okay), -1 on an internal error. */ +static int +_cert_has_distrusted_spki(X509 *cert) +{ + unsigned char *spki = NULL; + int spki_len = i2d_X509_PUBKEY(X509_get_X509_PUBKEY(cert), &spki); + if (spki_len <= 0 || spki == NULL) { + return -1; + } + + unsigned char digest[EVP_MAX_MD_SIZE]; + unsigned int digest_len = 0; + int ok = EVP_Digest(spki, (size_t)spki_len, digest, &digest_len, + EVP_sha256(), NULL); + OPENSSL_free(spki); + assert(sizeof(_distrusted_spki[0]) == SHA256_DIGEST_LENGTH); + if (!ok + || digest_len != SHA256_DIGEST_LENGTH + || sizeof(_distrusted_spki[0]) != SHA256_DIGEST_LENGTH) { + return -1; + } + + for (size_t i = 0; i < Py_ARRAY_LENGTH(_distrusted_spki); i++) { + if (memcmp(digest, _distrusted_spki[i], SHA256_DIGEST_LENGTH) == 0) { + return 1; + } + } + return 0; +} + +/* Verify callback, called once per cert. Reject distrusted SPKIs. */ +static int +_ssl_verify_callback(int preverify_ok, X509_STORE_CTX *x509_ctx) +{ + /* Don't override an error OpenSSL already reported for this certificate. */ + if (!preverify_ok) { + return preverify_ok; + } + X509 *cert = X509_STORE_CTX_get_current_cert(x509_ctx); + if (cert == NULL) { + return preverify_ok; + } + switch (_cert_has_distrusted_spki(cert)) { + case 0: + /* okay: not on the distrusted list */ + return preverify_ok; + case 1: + /* rejected: public key is on the distrusted list */ + X509_STORE_CTX_set_error(x509_ctx, X509_V_ERR_CERT_REJECTED); + return 0; + default: + /* internal error */ + X509_STORE_CTX_set_error(x509_ctx, X509_V_ERR_UNSPECIFIED); + return 0; + } +} + static int _set_verify_mode(PySSLContext *self, enum py_ssl_cert_requirements n) { int mode; - int (*verify_cb)(int, X509_STORE_CTX *) = NULL; switch(n) { case PY_SSL_CERT_NONE: @@ -3010,9 +3071,7 @@ _set_verify_mode(PySSLContext *self, enum py_ssl_cert_requirements n) /* bpo-37428: newPySSLSocket() sets SSL_VERIFY_POST_HANDSHAKE flag for * server sockets and SSL_set_post_handshake_auth() for client. */ - /* keep current verify cb */ - verify_cb = SSL_CTX_get_verify_callback(self->ctx); - SSL_CTX_set_verify(self->ctx, mode, verify_cb); + SSL_CTX_set_verify(self->ctx, mode, _ssl_verify_callback); return 0; } diff --git a/Modules/_ssl/distrusted_spki.h b/Modules/_ssl/distrusted_spki.h new file mode 100644 index 000000000000000..bbdb56dcfe671ed --- /dev/null +++ b/Modules/_ssl/distrusted_spki.h @@ -0,0 +1,75 @@ +/* Denylist of distrusted SubjectPublicKeyInfo (SPKI). + * + * Verification rejects any chain with a cert whose SPKI SHA-256 digest is in + * the _distrusted_spki table. We match the key, not a cert fingerprint, so + * cross-signs that reuse the key are blocked too. + * + * This is an extraordinary measure and last line of defense. Removing trust is + * primarily the job of the platform root store. This list only blocks keys while + * that update is still propagating to the OS, container images, or root CAs + * bundled in Python packages. The threshold for adding an entry is very high. + * + * See https://github.com/python/cpython/issues/160040 for the eMudhra discussion. + * + * To block another key, add the SHA-256 digest bytes of its SPKI: + * + * $ openssl x509 -in cert.pem -noout -pubkey | openssl pkey -pubin -outform DER | openssl dgst -sha256 -binary | xxd -i + */ +#ifndef Py_SSL_DISTRUSTED_SPKI_H +#define Py_SSL_DISTRUSTED_SPKI_H + +#include "openssl/sha.h" + +static const unsigned char _distrusted_spki[][SHA256_DIGEST_LENGTH] = { + /* Compromised eMudhra intermediates, https://emudhra.we-have-a-problem.com/ */ + /* CN=EM OV TLS CA - G3B,O=eMudhra BV,C=NL */ + { + 0x43, 0xbf, 0xfa, 0x19, 0xd6, 0xef, 0x65, 0x14, 0x51, 0xf6, 0x84, 0x1e, + 0x01, 0x99, 0x31, 0xe9, 0x6f, 0x6f, 0x2e, 0xae, 0xff, 0x28, 0xab, 0x4b, + 0x6f, 0x1c, 0xe4, 0x57, 0x6d, 0xe2, 0x71, 0x7b, + }, + /* CN=EM DV TLS CA - G3C,O=eMudhra Inc,C=US */ + { + 0x49, 0xdd, 0x35, 0x3d, 0xc2, 0xb9, 0x19, 0xa3, 0x77, 0x06, 0x45, 0x11, + 0x7d, 0x0e, 0x60, 0xc1, 0x61, 0x15, 0xe7, 0xd1, 0xbc, 0x25, 0x9e, 0xef, + 0xe7, 0x34, 0xb0, 0x0e, 0xa4, 0x34, 0xfe, 0x5f, + }, + /* CN=EM EV TLS CA - G3C,O=eMudhra Inc,C=US */ + { + 0x5c, 0x06, 0x60, 0xa2, 0xb1, 0x64, 0x91, 0x0d, 0x8f, 0xf5, 0x89, 0xc1, + 0x8e, 0x14, 0x55, 0x2e, 0x9e, 0x0e, 0xc2, 0x37, 0xdc, 0xef, 0xe2, 0xcb, + 0x73, 0x6d, 0x41, 0xa4, 0x81, 0x1f, 0x81, 0xa0, + }, + /* CN=EM OV TLS CA - G3C,O=eMudhra Inc,C=US */ + { + 0x63, 0xec, 0xef, 0xee, 0x43, 0x1d, 0x49, 0x52, 0x3b, 0xc1, 0x13, 0x6c, + 0x4e, 0x72, 0xd0, 0x20, 0x6a, 0x47, 0x70, 0x86, 0xca, 0xf6, 0x06, 0xcd, + 0x5c, 0xf0, 0x25, 0x68, 0x6c, 0x10, 0x9c, 0xf8, + }, + /* CN=EM OV CS CA - G3C,O=eMudhra Inc,C=US */ + { + 0x92, 0xc0, 0x63, 0xd6, 0x7e, 0x8f, 0x14, 0xdb, 0x71, 0xd2, 0x88, 0xe1, + 0x12, 0x66, 0xff, 0xda, 0xf6, 0x9c, 0xbf, 0x11, 0x5f, 0x7b, 0xab, 0x14, + 0xf3, 0xbc, 0x2f, 0x68, 0xd2, 0x5e, 0x0e, 0x30, + }, + /* CN=EM EV TLS CA - G3B,O=eMudhra BV,C=NL */ + { + 0xe0, 0x25, 0x4c, 0x7f, 0xcd, 0xc2, 0x65, 0xe2, 0xf0, 0xf8, 0xa4, 0x28, + 0xda, 0xbb, 0x75, 0xab, 0x4a, 0xe0, 0x39, 0xb4, 0x87, 0x2d, 0x62, 0xf5, + 0xee, 0x57, 0xf0, 0x87, 0x7d, 0x5a, 0x8a, 0x41, + }, + /* CN=EM DV TLS CA - G3B,O=eMudhra BV,C=NL */ + { + 0xeb, 0xed, 0x64, 0x75, 0xc2, 0xc2, 0xd8, 0x0f, 0xcb, 0x19, 0x9f, 0x3f, + 0xe8, 0xdd, 0x6e, 0x6b, 0x3c, 0xe6, 0x0b, 0x77, 0xeb, 0x15, 0xd4, 0xe4, + 0x2c, 0x3d, 0xec, 0x0c, 0x6f, 0x5b, 0x51, 0x05, + }, + /* CN=EM OV CS CA - G3B,O=eMudhra BV,C=NL */ + { + 0xec, 0x4f, 0xfa, 0x1a, 0xd0, 0xac, 0x88, 0x08, 0x13, 0x14, 0xb7, 0xaa, + 0xff, 0xef, 0x6a, 0x91, 0xf9, 0x7c, 0xd5, 0x72, 0x91, 0xbb, 0x5b, 0xa6, + 0x86, 0xc5, 0x41, 0x5f, 0x5e, 0xb4, 0xf6, 0xb6, + }, +}; + +#endif /* !Py_SSL_DISTRUSTED_SPKI_H */ diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 4c3851d50a5fdf3..1b2852dd76cb487 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -48,7 +48,7 @@ #ifndef Expat_INCLUDED # define Expat_INCLUDED 1 -# include // for uint8_t +# include // for int64_t, uint8_t, uint64_t # include # include "expat_external.h" @@ -140,6 +140,26 @@ enum XML_Error { XML_ERROR_NOT_STARTED, }; +/* Added in 2.9.0. */ +enum XML_Prop_Error { + XML_PROP_ERROR_NONE = 0, // i.e. success + + XML_PROP_ERROR_INVALID_KEY = 1, // i.e. the property is not known at all + XML_PROP_ERROR_INVALID_TYPE = 2, // i.e. the property is known but expects a + // different type (so a different + // getter/setter would need to be called) + XML_PROP_ERROR_INVALID_VALUE = 3, // i.e. the value is invalid (with a setter + // function) or a NULL-pointer (with a + // getter function) + + XML_PROP_ERROR_PARSER_NULL = 4, // i.e. the parser argument is NULL + XML_PROP_ERROR_PARSER_NOT_ROOT = 5, // i.e. the parser is not a root parser + // but a subparser (and a root parser is + // needed) + + /* potentially more error codes upcoming here */ +}; + enum XML_Content_Type { XML_CTYPE_EMPTY = 1, XML_CTYPE_ANY, @@ -757,29 +777,6 @@ XML_GetSpecifiedAttributeCount(XML_Parser parser); XMLPARSEAPI(int) XML_GetIdAttributeIndex(XML_Parser parser); -# ifdef XML_ATTR_INFO -/* Source file byte offsets for the start and end of attribute names and values. - The value indices are exclusive of surrounding quotes; thus in a UTF-8 source - file an attribute value of "blah" will yield: - info->valueEnd - info->valueStart = 4 bytes. -*/ -typedef struct { - XML_Index nameStart; /* Offset to beginning of the attribute name. */ - XML_Index nameEnd; /* Offset after the attribute name's last byte. */ - XML_Index valueStart; /* Offset to beginning of the attribute value. */ - XML_Index valueEnd; /* Offset after the attribute value's last byte. */ -} XML_AttrInfo; - -/* Returns an array of XML_AttrInfo structures for the attribute/value pairs - passed in last call to the XML_StartElementHandler that were specified - in the start-tag rather than defaulted. Each attribute/value pair counts - as 1; thus the number of entries in the array is - XML_GetSpecifiedAttributeCount(parser) / 2. -*/ -XMLPARSEAPI(const XML_AttrInfo *) -XML_GetAttributeInfo(XML_Parser parser); -# endif - /* Parses some input. Returns XML_STATUS_ERROR if a fatal error is detected. The last call to XML_Parse must have isFinal true; len may be zero for this call (or any other). @@ -964,15 +961,24 @@ XML_GetErrorCode(XML_Parser parser); return 0 to indicate an error. Note: XML_GetCurrentByteIndex returns -1 to indicate an error. */ +XML_ATTR_DEPRECATED("please use XML_GetCurrentLineNumber64 instead") XMLPARSEAPI(XML_Size) XML_GetCurrentLineNumber(XML_Parser parser); +XMLPARSEAPI(uint64_t) XML_GetCurrentLineNumber64(XML_Parser parser); +XML_ATTR_DEPRECATED("please use XML_GetCurrentColumnNumber64 instead") XMLPARSEAPI(XML_Size) XML_GetCurrentColumnNumber(XML_Parser parser); +XMLPARSEAPI(uint64_t) XML_GetCurrentColumnNumber64(XML_Parser parser); +XML_ATTR_DEPRECATED("please use XML_GetCurrentByteIndex64 instead") XMLPARSEAPI(XML_Index) XML_GetCurrentByteIndex(XML_Parser parser); +XMLPARSEAPI(int64_t) XML_GetCurrentByteIndex64(XML_Parser parser); /* Return the number of bytes in the current event. Returns 0 if the event is in an internal entity. */ +XML_ATTR_DEPRECATED("please use XML_GetCurrentByteCount64 instead") XMLPARSEAPI(int) XML_GetCurrentByteCount(XML_Parser parser); +XMLPARSEAPI(uint64_t) +XML_GetCurrentByteCount64(XML_Parser parser); /* If XML_CONTEXT_BYTES is >=1, returns the input buffer, sets the integer pointed to by offset to the offset within this buffer @@ -984,8 +990,11 @@ XML_GetCurrentByteCount(XML_Parser parser); NOTE: The character pointer returned should not be used outside the handler that makes the call. */ +XML_ATTR_DEPRECATED("please use XML_GetInputContext64 instead") XMLPARSEAPI(const char *) XML_GetInputContext(XML_Parser parser, int *offset, int *size); +XMLPARSEAPI(const char *) +XML_GetInputContext64(XML_Parser parser, int64_t *offset, uint64_t *size); /* For backwards compatibility with previous versions. */ # define XML_GetErrorLineNumber XML_GetCurrentLineNumber @@ -1040,6 +1049,7 @@ enum XML_FeatureEnum { XML_FEATURE_UNICODE_WCHAR_T, XML_FEATURE_DTD, XML_FEATURE_CONTEXT_BYTES, + /* Added in Expat 1.95.5, removed in Expat 2.9.0. */ XML_FEATURE_MIN_SIZE, XML_FEATURE_SIZEOF_XML_CHAR, XML_FEATURE_SIZEOF_XML_LCHAR, @@ -1047,7 +1057,7 @@ enum XML_FeatureEnum { XML_FEATURE_NS, /* Added in Expat 2.0.1. */ XML_FEATURE_LARGE_SIZE, - /* Added in Expat 2.1.0. */ + /* Added in Expat 2.1.0, removed in Expat 2.9.0. */ XML_FEATURE_ATTR_INFO, /* Added in Expat 2.4.0. */ XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT, @@ -1069,6 +1079,41 @@ typedef struct { XMLPARSEAPI(const XML_Feature *) XML_GetFeatureList(void); +/* Added in Expat 2.9.0. */ +enum XML_Parser_Property { +# if defined(XML_TESTING) + XML_PROP_INVALID = 0, +# endif +# if XML_GE == 1 + XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD = 1, // of type `uint64_t` + XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION = 2, // of type `double` + XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD = 3, // of type `uint64_t` + XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION = 4, // of type `double` +# endif + XML_PROP_REPARSE_DEFERRAL_ENABLED = 5, // of type `XML_Bool` + /* more properties upcoming here */ +}; + +/* Added in Expat 2.9.0. */ +XMLPARSEAPI(enum XML_Prop_Error) +XML_GetPropertyBool(XML_Parser parser, enum XML_Parser_Property property, + XML_Bool *value); +XMLPARSEAPI(enum XML_Prop_Error) +XML_GetPropertyDouble(XML_Parser parser, enum XML_Parser_Property property, + double *value); +XMLPARSEAPI(enum XML_Prop_Error) +XML_GetPropertyUInt64(XML_Parser parser, enum XML_Parser_Property property, + uint64_t *value); +XMLPARSEAPI(enum XML_Prop_Error) +XML_SetPropertyBool(XML_Parser parser, enum XML_Parser_Property property, + XML_Bool value); +XMLPARSEAPI(enum XML_Prop_Error) +XML_SetPropertyDouble(XML_Parser parser, enum XML_Parser_Property property, + double value); +XMLPARSEAPI(enum XML_Prop_Error) +XML_SetPropertyUInt64(XML_Parser parser, enum XML_Parser_Property property, + uint64_t value); + # if defined(XML_DTD) || (defined(XML_GE) && XML_GE == 1) /* Added in Expat 2.4.0 for XML_DTD defined and * added in Expat 2.6.0 for XML_GE == 1. */ @@ -1101,8 +1146,8 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); See https://semver.org */ # define XML_MAJOR_VERSION 2 -# define XML_MINOR_VERSION 8 -# define XML_MICRO_VERSION 5 +# define XML_MINOR_VERSION 9 +# define XML_MICRO_VERSION 0 # ifdef __cplusplus } diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h index a6c8a97881795e5..120a30e7ef3e498 100644 --- a/Modules/expat/expat_external.h +++ b/Modules/expat/expat_external.h @@ -169,6 +169,9 @@ typedef char XML_LChar; # endif /* XML_UNICODE */ # ifdef XML_LARGE_SIZE /* Use large integers for file/stream positions. */ +# if defined(__clang__) || defined(__GNUC__) +# warning Macro XML_LARGE_SIZE is deprecated, please use the 64bit location API functions instead. +# endif typedef long long XML_Index; typedef unsigned long long XML_Size; # else diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h index 86655371d29033e..de34cd7dfc55c58 100644 --- a/Modules/expat/internal.h +++ b/Modules/expat/internal.h @@ -56,13 +56,11 @@ #if ! defined(INTERNAL_H) # define INTERNAL_H 1 -# ifndef XML_MIN_SIZE -# if ! defined(inline) -# ifdef __GNUC__ -# define inline __inline -# endif /* __GNUC__ */ -# endif -# endif /* XML_MIN_SIZE */ +# if ! defined(inline) +# ifdef __GNUC__ +# define inline __inline +# endif /* __GNUC__ */ +# endif # ifndef inline # define inline diff --git a/Modules/expat/pyexpatns.h b/Modules/expat/pyexpatns.h index fc6b482d587e0d6..4c550b7d0131819 100644 --- a/Modules/expat/pyexpatns.h +++ b/Modules/expat/pyexpatns.h @@ -50,14 +50,22 @@ #define XML_GetBase PyExpat_XML_GetBase #define XML_GetBuffer PyExpat_XML_GetBuffer #define XML_GetCurrentByteCount PyExpat_XML_GetCurrentByteCount +#define XML_GetCurrentByteCount64 PyExpat_XML_GetCurrentByteCount64 #define XML_GetCurrentByteIndex PyExpat_XML_GetCurrentByteIndex +#define XML_GetCurrentByteIndex64 PyExpat_XML_GetCurrentByteIndex64 #define XML_GetCurrentColumnNumber PyExpat_XML_GetCurrentColumnNumber +#define XML_GetCurrentColumnNumber64 PyExpat_XML_GetCurrentColumnNumber64 #define XML_GetCurrentLineNumber PyExpat_XML_GetCurrentLineNumber +#define XML_GetCurrentLineNumber64 PyExpat_XML_GetCurrentLineNumber64 #define XML_GetErrorCode PyExpat_XML_GetErrorCode #define XML_GetFeatureList PyExpat_XML_GetFeatureList #define XML_GetIdAttributeIndex PyExpat_XML_GetIdAttributeIndex #define XML_GetInputContext PyExpat_XML_GetInputContext +#define XML_GetInputContext64 PyExpat_XML_GetInputContext64 #define XML_GetParsingStatus PyExpat_XML_GetParsingStatus +#define XML_GetPropertyBool PyExpat_XML_GetPropertyBool +#define XML_GetPropertyDouble PyExpat_XML_GetPropertyDouble +#define XML_GetPropertyUInt64 PyExpat_XML_GetPropertyUInt64 #define XML_GetSpecifiedAttributeCount PyExpat_XML_GetSpecifiedAttributeCount #define XmlGetUtf16InternalEncoding PyExpat_XmlGetUtf16InternalEncoding #define XmlGetUtf16InternalEncodingNS PyExpat_XmlGetUtf16InternalEncodingNS @@ -110,6 +118,9 @@ #define XML_SetNotStandaloneHandler PyExpat_XML_SetNotStandaloneHandler #define XML_SetParamEntityParsing PyExpat_XML_SetParamEntityParsing #define XML_SetProcessingInstructionHandler PyExpat_XML_SetProcessingInstructionHandler +#define XML_SetPropertyBool PyExpat_XML_SetPropertyBool +#define XML_SetPropertyDouble PyExpat_XML_SetPropertyDouble +#define XML_SetPropertyUInt64 PyExpat_XML_SetPropertyUInt64 #define XML_SetReparseDeferralEnabled PyExpat_XML_SetReparseDeferralEnabled #define XML_SetReturnNSTriplet PyExpat_XML_SetReturnNSTriplet #define XML_SetSkippedEntityHandler PyExpat_XML_SetSkippedEntityHandler diff --git a/Modules/expat/refresh.sh b/Modules/expat/refresh.sh index 7203f2a3a8d6650..740065935b860df 100755 --- a/Modules/expat/refresh.sh +++ b/Modules/expat/refresh.sh @@ -12,9 +12,9 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. These values are used for verifying the SBOM, too. -expected_libexpat_tag="R_2_8_5" -expected_libexpat_version="2.8.5" -expected_libexpat_sha256="920dde485e15eda0cce8d2310b41d492c534e5e3d89ad407a0b4176dd2ff88fe" +expected_libexpat_tag="R_2_9_0" +expected_libexpat_version="2.9.0" +expected_libexpat_sha256="16afbb9cefead2aa278105cf27d9f597bde7fbf3dbb85015857ca7ca6a4e89ba" expat_dir="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")" cd ${expat_dir} @@ -43,7 +43,8 @@ lib_files=( siphash.h utf8tab.h winconfig.h - xcsinc.c + xcs.c + xcs.h xmlparse.c xmlrole.c xmlrole.h diff --git a/Modules/expat/xcs.c b/Modules/expat/xcs.c new file mode 100644 index 000000000000000..ae5e042d778fffa --- /dev/null +++ b/Modules/expat/xcs.c @@ -0,0 +1,90 @@ +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2022-2026 Sebastian Pipping + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT +*/ + +#include "xcs.h" + +#if defined(XML_UNICODE) +# if defined(XML_UNICODE_WCHAR_T) +# include // for wcscmp, wcslen, wcsncmp +# endif +#else +# include // for strcmp, strlen, strncmp +#endif + +size_t +xcslen(const XML_Char *s) { +#ifdef XML_UNICODE +# ifdef XML_UNICODE_WCHAR_T + return wcslen(s); +# else + // XML_Char is unsigned short + size_t len = 0; + while (s[len]) { + len++; + } + return len; +# endif +#else + return strlen(s); +#endif +} + +int +xcscmp(const XML_Char *a, const XML_Char *b) { +#if defined(XML_UNICODE) +# if defined(XML_UNICODE_WCHAR_T) + return wcscmp(a, b); +# else + for (; a[0] && b[0] && a[0] == b[0]; a++, b++) + ; + return a[0] - b[0]; +# endif +#else + return strcmp(a, b); +#endif +} + +int +xcsncmp(const XML_Char *a, const XML_Char *b, size_t len) { +#if defined(XML_UNICODE) +# if defined(XML_UNICODE_WCHAR_T) + return wcsncmp(a, b, len); +# else + for (; len > 0 && a[0] && b[0] && a[0] == b[0]; len--, a++, b++) { + } + return (len == 0) ? 0 : (a[0] - b[0]); +# endif +#else + return strncmp(a, b, len); +#endif +} diff --git a/Modules/expat/xcsinc.c b/Modules/expat/xcs.h similarity index 76% rename from Modules/expat/xcsinc.c rename to Modules/expat/xcs.h index 675b2844c708d6a..1b942647d333093 100644 --- a/Modules/expat/xcsinc.c +++ b/Modules/expat/xcs.h @@ -1,4 +1,4 @@ -/* This file is included from other .c files! +/* __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -6,7 +6,7 @@ \___/_/\_\ .__/ \__,_|\__| |_| XML parser - Copyright (c) 2022-2026 Sebastian Pipping + Copyright (c) 2026 Sebastian Pipping Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -31,24 +31,14 @@ SPDX-License-Identifier: MIT */ -#if defined(XML_UNICODE) && defined(XML_UNICODE_WCHAR_T) -# include -#endif - -static size_t -xcslen(const XML_Char *s) { -#ifdef XML_UNICODE -# ifdef XML_UNICODE_WCHAR_T - return wcslen(s); -# else - // XML_Char is unsigned short - size_t len = 0; - while (s[len]) { - len++; - } - return len; -# endif -#else - return strlen(s); -#endif -} +#if ! defined(EXPAT_XCS_H) + +# include // size_t +# include "expat_external.h" // for XML_Char + +extern size_t xcslen(const XML_Char *s); + +extern int xcscmp(const XML_Char *a, const XML_Char *b); +extern int xcsncmp(const XML_Char *a, const XML_Char *b, size_t len); + +#endif // ! defined(EXPAT_XCS_H) diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index 9fb4f9d480a2e3a..eefc74599cbe839 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* 0864fe2d216f47b742263b698bc051c865b342e8b820e42c234717098ea507e3 (2.8.5+) +/* e3ca845466942140c9af531ac57808c4650116df5d6dbeda3b6546d2548a586e (2.9.0+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -56,6 +56,8 @@ Copyright (c) 2026 Zeyou Liu Copyright (c) 2026 Stan Ulbrych Copyright (c) 2026 Braian Plaku + Copyright (c) 2026 Filippo Tedeschi + Copyright (c) 2026 Junki Lee Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -107,6 +109,7 @@ #include /* SIZE_MAX, UINT64_MAX, uint64_t, uintptr_t */ #include /* isnan */ #include +#include /* wcsncmp */ #ifdef _WIN32 # define getpid GetCurrentProcessId @@ -125,7 +128,7 @@ #include "ascii.h" #include "expat.h" #include "siphash.h" -#include "xcsinc.c" +#include "xcs.h" #if defined(HAVE_ARC4RANDOM) # include "random_arc4random.h" @@ -462,8 +465,8 @@ typedef struct accounting { } ACCOUNTING; typedef struct MALLOC_TRACKER { - XmlBigCount bytesAllocated; - XmlBigCount peakBytesAllocated; // updated live only for debug level >=2 + size_t bytesAllocated; + size_t peakBytesAllocated; // updated live only for debug level >=2 unsigned long debugLevel; float maximumAmplificationFactor; // >=1.0 XmlBigCount activationThresholdBytes; @@ -646,6 +649,25 @@ static bool poolAppendChar(STRING_POOL *pool, XML_Char c); static bool poolAppendChars(STRING_POOL *pool, const XML_Char *s, size_t len); +#if XML_GE == 1 +static enum XML_Prop_Error setBillionLaughsAttackProtectionMaximumAmplification( + XML_Parser parser, float maximumAmplificationFactor); + +static enum XML_Prop_Error setBillionLaughsAttackProtectionActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes); + +static enum XML_Prop_Error +setAllocTrackerMaximumAmplification(XML_Parser parser, + float maximumAmplificationFactor); + +static enum XML_Prop_Error +setAllocTrackerActivationThreshold(XML_Parser parser, + unsigned long long activationThresholdBytes); +#endif /* XML_GE == 1 */ + +static enum XML_Prop_Error setReparseDeferralEnabled(XML_Parser parser, + XML_Bool enabled); + #define poolStart(pool) ((pool)->start) #define poolLength(pool) ((pool)->ptr - (pool)->start) #define poolChop(pool) ((void)--(pool->ptr)) @@ -785,9 +807,6 @@ struct XML_ParserStruct { NS_ATT *m_nsAtts; unsigned long m_nsAttsVersion; unsigned char m_nsAttsPower; -#ifdef XML_ATTR_INFO - XML_AttrInfo *m_attInfo; -#endif POSITION m_position; STRING_POOL m_tempPool; STRING_POOL m_temp2Pool; @@ -824,30 +843,30 @@ struct XML_ParserStruct { #if XML_GE == 1 static void -expat_heap_stat(XML_Parser rootParser, char operator, XmlBigCount absDiff, - XmlBigCount newTotal, XmlBigCount peakTotal, int sourceLine) { +expat_heap_stat(XML_Parser rootParser, char operator, size_t absDiff, + size_t newTotal, size_t peakTotal, int sourceLine) { // NOTE: This can be +infinity or -nan const float amplification = (float)newTotal / (float)rootParser->m_accounting.countBytesDirect; fprintf( stderr, - "expat: Allocations(%p): Direct " EXPAT_FMT_ULL("10") ", allocated %c" EXPAT_FMT_ULL( - "10") " to " EXPAT_FMT_ULL("10") " (" EXPAT_FMT_ULL("10") " peak), amplification %8.2f (xmlparse.c:%d)\n", + "expat: Allocations(%p): Direct " EXPAT_FMT_ULL("10") ", allocated %c" EXPAT_FMT_SIZE_T( + "10") " to " EXPAT_FMT_SIZE_T("10") " (" EXPAT_FMT_SIZE_T("10") " peak), amplification %8.2f (xmlparse.c:%d)\n", (void *)rootParser, rootParser->m_accounting.countBytesDirect, operator, absDiff, newTotal, peakTotal, (double)amplification, sourceLine); } static bool -expat_heap_increase_tolerable(XML_Parser rootParser, XmlBigCount increase, +expat_heap_increase_tolerable(XML_Parser rootParser, size_t increase, int sourceLine) { assert(rootParser != NULL); assert(increase > 0); - XmlBigCount newTotal = 0; + size_t newTotal = 0; bool tolerable = true; // Detect integer overflow - if ((XmlBigCount)-1 - rootParser->m_alloc_tracker.bytesAllocated < increase) { + if (SIZE_MAX - rootParser->m_alloc_tracker.bytesAllocated < increase) { tolerable = false; } else { newTotal = rootParser->m_alloc_tracker.bytesAllocated + increase; @@ -887,8 +906,7 @@ expat_malloc(XML_Parser parser, size_t size, int sourceLine) { const size_t bytesToAllocate = sizeof(size_t) + EXPAT_MALLOC_PADDING + size; - if ((XmlBigCount)-1 - rootParser->m_alloc_tracker.bytesAllocated - < bytesToAllocate) { + if (SIZE_MAX - rootParser->m_alloc_tracker.bytesAllocated < bytesToAllocate) { return NULL; // i.e. signal integer overflow as out-of-memory } @@ -998,9 +1016,10 @@ expat_realloc(XML_Parser parser, void *ptr, size_t size, int sourceLine) { } } - // NOTE: Integer overflow detection has already been done for us - // by expat_heap_increase_tolerable(..) above - assert(SIZE_MAX - sizeof(size_t) - EXPAT_MALLOC_PADDING >= size); + // Detect and prevent integer overflow + if (size > SIZE_MAX - sizeof(size_t) - EXPAT_MALLOC_PADDING) { + return NULL; + } // Actually allocate mallocedPtr = parser->m_mem.realloc_fcn( @@ -1012,8 +1031,7 @@ expat_realloc(XML_Parser parser, void *ptr, size_t size, int sourceLine) { // Update accounting if (isIncrease) { - assert((XmlBigCount)-1 - rootParser->m_alloc_tracker.bytesAllocated - >= absDiff); + assert(SIZE_MAX - rootParser->m_alloc_tracker.bytesAllocated >= absDiff); rootParser->m_alloc_tracker.bytesAllocated += absDiff; } else { // i.e. decrease assert(rootParser->m_alloc_tracker.bytesAllocated >= absDiff); @@ -1389,20 +1407,9 @@ parserCreate(const XML_Char *encodingName, FREE(parser, parser); return NULL; } -#ifdef XML_ATTR_INFO - parser->m_attInfo = MALLOC(parser, parser->m_attsSize * sizeof(XML_AttrInfo)); - if (parser->m_attInfo == NULL) { - FREE(parser, parser->m_atts); - FREE(parser, parser); - return NULL; - } -#endif parser->m_dataBuf = MALLOC(parser, INIT_DATA_BUF_SIZE * sizeof(XML_Char)); if (parser->m_dataBuf == NULL) { FREE(parser, parser->m_atts); -#ifdef XML_ATTR_INFO - FREE(parser, parser->m_attInfo); -#endif FREE(parser, parser); return NULL; } @@ -1415,9 +1422,6 @@ parserCreate(const XML_Char *encodingName, if (parser->m_dtd == NULL) { FREE(parser, parser->m_dataBuf); FREE(parser, parser->m_atts); -#ifdef XML_ATTR_INFO - FREE(parser, parser->m_attInfo); -#endif FREE(parser, parser); return NULL; } @@ -1596,6 +1600,10 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { if (parser->m_parentParser) return XML_FALSE; + // The application-defined release callback may access the parser, so it + // must run before any parser state is freed. + if (parser->m_unknownEncodingRelease) + callUnknownEncodingRelease(parser); /* move m_tagStack to m_freeTagList */ tStk = parser->m_tagStack; while (tStk) { @@ -1616,14 +1624,16 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { moveEntityList(&parser->m_freeEntities, &parser->m_openValueEntities); moveToFreeBindingList(parser, parser->m_inheritedBindings); FREE(parser, parser->m_unknownEncodingMem); - if (parser->m_unknownEncodingRelease) - callUnknownEncodingRelease(parser); poolClear(&parser->m_tempPool); poolClear(&parser->m_temp2Pool); FREE(parser, (void *)parser->m_protocolEncodingName); parser->m_protocolEncodingName = NULL; parserInit(parser, encodingName); dtdReset(parser->m_dtd, parser); + if (encodingName && ! parser->m_protocolEncodingName) { + parser->m_errorCode = XML_ERROR_NO_MEMORY; + return XML_FALSE; + } return XML_TRUE; } @@ -1856,6 +1866,10 @@ XML_ParserFree(XML_Parser parser) { TAG *tagList; if ((parser == NULL) || isCalledFromInsideHandler(parser)) return; + // The application-defined release callback may access the parser, so it + // must run before any parser state is freed. + if (parser->m_unknownEncodingRelease) + callUnknownEncodingRelease(parser); /* free m_tagStack and m_freeTagList */ tagList = parser->m_tagStack; for (;;) { @@ -1916,9 +1930,6 @@ XML_ParserFree(XML_Parser parser) { #endif /* XML_DTD */ dtdDestroy(parser->m_dtd, (XML_Bool)! parser->m_parentParser, parser); FREE(parser, parser->m_atts); -#ifdef XML_ATTR_INFO - FREE(parser, parser->m_attInfo); -#endif FREE(parser, parser->m_groupConnector); // NOTE: We are avoiding FREE(..) here because parser->m_buffer // is not being allocated with MALLOC(..) but with plain @@ -1927,8 +1938,6 @@ XML_ParserFree(XML_Parser parser) { FREE(parser, parser->m_dataBuf); FREE(parser, parser->m_nsAtts); FREE(parser, parser->m_unknownEncodingMem); - if (parser->m_unknownEncodingRelease) - callUnknownEncodingRelease(parser); FREE(parser, parser); } @@ -2009,15 +2018,6 @@ XML_GetIdAttributeIndex(XML_Parser parser) { return parser->m_idAttIndex; } -#ifdef XML_ATTR_INFO -const XML_AttrInfo *XMLCALL -XML_GetAttributeInfo(XML_Parser parser) { - if (parser == NULL) - return NULL; - return parser->m_attInfo; -} -#endif - void XMLCALL XML_SetElementHandler(XML_Parser parser, XML_StartElementHandler start, XML_EndElementHandler end) { @@ -2446,6 +2446,11 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { parser->m_parsingStatus.parsing = XML_PARSING; } + if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd)) { + parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT; + return XML_STATUS_ERROR; + } + // Detect and avoid integer overflow if ((uint64_t)len > UINT64_MAX - parser->m_parseEndByteIndex) { parser->m_errorCode = XML_ERROR_NO_MEMORY; @@ -2705,41 +2710,57 @@ XML_GetErrorCode(XML_Parser parser) { return parser->m_errorCode; } -XML_Index XMLCALL -XML_GetCurrentByteIndex(XML_Parser parser) { +int64_t XMLCALL +XML_GetCurrentByteIndex64(XML_Parser parser) { if (parser == NULL) return -1; if (parser->m_eventPtr) { - // NOTE: XML_Index is known to wrap around for >2 GiB content - // on 32bit machines and 64bit Windows, unless (non-default and - // uncommon) XML_LARGE_SIZE is defined. - // That's a bug and it only lives on because we cannot break - // ABI compatibility of public API. - return (XML_Index)(parser->m_parseEndByteIndex - - (parser->m_parseEndPtr - parser->m_eventPtr)); + return (int64_t)(parser->m_parseEndByteIndex + - (parser->m_parseEndPtr - parser->m_eventPtr)); } return -1; } -int XMLCALL -XML_GetCurrentByteCount(XML_Parser parser) { +// DEPRECATED since Expat 2.9.0. +XML_Index XMLCALL +XML_GetCurrentByteIndex(XML_Parser parser) { + // NOTE: XML_Index is known to wrap around for >2 GiB content + // on 32bit machines and 64bit Windows, unless (non-default and + // uncommon) XML_LARGE_SIZE is defined. + // That's a bug and it only lives on because we cannot break + // ABI compatibility of public API. + return (XML_Index)XML_GetCurrentByteIndex64(parser); +} + +uint64_t XMLCALL +XML_GetCurrentByteCount64(XML_Parser parser) { if (parser == NULL) return 0; - if (parser->m_eventEndPtr && parser->m_eventPtr) - return (int)(parser->m_eventEndPtr - parser->m_eventPtr); + if (parser->m_eventEndPtr && parser->m_eventPtr) { + return parser->m_eventEndPtr - parser->m_eventPtr; + } return 0; } +// DEPRECATED since Expat 2.9.0. +int XMLCALL +XML_GetCurrentByteCount(XML_Parser parser) { + // NOTE: int is known to wrap around for >2 GiB content. + // That's a bug and it only lives on because we cannot break + // ABI compatibility of public API. + return (int)XML_GetCurrentByteCount64(parser); +} + const char *XMLCALL -XML_GetInputContext(XML_Parser parser, int *offset, int *size) { +XML_GetInputContext64(XML_Parser parser, int64_t *offset, uint64_t *size) { #if XML_CONTEXT_BYTES > 0 if (parser == NULL) return NULL; if (parser->m_eventPtr && parser->m_buffer) { if (offset != NULL) - *offset = (int)(parser->m_eventPtr - parser->m_buffer); + *offset = parser->m_eventPtr - parser->m_buffer; if (size != NULL) - *size = (int)(parser->m_bufferEnd - parser->m_buffer); + *size = parser->m_bufferEnd - parser->m_buffer; return parser->m_buffer; } #else @@ -2750,8 +2771,41 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size) { return NULL; } -XML_Size XMLCALL -XML_GetCurrentLineNumber(XML_Parser parser) { +// DEPRECATED since Expat 2.9.0. +const char *XMLCALL +XML_GetInputContext(XML_Parser parser, int *offset, int *size) { +#if XML_CONTEXT_BYTES > 0 + if (parser == NULL) + return NULL; + + int64_t offset64; + uint64_t size64; + + const char *const buffer = XML_GetInputContext64(parser, &offset64, &size64); + + if (buffer == NULL) + return NULL; + + // NOTE: int is known to wrap around for >2 GiB content. + // That's a bug and it only lives on because we cannot break + // ABI compatibility of public API. + if (offset != NULL) + *offset = (int)offset64; + + if (size != NULL) + *size = (int)size64; + + return buffer; +#else + (void)parser; + (void)offset; + (void)size; +#endif /* XML_CONTEXT_BYTES > 0 */ + return NULL; +} + +uint64_t XMLCALL +XML_GetCurrentLineNumber64(XML_Parser parser) { if (parser == NULL) return 0; if (parser->m_eventPtr && parser->m_eventPtr >= parser->m_positionPtr) { @@ -2759,16 +2813,22 @@ XML_GetCurrentLineNumber(XML_Parser parser) { parser->m_eventPtr, &parser->m_position); parser->m_positionPtr = parser->m_eventPtr; } + return parser->m_position.lineNumber + 1; +} + +// DEPRECATED since Expat 2.9.0. +XML_Size XMLCALL +XML_GetCurrentLineNumber(XML_Parser parser) { // NOTE: XML_Size is known to wrap around for >4 GiB content // on 32bit machines and 64bit Windows, unless (non-default and // uncommon) XML_LARGE_SIZE is defined. // That's a bug and it only lives on because we cannot break // ABI compatibility of public API. - return (XML_Size)(parser->m_position.lineNumber + 1); + return (XML_Size)XML_GetCurrentLineNumber64(parser); } -XML_Size XMLCALL -XML_GetCurrentColumnNumber(XML_Parser parser) { +uint64_t XMLCALL +XML_GetCurrentColumnNumber64(XML_Parser parser) { if (parser == NULL) return 0; if (parser->m_eventPtr && parser->m_eventPtr >= parser->m_positionPtr) { @@ -2776,12 +2836,18 @@ XML_GetCurrentColumnNumber(XML_Parser parser) { parser->m_eventPtr, &parser->m_position); parser->m_positionPtr = parser->m_eventPtr; } + return parser->m_position.columnNumber; +} + +// DEPRECATED since Expat 2.9.0. +XML_Size XMLCALL +XML_GetCurrentColumnNumber(XML_Parser parser) { // NOTE: XML_Size is known to wrap around for >4 GiB content // on 32bit machines and 64bit Windows, unless (non-default and // uncommon) XML_LARGE_SIZE is defined. // That's a bug and it only lives on because we cannot break // ABI compatibility of public API. - return (XML_Size)parser->m_position.columnNumber; + return (XML_Size)XML_GetCurrentColumnNumber64(parser); } void XMLCALL @@ -2827,7 +2893,7 @@ XML_MemFree(XML_Parser parser, void *ptr) { void XMLCALL XML_DefaultCurrent(XML_Parser parser) { - if (parser == NULL) + if (parser == NULL || ! isCalledFromInsideHandler(parser)) return; if (parser->m_defaultHandler) { if (parser->m_openInternalEntities) @@ -2998,18 +3064,12 @@ XML_GetFeatureList(void) { {XML_FEATURE_CONTEXT_BYTES, XML_L("XML_CONTEXT_BYTES"), XML_CONTEXT_BYTES}, #endif -#ifdef XML_MIN_SIZE - {XML_FEATURE_MIN_SIZE, XML_L("XML_MIN_SIZE"), 0}, -#endif #ifdef XML_NS {XML_FEATURE_NS, XML_L("XML_NS"), 0}, #endif #ifdef XML_LARGE_SIZE {XML_FEATURE_LARGE_SIZE, XML_L("XML_LARGE_SIZE"), 0}, #endif -#ifdef XML_ATTR_INFO - {XML_FEATURE_ATTR_INFO, XML_L("XML_ATTR_INFO"), 0}, -#endif #if XML_GE == 1 /* Added in Expat 2.4.0 for XML_DTD defined and * added in Expat 2.6.0 for XML_GE == 1. */ @@ -3036,59 +3096,294 @@ XML_GetFeatureList(void) { } #if XML_GE == 1 +static enum XML_Prop_Error +setBillionLaughsAttackProtectionMaximumAmplification( + XML_Parser parser, float maximumAmplificationFactor) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (parser->m_parentParser != NULL) + return XML_PROP_ERROR_PARSER_NOT_ROOT; + + if (isnan(maximumAmplificationFactor) || (maximumAmplificationFactor < 1.0f)) + return XML_PROP_ERROR_INVALID_VALUE; + + parser->m_accounting.maximumAmplificationFactor = maximumAmplificationFactor; + + return XML_PROP_ERROR_NONE; +} + XML_Bool XMLCALL XML_SetBillionLaughsAttackProtectionMaximumAmplification( XML_Parser parser, float maximumAmplificationFactor) { - if ((parser == NULL) || (parser->m_parentParser != NULL) - || isnan(maximumAmplificationFactor) - || (maximumAmplificationFactor < 1.0f)) { - return XML_FALSE; - } - parser->m_accounting.maximumAmplificationFactor = maximumAmplificationFactor; - return XML_TRUE; + return (setBillionLaughsAttackProtectionMaximumAmplification( + parser, maximumAmplificationFactor) + == XML_PROP_ERROR_NONE) + ? XML_TRUE + : XML_FALSE; +} + +static enum XML_Prop_Error +setBillionLaughsAttackProtectionActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (parser->m_parentParser != NULL) + return XML_PROP_ERROR_PARSER_NOT_ROOT; + + parser->m_accounting.activationThresholdBytes = activationThresholdBytes; + + return XML_PROP_ERROR_NONE; } XML_Bool XMLCALL XML_SetBillionLaughsAttackProtectionActivationThreshold( XML_Parser parser, unsigned long long activationThresholdBytes) { - if ((parser == NULL) || (parser->m_parentParser != NULL)) { - return XML_FALSE; - } - parser->m_accounting.activationThresholdBytes = activationThresholdBytes; - return XML_TRUE; + return (setBillionLaughsAttackProtectionActivationThreshold( + parser, activationThresholdBytes) + == XML_PROP_ERROR_NONE) + ? XML_TRUE + : XML_FALSE; +} + +static enum XML_Prop_Error +setAllocTrackerMaximumAmplification(XML_Parser parser, + float maximumAmplificationFactor) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (parser->m_parentParser != NULL) + return XML_PROP_ERROR_PARSER_NOT_ROOT; + + if (isnan(maximumAmplificationFactor) || (maximumAmplificationFactor < 1.0f)) + return XML_PROP_ERROR_INVALID_VALUE; + + parser->m_alloc_tracker.maximumAmplificationFactor + = maximumAmplificationFactor; + + return XML_PROP_ERROR_NONE; } XML_Bool XMLCALL XML_SetAllocTrackerMaximumAmplification(XML_Parser parser, float maximumAmplificationFactor) { - if ((parser == NULL) || (parser->m_parentParser != NULL) - || isnan(maximumAmplificationFactor) - || (maximumAmplificationFactor < 1.0f)) { - return XML_FALSE; - } - parser->m_alloc_tracker.maximumAmplificationFactor - = maximumAmplificationFactor; - return XML_TRUE; + return (setAllocTrackerMaximumAmplification(parser, + maximumAmplificationFactor) + == XML_PROP_ERROR_NONE) + ? XML_TRUE + : XML_FALSE; +} + +static enum XML_Prop_Error +setAllocTrackerActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (parser->m_parentParser != NULL) + return XML_PROP_ERROR_PARSER_NOT_ROOT; + + parser->m_alloc_tracker.activationThresholdBytes = activationThresholdBytes; + + return XML_PROP_ERROR_NONE; } XML_Bool XMLCALL XML_SetAllocTrackerActivationThreshold( XML_Parser parser, unsigned long long activationThresholdBytes) { - if ((parser == NULL) || (parser->m_parentParser != NULL)) { - return XML_FALSE; - } - parser->m_alloc_tracker.activationThresholdBytes = activationThresholdBytes; - return XML_TRUE; + return (setAllocTrackerActivationThreshold(parser, activationThresholdBytes) + == XML_PROP_ERROR_NONE) + ? XML_TRUE + : XML_FALSE; } #endif /* XML_GE == 1 */ +static enum XML_Prop_Error +setReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (enabled != XML_TRUE && enabled != XML_FALSE) + return XML_PROP_ERROR_INVALID_VALUE; + + parser->m_reparseDeferralEnabled = enabled; + + return XML_PROP_ERROR_NONE; +} + XML_Bool XMLCALL XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled) { - if (parser != NULL && (enabled == XML_TRUE || enabled == XML_FALSE)) { - parser->m_reparseDeferralEnabled = enabled; - return XML_TRUE; + return (setReparseDeferralEnabled(parser, enabled) == XML_PROP_ERROR_NONE) + ? XML_TRUE + : XML_FALSE; +} + +enum XML_Prop_Error XMLCALL +XML_SetPropertyBool(XML_Parser parser, enum XML_Parser_Property property, + XML_Bool value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + switch (property) { +#if XML_GE == 1 + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: + return XML_PROP_ERROR_INVALID_TYPE; +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + return setReparseDeferralEnabled(parser, value); + default: + return XML_PROP_ERROR_INVALID_KEY; + } + + assert(0 && "considered unreachable"); +} + +enum XML_Prop_Error XMLCALL +XML_SetPropertyDouble(XML_Parser parser, enum XML_Parser_Property property, + double value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + +#if XML_GE == 0 + UNUSED_P(value); +#endif + + switch (property) { +#if XML_GE == 1 + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + return setAllocTrackerMaximumAmplification(parser, (float)value); + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: + return setBillionLaughsAttackProtectionMaximumAmplification(parser, + (float)value); + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + return XML_PROP_ERROR_INVALID_TYPE; + default: + return XML_PROP_ERROR_INVALID_KEY; + } + + assert(0 && "considered unreachable"); +} + +enum XML_Prop_Error XMLCALL +XML_SetPropertyUInt64(XML_Parser parser, enum XML_Parser_Property property, + uint64_t value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + +#if XML_GE == 0 + UNUSED_P(value); +#endif + + switch (property) { +#if XML_GE == 1 + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + return setAllocTrackerActivationThreshold(parser, + (unsigned long long)value); + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: + return setBillionLaughsAttackProtectionActivationThreshold( + parser, (unsigned long long)value); + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + return XML_PROP_ERROR_INVALID_TYPE; + default: + return XML_PROP_ERROR_INVALID_KEY; + } + + assert(0 && "considered unreachable"); +} + +enum XML_Prop_Error XMLCALL +XML_GetPropertyBool(XML_Parser parser, enum XML_Parser_Property property, + XML_Bool *value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (value == NULL) + return XML_PROP_ERROR_INVALID_VALUE; + + switch (property) { +#if XML_GE == 1 + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: + return XML_PROP_ERROR_INVALID_TYPE; +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + *value = parser->m_reparseDeferralEnabled; + break; + default: + return XML_PROP_ERROR_INVALID_KEY; + } + + return XML_PROP_ERROR_NONE; +} + +enum XML_Prop_Error XMLCALL +XML_GetPropertyDouble(XML_Parser parser, enum XML_Parser_Property property, + double *value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (value == NULL) + return XML_PROP_ERROR_INVALID_VALUE; + + switch (property) { +#if XML_GE == 1 + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: + *value = (double)parser->m_accounting.maximumAmplificationFactor; + break; + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + *value = (double)parser->m_alloc_tracker.maximumAmplificationFactor; + break; + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + return XML_PROP_ERROR_INVALID_TYPE; + default: + return XML_PROP_ERROR_INVALID_KEY; } - return XML_FALSE; + + return XML_PROP_ERROR_NONE; +} + +enum XML_Prop_Error XMLCALL +XML_GetPropertyUInt64(XML_Parser parser, enum XML_Parser_Property property, + uint64_t *value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (value == NULL) + return XML_PROP_ERROR_INVALID_VALUE; + + switch (property) { +#if XML_GE == 1 + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: + *value = parser->m_accounting.activationThresholdBytes; + break; + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + *value = parser->m_alloc_tracker.activationThresholdBytes; + break; + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + return XML_PROP_ERROR_INVALID_TYPE; + default: + return XML_PROP_ERROR_INVALID_KEY; + } + + return XML_PROP_ERROR_NONE; } /* Initially tag->rawName always points into the parse buffer; @@ -3901,21 +4196,6 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, return XML_ERROR_NO_MEMORY; } parser->m_atts = temp; -#ifdef XML_ATTR_INFO - /* Detect and prevent integer overflow. */ - if (parser->m_attsSize > SIZE_MAX / sizeof(XML_AttrInfo)) { - parser->m_attsSize = oldAttsSize; - return XML_ERROR_NO_MEMORY; - } - - XML_AttrInfo *const temp2 = REALLOC( - parser, parser->m_attInfo, parser->m_attsSize * sizeof(XML_AttrInfo)); - if (temp2 == NULL) { - parser->m_attsSize = oldAttsSize; - return XML_ERROR_NO_MEMORY; - } - parser->m_attInfo = temp2; -#endif if (n > oldAttsSize) { /* Detect and prevent integer overflow. */ if (n > (size_t)INT_MAX) @@ -3928,33 +4208,12 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, const XML_Char **const appAtts = (const XML_Char **)parser->m_atts; for (size_t i = 0; i < n; i++) { ATTRIBUTE *currAtt = &parser->m_atts[i]; -#ifdef XML_ATTR_INFO - XML_AttrInfo *currAttInfo = &parser->m_attInfo[i]; -#endif /* add the name and value to the attribute list */ ATTRIBUTE_ID *attId = getAttributeId(parser, enc, currAtt->name, currAtt->name + XmlNameLength(enc, currAtt->name)); if (! attId) return XML_ERROR_NO_MEMORY; -#ifdef XML_ATTR_INFO - // NOTE: XML_Index is known to wrap around for >2 GiB content - // on 32bit machines and 64bit Windows, unless (non-default and - // uncommon) XML_LARGE_SIZE is defined. - // That's a bug and it only lives on because we cannot break - // ABI compatibility of public API. - currAttInfo->nameStart - = (XML_Index)(parser->m_parseEndByteIndex - - (parser->m_parseEndPtr - currAtt->name)); - currAttInfo->nameEnd - = currAttInfo->nameStart + XmlNameLength(enc, currAtt->name); - currAttInfo->valueStart - = (XML_Index)(parser->m_parseEndByteIndex - - (parser->m_parseEndPtr - currAtt->valuePtr)); - currAttInfo->valueEnd - = (XML_Index)(parser->m_parseEndByteIndex - - (parser->m_parseEndPtr - currAtt->valueEnd)); -#endif /* Detect duplicate attributes by their QNames. This does not work when namespace processing is turned on and different prefixes for the same namespace are used. For this case we have a check further down. diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index a3bf5e0e95b0e03..2f837acc4cbf72a 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -193,13 +193,6 @@ utf8_isInvalid4(const ENCODING *enc, const char *p) { struct normal_encoding { ENCODING enc; unsigned char type[256]; -#ifdef XML_MIN_SIZE - int (*byteType)(const ENCODING *, const char *); - int (*isNameMin)(const ENCODING *, const char *); - int (*isNmstrtMin)(const ENCODING *, const char *); - int (*byteToAscii)(const ENCODING *, const char *); - int (*charMatches)(const ENCODING *, const char *, int); -#endif /* XML_MIN_SIZE */ int (*isName2)(const ENCODING *, const char *); int (*isName3)(const ENCODING *, const char *); int (*isName4)(const ENCODING *, const char *); @@ -213,16 +206,7 @@ struct normal_encoding { #define AS_NORMAL_ENCODING(enc) ((const struct normal_encoding *)(enc)) -#ifdef XML_MIN_SIZE - -# define STANDARD_VTABLE(E) \ - E##byteType, E##isNameMin, E##isNmstrtMin, E##byteToAscii, E##charMatches, - -#else - -# define STANDARD_VTABLE(E) /* as nothing */ - -#endif +#define STANDARD_VTABLE(E) /* as nothing */ #define NORMAL_VTABLE(E) \ E##isName2, E##isName3, E##isName4, E##isNmstrt2, E##isNmstrt3, \ @@ -233,93 +217,33 @@ struct normal_encoding { /* isNmstrt2 */ NULL, /* isNmstrt3 */ NULL, /* isNmstrt4 */ NULL, \ /* isInvalid2 */ NULL, /* isInvalid3 */ NULL, /* isInvalid4 */ NULL -/* Like NULL_VTABLE but with a real isInvalid4 so the UTF-16 encodings reject a - high surrogate that is not followed by a low surrogate. Only needed for the - XML_MIN_SIZE build, where the shared tokenizer dispatches through the vtable; - the regular build inlines the same check via IS_INVALID_CHAR. */ -#ifdef XML_MIN_SIZE -# define UTF16_NULL_VTABLE(E) \ - /* isName2 */ NULL, /* isName3 */ NULL, /* isName4 */ NULL, \ - /* isNmstrt2 */ NULL, /* isNmstrt3 */ NULL, /* isNmstrt4 */ NULL, \ - /* isInvalid2 */ NULL, /* isInvalid3 */ NULL, E##isInvalid4 -#else -# define UTF16_NULL_VTABLE(E) NULL_VTABLE -#endif +#define UTF16_NULL_VTABLE(E) NULL_VTABLE static int checkCharRefNumber(int result); #include "xmltok_impl.h" #include "ascii.h" -#ifdef XML_MIN_SIZE -# define sb_isNameMin isNever -# define sb_isNmstrtMin isNever -#endif - -#ifdef XML_MIN_SIZE -# define MINBPC(enc) ((enc)->minBytesPerChar) -#else /* minimum bytes per character */ -# define MINBPC(enc) 1 -#endif +#define MINBPC(enc) 1 #define SB_BYTE_TYPE(enc, p) \ (((const struct normal_encoding *)(enc))->type[(unsigned char)*(p)]) -#ifdef XML_MIN_SIZE -static int -sb_byteType(const ENCODING *enc, const char *p) { - return SB_BYTE_TYPE(enc, p); -} -# define BYTE_TYPE(enc, p) (AS_NORMAL_ENCODING(enc)->byteType(enc, p)) -#else -# define BYTE_TYPE(enc, p) SB_BYTE_TYPE(enc, p) -#endif +#define BYTE_TYPE(enc, p) SB_BYTE_TYPE(enc, p) -#ifdef XML_MIN_SIZE -# define BYTE_TO_ASCII(enc, p) (AS_NORMAL_ENCODING(enc)->byteToAscii(enc, p)) -static int -sb_byteToAscii(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return *p; -} -#else -# define BYTE_TO_ASCII(enc, p) (*(p)) -#endif +#define BYTE_TO_ASCII(enc, p) (*(p)) #define IS_NAME_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isName##n(enc, p)) #define IS_NMSTRT_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isNmstrt##n(enc, p)) -#ifdef XML_MIN_SIZE -# define IS_INVALID_CHAR(enc, p, n) \ - (AS_NORMAL_ENCODING(enc)->isInvalid##n \ - && AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p)) -#else -# define IS_INVALID_CHAR(enc, p, n) \ - (AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p)) -#endif +#define IS_INVALID_CHAR(enc, p, n) \ + (AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p)) -#ifdef XML_MIN_SIZE -# define IS_NAME_CHAR_MINBPC(enc, p) \ - (AS_NORMAL_ENCODING(enc)->isNameMin(enc, p)) -# define IS_NMSTRT_CHAR_MINBPC(enc, p) \ - (AS_NORMAL_ENCODING(enc)->isNmstrtMin(enc, p)) -#else -# define IS_NAME_CHAR_MINBPC(enc, p) (0) -# define IS_NMSTRT_CHAR_MINBPC(enc, p) (0) -#endif +#define IS_NAME_CHAR_MINBPC(enc, p) (0) +#define IS_NMSTRT_CHAR_MINBPC(enc, p) (0) -#ifdef XML_MIN_SIZE -# define CHAR_MATCHES(enc, p, c) \ - (AS_NORMAL_ENCODING(enc)->charMatches(enc, p, c)) -static int -sb_charMatches(const ENCODING *enc, const char *p, int c) { - UNUSED_P(enc); - return *p == c; -} -#else /* c is an ASCII character */ -# define CHAR_MATCHES(enc, p, c) (*(p) == (c)) -#endif +#define CHAR_MATCHES(enc, p, c) (*(p) == (c)) #define PREFIX(ident) normal_##ident #define XML_TOK_IMPL_C @@ -769,76 +693,32 @@ DEFINE_UTF16_TO_UTF16(big2_) #define LITTLE2_IS_INVALID_CHAR(p, n) \ ((n) == 4 && ((unsigned char)(p)[3] & 0xFC) != 0xDC) -#ifdef XML_MIN_SIZE - -static int -little2_byteType(const ENCODING *enc, const char *p) { - return LITTLE2_BYTE_TYPE(enc, p); -} - -static int -little2_byteToAscii(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return LITTLE2_BYTE_TO_ASCII(p); -} - -static int -little2_charMatches(const ENCODING *enc, const char *p, int c) { - UNUSED_P(enc); - return LITTLE2_CHAR_MATCHES(p, c); -} - -static int -little2_isNameMin(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return LITTLE2_IS_NAME_CHAR_MINBPC(p); -} - -static int -little2_isNmstrtMin(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return LITTLE2_IS_NMSTRT_CHAR_MINBPC(p); -} - -static int -little2_isInvalid4(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return LITTLE2_IS_INVALID_CHAR(p, 4); -} - -# undef VTABLE -# define VTABLE VTABLE1, little2_toUtf8, little2_toUtf16 +#undef PREFIX +#define PREFIX(ident) little2_##ident +#define MINBPC(enc) 2 +/* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */ +#define BYTE_TYPE(enc, p) LITTLE2_BYTE_TYPE(enc, p) +#define BYTE_TO_ASCII(enc, p) LITTLE2_BYTE_TO_ASCII(p) +#define CHAR_MATCHES(enc, p, c) LITTLE2_CHAR_MATCHES(p, c) +#define IS_NAME_CHAR(enc, p, n) 0 +#define IS_NAME_CHAR_MINBPC(enc, p) LITTLE2_IS_NAME_CHAR_MINBPC(p) +#define IS_NMSTRT_CHAR(enc, p, n) (0) +#define IS_NMSTRT_CHAR_MINBPC(enc, p) LITTLE2_IS_NMSTRT_CHAR_MINBPC(p) +#define IS_INVALID_CHAR(enc, p, n) LITTLE2_IS_INVALID_CHAR(p, n) -#else /* not XML_MIN_SIZE */ +#define XML_TOK_IMPL_C +#include "xmltok_impl.c" +#undef XML_TOK_IMPL_C -# undef PREFIX -# define PREFIX(ident) little2_##ident -# define MINBPC(enc) 2 -/* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */ -# define BYTE_TYPE(enc, p) LITTLE2_BYTE_TYPE(enc, p) -# define BYTE_TO_ASCII(enc, p) LITTLE2_BYTE_TO_ASCII(p) -# define CHAR_MATCHES(enc, p, c) LITTLE2_CHAR_MATCHES(p, c) -# define IS_NAME_CHAR(enc, p, n) 0 -# define IS_NAME_CHAR_MINBPC(enc, p) LITTLE2_IS_NAME_CHAR_MINBPC(p) -# define IS_NMSTRT_CHAR(enc, p, n) (0) -# define IS_NMSTRT_CHAR_MINBPC(enc, p) LITTLE2_IS_NMSTRT_CHAR_MINBPC(p) -# define IS_INVALID_CHAR(enc, p, n) LITTLE2_IS_INVALID_CHAR(p, n) - -# define XML_TOK_IMPL_C -# include "xmltok_impl.c" -# undef XML_TOK_IMPL_C - -# undef MINBPC -# undef BYTE_TYPE -# undef BYTE_TO_ASCII -# undef CHAR_MATCHES -# undef IS_NAME_CHAR -# undef IS_NAME_CHAR_MINBPC -# undef IS_NMSTRT_CHAR -# undef IS_NMSTRT_CHAR_MINBPC -# undef IS_INVALID_CHAR - -#endif /* not XML_MIN_SIZE */ +#undef MINBPC +#undef BYTE_TYPE +#undef BYTE_TO_ASCII +#undef CHAR_MATCHES +#undef IS_NAME_CHAR +#undef IS_NAME_CHAR_MINBPC +#undef IS_NMSTRT_CHAR +#undef IS_NMSTRT_CHAR_MINBPC +#undef IS_INVALID_CHAR #ifdef XML_NS @@ -914,76 +794,32 @@ static const struct normal_encoding internal_little2_encoding #define BIG2_IS_INVALID_CHAR(p, n) \ ((n) == 4 && ((unsigned char)(p)[2] & 0xFC) != 0xDC) -#ifdef XML_MIN_SIZE - -static int -big2_byteType(const ENCODING *enc, const char *p) { - return BIG2_BYTE_TYPE(enc, p); -} - -static int -big2_byteToAscii(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return BIG2_BYTE_TO_ASCII(p); -} - -static int -big2_charMatches(const ENCODING *enc, const char *p, int c) { - UNUSED_P(enc); - return BIG2_CHAR_MATCHES(p, c); -} - -static int -big2_isNameMin(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return BIG2_IS_NAME_CHAR_MINBPC(p); -} - -static int -big2_isNmstrtMin(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return BIG2_IS_NMSTRT_CHAR_MINBPC(p); -} - -static int -big2_isInvalid4(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return BIG2_IS_INVALID_CHAR(p, 4); -} - -# undef VTABLE -# define VTABLE VTABLE1, big2_toUtf8, big2_toUtf16 +#undef PREFIX +#define PREFIX(ident) big2_##ident +#define MINBPC(enc) 2 +/* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */ +#define BYTE_TYPE(enc, p) BIG2_BYTE_TYPE(enc, p) +#define BYTE_TO_ASCII(enc, p) BIG2_BYTE_TO_ASCII(p) +#define CHAR_MATCHES(enc, p, c) BIG2_CHAR_MATCHES(p, c) +#define IS_NAME_CHAR(enc, p, n) 0 +#define IS_NAME_CHAR_MINBPC(enc, p) BIG2_IS_NAME_CHAR_MINBPC(p) +#define IS_NMSTRT_CHAR(enc, p, n) (0) +#define IS_NMSTRT_CHAR_MINBPC(enc, p) BIG2_IS_NMSTRT_CHAR_MINBPC(p) +#define IS_INVALID_CHAR(enc, p, n) BIG2_IS_INVALID_CHAR(p, n) -#else /* not XML_MIN_SIZE */ +#define XML_TOK_IMPL_C +#include "xmltok_impl.c" +#undef XML_TOK_IMPL_C -# undef PREFIX -# define PREFIX(ident) big2_##ident -# define MINBPC(enc) 2 -/* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */ -# define BYTE_TYPE(enc, p) BIG2_BYTE_TYPE(enc, p) -# define BYTE_TO_ASCII(enc, p) BIG2_BYTE_TO_ASCII(p) -# define CHAR_MATCHES(enc, p, c) BIG2_CHAR_MATCHES(p, c) -# define IS_NAME_CHAR(enc, p, n) 0 -# define IS_NAME_CHAR_MINBPC(enc, p) BIG2_IS_NAME_CHAR_MINBPC(p) -# define IS_NMSTRT_CHAR(enc, p, n) (0) -# define IS_NMSTRT_CHAR_MINBPC(enc, p) BIG2_IS_NMSTRT_CHAR_MINBPC(p) -# define IS_INVALID_CHAR(enc, p, n) BIG2_IS_INVALID_CHAR(p, n) - -# define XML_TOK_IMPL_C -# include "xmltok_impl.c" -# undef XML_TOK_IMPL_C - -# undef MINBPC -# undef BYTE_TYPE -# undef BYTE_TO_ASCII -# undef CHAR_MATCHES -# undef IS_NAME_CHAR -# undef IS_NAME_CHAR_MINBPC -# undef IS_NMSTRT_CHAR -# undef IS_NMSTRT_CHAR_MINBPC -# undef IS_INVALID_CHAR - -#endif /* not XML_MIN_SIZE */ +#undef MINBPC +#undef BYTE_TYPE +#undef BYTE_TO_ASCII +#undef CHAR_MATCHES +#undef IS_NAME_CHAR +#undef IS_NAME_CHAR_MINBPC +#undef IS_NMSTRT_CHAR +#undef IS_NMSTRT_CHAR_MINBPC +#undef IS_INVALID_CHAR #ifdef XML_NS diff --git a/Modules/expat/xmltok_impl.c b/Modules/expat/xmltok_impl.c index a71a2fd88c704b6..7900e94701e2768 100644 --- a/Modules/expat/xmltok_impl.c +++ b/Modules/expat/xmltok_impl.c @@ -1,4 +1,4 @@ -/* This file is included (from xmltok.c, 1-3 times depending on XML_MIN_SIZE)! +/* This file is included (from xmltok.c, 3 times)! __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 80bc6dc70b5b03a..702cc371d5b34a8 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -18,6 +18,16 @@ module pyexpat #define XML_COMBINED_VERSION (10000*XML_MAJOR_VERSION+100*XML_MINOR_VERSION+XML_MICRO_VERSION) +#if XML_COMBINED_VERSION >= 20900 +# define EXPAT_GetCurrentLineNumber XML_GetCurrentLineNumber64 +# define EXPAT_GetCurrentColumnNumber XML_GetCurrentColumnNumber64 +# define EXPAT_GetCurrentByteIndex XML_GetCurrentByteIndex64 +#else +# define EXPAT_GetCurrentLineNumber XML_GetCurrentLineNumber +# define EXPAT_GetCurrentColumnNumber XML_GetCurrentColumnNumber +# define EXPAT_GetCurrentByteIndex XML_GetCurrentByteIndex +#endif + static XML_Memory_Handling_Suite ExpatMemoryHandler = { PyObject_Malloc, PyObject_Realloc, PyObject_Free}; @@ -127,9 +137,10 @@ set_xml_error_attr_code(pyexpat_state *state, PyObject *err, enum XML_Error code * false on an exception. */ static int -set_xml_error_attr_location(PyObject *err, const char *name, XML_Size value) +set_xml_error_attr_location(PyObject *err, const char *name, + unsigned long long value) { - PyObject *v = PyLong_FromSize_t((size_t)value); + PyObject *v = PyLong_FromUnsignedLongLong(value); int ok = v != NULL && PyObject_SetAttrString(err, name, v) != -1; Py_XDECREF(v); return ok; @@ -138,15 +149,16 @@ set_xml_error_attr_location(PyObject *err, const char *name, XML_Size value) static PyObject * set_xml_error(pyexpat_state *state, - enum XML_Error code, XML_Size lineno, XML_Size column, + enum XML_Error code, + unsigned long long lineno, unsigned long long column, const char *errmsg) { PyObject *arg; if (errmsg == NULL) { arg = PyUnicode_FromFormat( - "%s: line %zu, column %zu", + "%s: line %llu, column %llu", XML_ErrorString(code), - (size_t)lineno, (size_t)column + lineno, column ); } else { @@ -173,8 +185,10 @@ set_xml_error(pyexpat_state *state, do { \ XML_Parser parser = SELF->itself; \ assert(parser != NULL); \ - XML_Size lineno = XML_GetCurrentLineNumber(parser); \ - XML_Size column = XML_GetCurrentColumnNumber(parser); \ + unsigned long long lineno \ + = EXPAT_GetCurrentLineNumber(parser); \ + unsigned long long column \ + = EXPAT_GetCurrentColumnNumber(parser); \ (void)set_xml_error(state, CODE, lineno, column, ERRMSG); \ } while (0) @@ -1003,13 +1017,20 @@ pyexpat_xmlparser_GetInputContext_impl(xmlparseobject *self) /*[clinic end generated code: output=a88026d683fc22cc input=034df8712db68379]*/ { if (self->in_callback) { +#if XML_COMBINED_VERSION >= 20900 + int64_t offset; + uint64_t size; + const char *buffer + = XML_GetInputContext64(self->itself, &offset, &size); +#else int offset, size; const char *buffer = XML_GetInputContext(self->itself, &offset, &size); +#endif if (buffer != NULL) return PyBytes_FromStringAndSize(buffer + offset, - size - offset); + (Py_ssize_t)(size - offset)); else Py_RETURN_NONE; } @@ -1583,19 +1604,19 @@ xmlparse_handler_setter(xmlparseobject *self, PyObject *v, struct HandlerInfo *h return 0; } -#define INT_GETTER(name) \ +#define INT_GETTER(name, func) \ static PyObject * \ xmlparse_##name##_getter(xmlparseobject *self, void *closure) \ { \ - return PyLong_FromLong((long) XML_Get##name(self->itself)); \ + return PyLong_FromLongLong((long long)func(self->itself)); \ } -INT_GETTER(ErrorCode) -INT_GETTER(ErrorLineNumber) -INT_GETTER(ErrorColumnNumber) -INT_GETTER(ErrorByteIndex) -INT_GETTER(CurrentLineNumber) -INT_GETTER(CurrentColumnNumber) -INT_GETTER(CurrentByteIndex) +INT_GETTER(ErrorCode, XML_GetErrorCode) +INT_GETTER(ErrorLineNumber, EXPAT_GetCurrentLineNumber) +INT_GETTER(ErrorColumnNumber, EXPAT_GetCurrentColumnNumber) +INT_GETTER(ErrorByteIndex, EXPAT_GetCurrentByteIndex) +INT_GETTER(CurrentLineNumber, EXPAT_GetCurrentLineNumber) +INT_GETTER(CurrentColumnNumber, EXPAT_GetCurrentColumnNumber) +INT_GETTER(CurrentByteIndex, EXPAT_GetCurrentByteIndex) #undef INT_GETTER @@ -2243,8 +2264,11 @@ pyexpat_exec(PyObject *mod) capi.MICRO_VERSION = XML_MICRO_VERSION; capi.ErrorString = XML_ErrorString; capi.GetErrorCode = XML_GetErrorCode; +_Py_COMP_DIAG_PUSH +_Py_COMP_DIAG_IGNORE_DEPR_DECLS capi.GetErrorColumnNumber = XML_GetErrorColumnNumber; capi.GetErrorLineNumber = XML_GetErrorLineNumber; +_Py_COMP_DIAG_POP capi.Parse = XML_Parse; capi.ParserCreate_MM = XML_ParserCreate_MM; capi.ParserFree = XML_ParserFree; diff --git a/PCbuild/_elementtree.vcxproj b/PCbuild/_elementtree.vcxproj index 9c4210cfef9c202..3f4dea5c4ed46a6 100644 --- a/PCbuild/_elementtree.vcxproj +++ b/PCbuild/_elementtree.vcxproj @@ -113,11 +113,13 @@ + + diff --git a/PCbuild/_elementtree.vcxproj.filters b/PCbuild/_elementtree.vcxproj.filters index 9e2e062af3f3877..2ce694e1b5390fe 100644 --- a/PCbuild/_elementtree.vcxproj.filters +++ b/PCbuild/_elementtree.vcxproj.filters @@ -63,6 +63,9 @@ Header Files\expat + + Header Files\expat + Header Files\expat @@ -74,6 +77,9 @@ Source Files + + Source Files\expat + Source Files\expat diff --git a/PCbuild/_ssl.vcxproj b/PCbuild/_ssl.vcxproj index 4907f49b6628fa6..965cc1f3acb762f 100644 --- a/PCbuild/_ssl.vcxproj +++ b/PCbuild/_ssl.vcxproj @@ -97,6 +97,9 @@ ws2_32.lib;crypt32.lib;%(AdditionalDependencies) + + + diff --git a/PCbuild/_ssl.vcxproj.filters b/PCbuild/_ssl.vcxproj.filters index 716a69a41af3517..3df9d70532edc57 100644 --- a/PCbuild/_ssl.vcxproj.filters +++ b/PCbuild/_ssl.vcxproj.filters @@ -4,6 +4,9 @@ {695348f7-e9f6-4fe1-bc03-5f08ffc8095b} + + {7c1bd5da-8912-4107-b6ac-f3930f2d90c7} + {1b18a2e6-040d-46c7-a9ac-ac2ec64fb5d6} @@ -16,6 +19,11 @@ Source Files + + + Header Files + + Resource Files diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat index c59479876627dfd..4b7d27838a86ffb 100644 --- a/PCbuild/find_python.bat +++ b/PCbuild/find_python.bat @@ -37,7 +37,7 @@ @if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 8)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found @rem If py.exe finds a recent enough version, use that one -@for %%p in (3.9 3.8) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found +@for %%p in (3.14 3.13 3.12 3.11 3.10 3.9 3.8) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found @if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%" @set _Py_NUGET=%NUGET% diff --git a/PCbuild/pyexpat.vcxproj b/PCbuild/pyexpat.vcxproj index a81383a81304bea..2e3f865f68b3de9 100644 --- a/PCbuild/pyexpat.vcxproj +++ b/PCbuild/pyexpat.vcxproj @@ -98,11 +98,13 @@ + + diff --git a/PCbuild/pyexpat.vcxproj.filters b/PCbuild/pyexpat.vcxproj.filters index e63cb9c63063aca..baa06e4491f6850 100644 --- a/PCbuild/pyexpat.vcxproj.filters +++ b/PCbuild/pyexpat.vcxproj.filters @@ -21,6 +21,9 @@ Header Files + + Header Files + Header Files @@ -32,6 +35,9 @@ Source Files + + Source Files + Source Files diff --git a/Python/makeopcodetargets.py b/Python/makeopcodetargets.py index 023c9e6c9f1adc1..4486cf65a9458dc 100755 --- a/Python/makeopcodetargets.py +++ b/Python/makeopcodetargets.py @@ -3,28 +3,20 @@ (for compilers supporting computed gotos or "labels-as-values", such as gcc). """ +import importlib.util import os import sys -try: - from importlib.machinery import SourceFileLoader -except ImportError: - import imp - - def find_module(modname): - """Finds and returns a module in the local dist/checkout. - """ - modpath = os.path.join( - os.path.dirname(os.path.dirname(__file__)), "Lib") - return imp.load_module(modname, *imp.find_module(modname, [modpath])) -else: - def find_module(modname): - """Finds and returns a module in the local dist/checkout. - """ - modpath = os.path.join( - os.path.dirname(os.path.dirname(__file__)), "Lib", modname + ".py") - return SourceFileLoader(modname, modpath).load_module() +def find_module(modname): + """Finds and returns a module in the local dist/checkout. + """ + modpath = os.path.join( + os.path.dirname(os.path.dirname(__file__)), "Lib", modname + ".py") + spec = importlib.util.spec_from_file_location(modname, modpath) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module def write_contents(f): diff --git a/setup.py b/setup.py index a39610a1c7cd7d4..7fb4a72adbf70f9 100644 --- a/setup.py +++ b/setup.py @@ -1789,7 +1789,8 @@ def detect_expat_elementtree(self): # bpo-44394: libexpat uses isnan() of math.h and needs linkage # against the libm expat_lib = ['m'] - expat_sources = ['expat/xmlparse.c', + expat_sources = ['expat/xcs.c', + 'expat/xmlparse.c', 'expat/xmlrole.c', 'expat/xmltok.c'] expat_depends = ['expat/ascii.h', @@ -1800,6 +1801,7 @@ def detect_expat_elementtree(self): 'expat/internal.h', 'expat/latin1tab.h', 'expat/utf8tab.h', + 'expat/xcs.h', 'expat/xmlrole.h', 'expat/xmltok.h', 'expat/xmltok_impl.h' @@ -2511,6 +2513,7 @@ def split_var(name, sep): '_ssl/debughelpers.c', '_ssl/misc.c', '_ssl/cert.c', + '_ssl/distrusted_spki.h', ], **openssl_extension_kwargs )