diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index d70d9b54f..bccfc91c1 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -33,7 +33,7 @@ jobs: - name: "Run unit tests" run: | - uv run python -m unittest tests/test_mirror.py tests/test_archive.py + uv run python -m unittest tests/test_mirror.py tests/test_archive.py tests/test_windows.py - name: "Run mirror integration test" run: | diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index a9ad6317b..08e146bcd 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -137,6 +137,23 @@ jobs: shell: pwsh run: ./scripts/install-cygwin.ps1 -Packages autoconf,automake,libtool + - name: Install x64 MSVC v143 tools for ARM64EC dependencies + if: ${{ matrix.arch == 'arm64ec' }} + shell: pwsh + run: | + $installer = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer" + $installation = & "$installer\vswhere.exe" -latest -products '*' -version '[18,19)' -property installationPath + $arguments = @( + 'modify', '--installPath', "`"$installation`"", + '--add', 'Microsoft.VisualStudio.Component.VC.14.44.17.14.x86.x64', + '--quiet', '--norestart', '--nocache' + ) + $process = Start-Process "$installer\setup.exe" -ArgumentList $arguments -Wait -PassThru + if ($process.ExitCode -notin @(0, 3010)) { + throw "Visual Studio setup failed with exit code $($process.ExitCode)" + } + Get-ChildItem "$installation\VC\Tools\MSVC\14.4*\lib\x64\msvcrt.lib" -ErrorAction Stop + - name: Set up uv uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 with: diff --git a/ci-matrix.py b/ci-matrix.py index c90744f59..71f84d7eb 100644 --- a/ci-matrix.py +++ b/ci-matrix.py @@ -411,10 +411,11 @@ def add_python_build_entries_for_config( python_versions = config["python_versions"] build_options = config["build_options"] arch = config["arch"] - runner = find_runner(runners, platform, arch, False) + host_arch = "aarch64" if platform == "windows" and arch == "arm64ec" else arch + runner = find_runner(runners, platform, host_arch, False) runner_arch = runners[runner]["arch"] # 64-bit Windows runners can also execute 32-bit Windows builds. - can_run = runner_arch == arch or ( + can_run = runner_arch == host_arch or ( platform == "windows" and runner_arch == "x86_64" and arch == "x86" ) diff --git a/ci-targets.yaml b/ci-targets.yaml index 83f430416..acf35e080 100644 --- a/ci-targets.yaml +++ b/ci-targets.yaml @@ -422,6 +422,27 @@ windows: - freethreaded+pgo minimum-python-version: "3.13" + arm64ec-pc-windows-msvc: + arch: arm64ec + python_versions: + - "3.11" + - "3.12" + - "3.13" + - "3.14" + - "3.15" + - "3.16" + vs_version: "2026" + platform_toolset: "v143" + vc_tools_version: "14.4" + build_options: + - debug + - pgo + build_options_conditional: + - options: + - freethreaded+debug + - freethreaded+pgo + minimum-python-version: "3.13" + aarch64-pc-windows-msvc: arch: aarch64 python_versions: diff --git a/cpython-windows/build.py b/cpython-windows/build.py index a87d9731a..02020f888 100644 --- a/cpython-windows/build.py +++ b/cpython-windows/build.py @@ -47,6 +47,7 @@ "i686-pc-windows-msvc": "x86", "x86_64-pc-windows-msvc": "amd64", "aarch64-pc-windows-msvc": "arm64", + "arm64ec-pc-windows-msvc": "arm64ec", } LOG_PREFIX = [None] @@ -734,6 +735,98 @@ def hack_project_files( pass +def prepare_arm64ec(cpython_source_path: pathlib.Path) -> None: + """Add ARM64EC configurations while retaining the x64 extension ABI.""" + pcbuild_path = cpython_source_path / "PCbuild" + + configurations = re.compile( + rb' .*?', + re.DOTALL, + ) + for project in pcbuild_path.glob("*.vcxproj"): + data = project.read_bytes() + newline = b"\r\n" if b"\r\n" in data else b"\n" + data = configurations.sub( + lambda match, newline=newline: ( + match[0] + newline + match[0].replace(b"ARM64", b"ARM64EC") + ), + data, + ) + # The stable-ABI DLLs export only forwarders and can be built for x64. + data = re.sub( + rb'(]*>)', + rb"\1\n Platform=x64", + data, + ) + project.write_bytes(data) + + # Use the x64 layout, dependency bundles, and extension suffixes. + static_replace_in_file( + pcbuild_path / "python.props", + b"and $(Platform) == 'x64'\">amd64", + b"and ($(Platform) == 'x64' or $(Platform) == 'ARM64EC')\">amd64", + ) + static_replace_in_file( + pcbuild_path / "pyproject.props", + b"MachineARM64", + b"MachineARM64\n" + b" MachineARM64EC", + ) + # Use the ARM64 redistributable for its ARM64X `vcruntime140.dll`. + static_replace_in_file( + pcbuild_path / "pyproject.props", + b"$(VCRedistDir)$(Platform)\\", + b"$(VCRedistDir)$(Platform)\\\n" + b" $(VCRedistDir)arm64\\", + ) + + # Use portable decimal arithmetic instead of x64 assembly. + decimal = pcbuild_path / "_decimal.vcxproj" + data = decimal.read_bytes() + for condition in (b"'$(Platform)'=='ARM64'", b"'$(Platform)' == 'ARM64'"): + data = data.replace( + condition, b"(" + condition + b" or '$(Platform)' == 'ARM64EC')" + ) + decimal.write_bytes(data) + + # Use the PGO runtime from the `arm64` directory for ARM64EC. + static_replace_in_file( + pcbuild_path / "python.vcxproj", + b"$(Platform) == 'ARM64'", + b"($(Platform) == 'ARM64' or $(Platform) == 'ARM64EC')", + ) + + projects = pcbuild_path / "pcbuild.proj" + data = re.sub( + rb'', + rb'x64', + projects.read_bytes(), + ) + projects.write_bytes(data) + + # ARM64EC defines _M_X64 for ABI compatibility, but needs ARM memory barriers. + for relative_path in ( + "Include/internal/pycore_atomic.h", + "Include/cpython/pyatomic_msc.h", + "Include/internal/mimalloc/mimalloc/atomic.h", + ): + path = cpython_source_path / relative_path + if not path.exists(): + continue + data = path.read_bytes() + data = data.replace( + b"defined(_M_X64)", b"(defined(_M_X64) && !defined(_M_ARM64EC))" + ).replace(b"defined(_M_ARM64)", b"(defined(_M_ARM64) || defined(_M_ARM64EC))") + path.write_bytes(data) + + # The Windows SDK defines `uint128_t` for ARM64EC, conflicting with HACL's definition. + for path in (cpython_source_path / "Modules" / "_hacl" / "include").rglob("*.h"): + data = path.read_bytes() + patched = re.sub(rb"\buint128_t\b", b"Hacl_uint128_t", data) + if patched != data: + path.write_bytes(patched) + + def run_msbuild( msbuild: pathlib.Path, pcbuild_path: pathlib.Path, @@ -1483,6 +1576,9 @@ def build_cpython( elif arch == "arm64": build_platform = "arm64" build_directory = "arm64" + elif arch == "arm64ec": + build_platform = "ARM64EC" + build_directory = "amd64" else: raise Exception(f"unhandled architecture: {arch}") @@ -1540,7 +1636,7 @@ def build_cpython( # We need all the OpenSSL library files in the same directory to appease # install rules. - openssl_arch = {"amd64": "amd64", "x86": "win32", "arm64": "arm64"}[arch] + openssl_arch = build_directory openssl_root = td / "openssl" / openssl_arch openssl_bin_path = openssl_root / "bin" openssl_lib_path = openssl_root / "lib" @@ -1605,6 +1701,8 @@ def build_cpython( zlib_entry=zlib_entry, tk_bin_entry=tk_bin_entry, ) + if arch == "arm64ec": + prepare_arm64ec(cpython_source_path) if pgo: run_msbuild( @@ -1646,7 +1744,7 @@ def build_cpython( [str(instrumented_python), "-m", "test", "--list-tests"], cwd=cpython_source_path, env=env, - check=False, + check=True, stdout=subprocess.PIPE, ).stdout @@ -2044,6 +2142,8 @@ def main() -> None: build_options = args.options target_triple = args.target_triple or default_target_triple() arch = TARGET_ARCHITECTURES[target_triple] + # OpenSSL, libffi, and Tcl/Tk use the x64 ABI in ARM64EC distributions. + dependency_arch = "amd64" if arch == "arm64ec" else arch platform_toolset = args.platform_toolset or ( "v145" if arch == "amd64" @@ -2061,7 +2161,9 @@ def main() -> None: log(f"building for {target_triple}") log(f"using {platform_toolset} with MSVC {vc_tools_version}") - build_env = get_visual_studio_environment(args.vs, arch, vc_tools_version) + build_env = get_visual_studio_environment( + args.vs, dependency_arch, vc_tools_version + ) os.environ.update(build_env) compiler_cache_key = f"{platform_toolset}-{vc_tools_version}" @@ -2084,7 +2186,7 @@ def main() -> None: build_openssl( openssl_entry, perl_path, - arch, + dependency_arch, dest_archive=openssl_archive, with_uplink=openssl_with_uplink, ) @@ -2095,7 +2197,7 @@ def main() -> None: if not libffi_archive.exists(): build_libffi( args.python, - arch, + dependency_arch, pathlib.Path(args.sh), args.vs, vc_tools_version, diff --git a/cpython-windows/xz-support/config.h b/cpython-windows/xz-support/config.h index 81ddf6b7f..78fd8d182 100644 --- a/cpython-windows/xz-support/config.h +++ b/cpython-windows/xz-support/config.h @@ -31,7 +31,7 @@ This is used by CPython, and is not part of the regular xz release. #define HAVE_ENCODER_SPARC 1 #define HAVE_ENCODER_X86 1 -#if defined(_M_ARM64) +#if defined(_M_ARM64) || defined(_M_ARM64EC) #undef HAVE_IMMINTRIN_H #undef HAVE_USABLE_CLMUL diff --git a/docs/building.md b/docs/building.md index be8fc8c77..1901d68b4 100644 --- a/docs/building.md +++ b/docs/building.md @@ -174,3 +174,16 @@ so a debug distribution only runs on a machine with Visual Studio installed. To build a 32-bit x86 binary, simply use an `x86 Native Tools Command Prompt` instead of `x64`. + +To build ARM64EC, use a Windows 11 on ARM host with the Visual Studio +ARM64/ARM64EC and x64 build tools installed: + +``` +$ uv run --no-dev build.py --target-triple arm64ec-pc-windows-msvc --sh c:\cygwin\bin\sh.exe +``` + +The build runs the generated interpreter to assemble the distribution, so +an x64 Windows host cannot build this target. CPython and its bundled +extensions compile as ARM64EC. The stable-ABI forwarding DLLs and the bundled +OpenSSL, libffi, and Tcl/Tk libraries are built for x64. ARM64EC distributions use the +x64-compatible extension ABI and `win_amd64` wheel platform tag. diff --git a/docs/building.rst b/docs/building.rst index ed5e8f5fd..e8f429f9e 100644 --- a/docs/building.rst +++ b/docs/building.rst @@ -161,3 +161,14 @@ so a debug distribution only runs on a machine with Visual Studio installed. To build a 32-bit x86 binary, simply use an ``x86 Native Tools Command Prompt`` instead of ``x64``. + +To build ARM64EC, use a Windows 11 on ARM host with the Visual Studio +ARM64/ARM64EC and x64 build tools installed:: + + $ uv run --no-dev build.py --target-triple arm64ec-pc-windows-msvc --sh c:\cygwin\bin\sh.exe + +The build runs the generated interpreter to assemble the distribution, so +an x64 Windows host cannot build this target. CPython and its bundled +extensions compile as ARM64EC. The stable-ABI forwarding DLLs and the bundled +OpenSSL, libffi, and Tcl/Tk libraries are built for x64. ARM64EC distributions use the +x64-compatible extension ABI and ``win_amd64`` wheel platform tag. diff --git a/docs/running.md b/docs/running.md index e8717c781..4ae9d5fa1 100644 --- a/docs/running.md +++ b/docs/running.md @@ -77,6 +77,12 @@ Distributions are produced for the following target triples: : Windows 64-bit ARM CPUs. Available for CPython 3.11 and newer. +`arm64ec-pc-windows-msvc` + +: Windows 11 on ARM, with support for x64 extension modules and + `win_amd64` wheels. Available for CPython 3.11 and newer. These builds + cannot run on x64 Windows or load native ARM64 extension modules. + `x86_64-unknown-linux-gnu` : Linux 64-bit Intel/AMD CPUs linking against GNU libc. diff --git a/docs/running.rst b/docs/running.rst index 897dd7964..b15dc1716 100644 --- a/docs/running.rst +++ b/docs/running.rst @@ -72,6 +72,11 @@ Distributions are produced for the following target triples: ``aarch64-pc-windows-msvc`` Windows 64-bit ARM CPUs. Available for CPython 3.11 and newer. +``arm64ec-pc-windows-msvc`` + Windows 11 on ARM, with support for x64 extension modules and + ``win_amd64`` wheels. Available for CPython 3.11 and newer. These builds + cannot run on x64 Windows or load native ARM64 extension modules. + ``x86_64-unknown-linux-gnu`` Linux 64-bit Intel/AMD CPUs linking against GNU libc. diff --git a/pythonbuild/disttests/__init__.py b/pythonbuild/disttests/__init__.py index d6ad302b8..ab5d670b3 100644 --- a/pythonbuild/disttests/__init__.py +++ b/pythonbuild/disttests/__init__.py @@ -129,6 +129,32 @@ def test_ctypes(self): def error_handler(fif, message): pass + @unittest.skipUnless( + os.environ["TARGET_TRIPLE"] == "arm64ec-pc-windows-msvc", + "Requires an ARM64EC distribution", + ) + def test_arm64ec_x64_abi(self): + import ctypes + import sysconfig + + self.assertEqual(sysconfig.get_platform(), "win-amd64") + + @ctypes.CFUNCTYPE(ctypes.c_double, ctypes.c_int, ctypes.c_double) + def callback(integer, floating): + return integer + floating + + self.assertEqual(callback(40, 2.5), 42.5) + + build_options = os.environ["BUILD_OPTIONS"].split("+") + suffix = "t" if "freethreaded" in build_options else "" + if "debug" in build_options: + suffix += "_d" + stable_abi = ctypes.PyDLL( + str(Path(sys.executable).parent / f"python3{suffix}.dll") + ) + stable_abi.Py_GetVersion.restype = ctypes.c_char_p + self.assertEqual(stable_abi.Py_GetVersion().decode("utf-8"), sys.version) + @unittest.skipIf(os.name == "nt", "curses not available on Windows") def test_curses_import(self): import curses diff --git a/pythonbuild/testdist.py b/pythonbuild/testdist.py index 875f00bb2..1e7cfd6f4 100644 --- a/pythonbuild/testdist.py +++ b/pythonbuild/testdist.py @@ -12,6 +12,7 @@ import shlex import subprocess import tempfile +import time from pathlib import Path from typing import Optional @@ -651,4 +652,13 @@ def main(pbs_source_dir: Path, raw_args: list[str]) -> int: finally: if td: - td.cleanup() + # DLLs can remain locked briefly after the test process exits on Windows. + for attempt in range(11): + try: + td.cleanup() + except PermissionError: + if os.name != "nt" or attempt == 10: + raise + time.sleep(1) + else: + break diff --git a/src/release.rs b/src/release.rs index e92c73b65..c691963ba 100644 --- a/src/release.rs +++ b/src/release.rs @@ -140,6 +140,19 @@ pub static RELEASE_TRIPLES: Lazy> = Lazy:: }], }, ); + h.insert( + "arm64ec-pc-windows-msvc", + TripleRelease { + suffixes: windows_suffixes.clone(), + install_only_suffix: "pgo", + freethreaded_install_only_suffix: "freethreaded+pgo", + python_version_requirement: Some(VersionSpecifier::from_str(">=3.11").unwrap()), + conditional_suffixes: vec![ConditionalSuffixes { + python_version_requirement: VersionSpecifier::from_str(">=3.13").unwrap(), + suffixes: windows_suffixes_313.clone(), + }], + }, + ); h.insert( "aarch64-pc-windows-msvc", TripleRelease { diff --git a/src/validation.rs b/src/validation.rs index fa30a1d3e..fe919927d 100644 --- a/src/validation.rs +++ b/src/validation.rs @@ -37,6 +37,7 @@ const RECOGNIZED_TRIPLES: &[&str] = &[ "aarch64-pc-windows-msvc", "aarch64-unknown-linux-gnu", "aarch64-unknown-linux-musl", + "arm64ec-pc-windows-msvc", "armv7-unknown-linux-gnueabi", "armv7-unknown-linux-gnueabihf", "i686-pc-windows-msvc", @@ -594,6 +595,7 @@ static PLATFORM_TAG_BY_TRIPLE: Lazy> = Lazy: [ ("aarch64-apple-darwin", "macosx-11.0-arm64"), ("aarch64-pc-windows-msvc", "win-arm64"), + ("arm64ec-pc-windows-msvc", "win-amd64"), ("aarch64-unknown-linux-gnu", "linux-aarch64"), ("aarch64-unknown-linux-musl", "linux-aarch64"), ("armv7-unknown-linux-gnueabi", "linux-arm"), diff --git a/tests/test_windows.py b/tests/test_windows.py new file mode 100644 index 000000000..fe1ae8bdd --- /dev/null +++ b/tests/test_windows.py @@ -0,0 +1,106 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +from __future__ import annotations + +import importlib.util +import tempfile +import unittest +from contextlib import ExitStack +from pathlib import Path +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parent.parent +SPEC = importlib.util.spec_from_file_location( + "windows_build", ROOT / "cpython-windows" / "build.py" +) +assert SPEC is not None and SPEC.loader is not None +windows_build = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(windows_build) + + +class WindowsBuildTests(unittest.TestCase): + def test_dependency_architecture(self) -> None: + for target, arch, dependency_arch in ( + ("i686", "x86", "x86"), + ("x86_64", "amd64", "amd64"), + ("aarch64", "arm64", "arm64"), + ("arm64ec", "arm64ec", "amd64"), + ): + with ( + self.subTest(target=target), + tempfile.TemporaryDirectory() as directory, + ): + root = Path(directory) + triple = f"{target}-pc-windows-msvc" + with ExitStack() as stack: + stack.enter_context(patch.object(windows_build, "BUILD", root)) + stack.enter_context( + patch.object(windows_build, "DIST", root / "dist") + ) + stack.enter_context( + patch.object( + windows_build.sys, + "argv", + [ + "build.py", + "--target-triple", + triple, + "--python", + "cpython-3.14", + "--sh", + "sh.exe", + ], + ) + ) + mocks = { + name: stack.enter_context(patch.object(windows_build, name)) + for name in ( + "get_visual_studio_environment", + "fetch_strawberry_perl", + "build_openssl", + "build_libffi", + "build_cpython", + "compress_python_archive", + "release_tag_from_git", + "log", + ) + } + mocks["get_visual_studio_environment"].return_value = {} + mocks["fetch_strawberry_perl"].return_value = root + mocks["build_cpython"].return_value = root / "python.tar" + windows_build.main() + + mocks["get_visual_studio_environment"].assert_called_once_with( + "2022", dependency_arch, "14.4" + ) + self.assertEqual( + mocks["build_openssl"].call_args.args[2], dependency_arch + ) + self.assertEqual( + mocks["build_libffi"].call_args.args[1], dependency_arch + ) + self.assertEqual( + mocks["build_cpython"].call_args.args[1:3], (triple, arch) + ) + + def test_arm64ec_does_not_enable_x64_tail_call_interpreter(self) -> None: + with patch.object(windows_build, "exec_and_log") as run: + windows_build.run_msbuild( + Path("MSBuild.exe"), + Path("PCbuild"), + "Release", + "ARM64EC", + "3.15.1", + "10.0.26100.0", + "v143", + False, + ) + args = run.call_args.args[0] + self.assertIn("/property:Platform=ARM64EC", args) + self.assertNotIn("/property:UseTailCallInterp=true", args) + + +if __name__ == "__main__": + unittest.main()