diff --git a/.gitignore b/.gitignore index 62de6f1bf9fbf..994ed9199e525 100644 --- a/.gitignore +++ b/.gitignore @@ -215,3 +215,8 @@ BenchmarkDotNet.Artifacts/ # Clangd uses these common ephemeral files .cache compile_commands.json + +# Bee build outputs +/artifacts/ +/builds/ +/build.gen.sln diff --git a/.yamato/build-linux-x64.yml b/.yamato/build-linux-x64.yml index 11ba71fd9e68b..32c276c6b29e4 100644 --- a/.yamato/build-linux-x64.yml +++ b/.yamato/build-linux-x64.yml @@ -2,7 +2,7 @@ name: Build protobuf - CentOS 7 x64 with Clang9 agent: type: Unity::VM - image: platform-foundation/linux-bokken:latest + image: build-system/unity-ubuntu-22.04:v2.9071485 flavor: b1.small artifacts: @@ -13,4 +13,4 @@ artifacts: # After building, recursively copy .h and .inc files from the source to the destination, then rename the folder. # This is a superset of what make does, but only a couple of extra test files are included. commands: -- bash ./bee/bee linux_x64_clang +- bash ./bee/bee linux_x64 diff --git a/.yamato/build-windows-all.yml b/.yamato/build-windows-all.yml index f0ea37418ba3d..58ef23ae70304 100644 --- a/.yamato/build-windows-all.yml +++ b/.yamato/build-windows-all.yml @@ -1,5 +1,5 @@ windows-c-libraries: - name: Build protobuf - Windows 10 x64 with VS 2017 + name: Build protobuf - Windows x64 and arm64 with VS 2026 agent: type: Unity::VM @@ -12,8 +12,8 @@ windows-c-libraries: - builds/** commands: - - .\bee\bee win_x64_vs2017_win7 - - .\bee\bee win_arm64_vs2019_win7 + - .\bee\bee win_x64 + - .\bee\bee win_arm64 csharp-bindings: name: Build C# Bindings diff --git a/Build.bee.cs b/Build.bee.cs index a9f796b583077..5376f93d65675 100644 --- a/Build.bee.cs +++ b/Build.bee.cs @@ -121,7 +121,7 @@ static NativeProgram SetupLibProtobuf(IEnumerable toolchains) var nativeProgramConfiguration = new NativeProgramConfiguration(codegen, toolchain, lump: true); var format = toolchain.StaticLibraryFormat; var deployedProgram = np.SetupSpecificConfiguration(nativeProgramConfiguration, format).DeployTo(GetBuildTargetDir(toolchain)); - Backend.Current.AddAliasDependency(toolchain.ActionName, deployedProgram.Path); + Backend.Current.AddAliasDependency(AliasFor(toolchain), deployedProgram.Path); if (toolchain.Platform is LinuxPlatform) { @@ -232,44 +232,64 @@ static NativeProgram SetupProtoc(IEnumerable toolchains, NativeProgra np.CompilerSettingsForGccLike().Add(s => s.WithGlibcxxUseCxx11Abi(true)); np.Libraries.Add(libProtobuf); - foreach (var toolchain in toolchains) + // On Windows protoc is built with the dynamic CRT (/MD) only + foreach (var toolchain in toolchains.Where(t => !IsStaticCrtWindows(t))) { var nativeProgramConfiguration = new NativeProgramConfiguration(CodeGen.Release, toolchain, lump: true); var format = toolchain.ExecutableFormat; var deployedProgram = np.SetupSpecificConfiguration(nativeProgramConfiguration, format).DeployTo(GetBuildTargetDir(toolchain, true)); - Backend.Current.AddAliasDependency(toolchain.ActionName, deployedProgram.Path); + Backend.Current.AddAliasDependency(AliasFor(toolchain), deployedProgram.Path); } return np; } + static bool IsStaticCrtWindows(ToolChain toolchain) => + toolchain is VisualStudioToolchain vs && vs.CompilerRuntimeLibrary == CompilerRuntimeLibrary.Static; + + static string AliasFor(ToolChain toolchain) + { + var platform = toolchain.Platform switch + { + WindowsPlatform => "win", + LinuxPlatform => "linux", + MacOSXPlatform => "mac", + _ => throw new ArgumentException($"Don't know alias for toolchain {toolchain.ActionName}") + }; + return $"{platform}_{toolchain.Architecture.DisplayName}"; + } + static NPath GetBuildTargetDir(ToolChain toolchain, bool isExecutable = false) { var mapping = new Dictionary { {"mac_x64", "macos/x64"}, {"mac_arm64", "macos/arm64"}, - {"linux_x64_clang", "linux"}, - {"win_x64_vs2017_win7", "win/x64"}, - {"win_arm64_vs2019_win7", "win/arm64"}, + {"linux_x64", "linux"}, + {"win_x64", "win/x64"}, + {"win_arm64", "win/arm64"}, }; - var name = toolchain.ActionName.ToLowerInvariant(); + var name = AliasFor(toolchain); if (!mapping.ContainsKey(name)) throw new ArgumentException($"Don't know output path for toolchain {name}"); + // Windows libraries built with the dynamic CRT (/MD, /MDd in Debug) go in an MD subfolder, for linking into a /MD Unity Editor var typePath = isExecutable ? "bin" : "lib"; - return $"builds/{typePath}/{mapping[name]}"; + var crtPath = !isExecutable && toolchain is VisualStudioToolchain { CompilerRuntimeLibrary: CompilerRuntimeLibrary.Dynamic } ? "/MD" : ""; + return $"builds/{typePath}/{mapping[name]}{crtPath}"; } static IEnumerable GetToolchains() { var toolchains = new ToolChain[] { - ToolChain.Store.Windows().VS2017().Sdk_17134().x64(), - ToolChain.Store.Windows().VS2019().Sdk_20348().Arm64(), - ToolChain.Store.Linux().Centos_7_7_1908().Clang_9_0_1().x64(), + ToolChain.Store.Windows().VS2026().Sdk_20348().x64(TargetWindowsVersion.Windows7, CompilerRuntimeLibrary.Static), + ToolChain.Store.Windows().VS2026().Sdk_20348().x64(TargetWindowsVersion.Windows7, CompilerRuntimeLibrary.Dynamic), + ToolChain.Store.Windows().VS2026().Sdk_20348().Arm64(TargetWindowsVersion.Windows7, CompilerRuntimeLibrary.Static), + ToolChain.Store.Windows().VS2026().Sdk_20348().Arm64(TargetWindowsVersion.Windows7, CompilerRuntimeLibrary.Dynamic), + ToolChain.Store.Linux().GLibC_2_35().LLVM_15_0().x64(), ToolChain.Store.Mac().Sdk_11_0().x64(), ToolChain.Store.Mac().Sdk_11_0().ARM64(), }.Where(t => t != null).ToArray(); diff --git a/bee/bee b/bee/bee index edc4cc0d561c4..2636856bbe66e 100644 --- a/bee/bee +++ b/bee/bee @@ -1,13 +1,24 @@ #!/bin/bash -#Bee bootstrap script. You typically invoke this script directly. It's responsible for -#downloading a dotnet runtime if required, and for downloading a bee distribution if required, and for starting the actual bee +# Bee bootstrap script. You typically invoke this script directly. It's responsible for +# downloading a dotnet runtime if required, and for downloading a bee distribution if +# required, and for starting the actual bee. #driver executable. set -euo pipefail -#these two values will be replaced by the buildprocess when this script is used in a bootstrap distribution. When not replaced -#this script assumes it lives next to the standalone driver -use_bee_from_steve="bee/4614ca8dfa47_e30d39dc9e8f44f17d5d0e36aed8224ec76a26ca0702ca4a2bc2679504ef7520.zip" +# Ensure some environment variables that can break the build are not set. +unset MSBuildSDKsPath # See BUILD-1219 + +# Require unzip. +if ! command -v unzip >/dev/null 2>&1; then + echo "Bee: unzip utility required but not found; for Ubuntu/WSL try 'sudo apt-get install unzip'" + exit 1 +fi + +# These two values will be replaced by the buildprocess when this script is used in a +# bootstrap distribution. When not replaced this script assumes it lives next to the +# standalone driver. +use_bee_from_steve="bee/b649be378334_1a7f70b95e6ff016b5a75212ef827687532a7698eb826ea856f49f4162fe95ed.zip" use_bee_from_steve_repo="https://public-stevedore.unity3d.com/r/public" if [[ "$use_bee_from_steve_repo" =~ .*testing.* ]]; then @@ -15,24 +26,41 @@ if [[ "$use_bee_from_steve_repo" =~ .*testing.* ]]; then fi if [[ "$OSTYPE" == "msys" ]]; then - echo "It looks like you're running the bee bash script on windows. This is not supported, run bee.bat or bee.ps1 instead" + echo "It looks like you're running the bee bash script on windows. This is not supported, run bee.bat instead" exit 1 fi -#stevedore artifacts for the dotnet runtimes. these are produced by the yamato CI for our "netcorerun" repo. The zip file contains -#an info file that describes from which git commit / yamato-id it was built. They are plain upstream packages, just unzipped & rezipped -#with this extra information added. -dotnet_runtime_linux="dotnet-runtime-linux-x64/a057d2d_456b6c52196a7058883b449c3c14035e93ad94047a136c89610fb53734095602.zip" #net-runtime 6.0.8 -dotnet_runtime_osx_x64="dotnet-runtime-osx-x64/a057d2d_112720e179fb36f93fdfd8fb77cbab0c24c6a6b61881e44d3c06e97904f69b73.zip" #net-runtime 6.0.8 -dotnet_runtime_osx_arm64="dotnet-runtime-osx-arm64/a057d2d_e4594857f1f7201a5038d6e278698a6f207d53428d4efe57365640b03e32b92d.zip" #net-runtime 6.0.8 +# Stevedore artifacts for the dotnet runtimes & SDKs. these are produced by the +# yamato CI UploadDotNetSdk. The zip file contains an info file that +# describes from which git commit / yamato-id it was built. They are plain +# upstream packages, just unzipped & rezipped with this extra information added. +dotnet_runtime_linux="dotnet-runtime-linux-x64/10.0.11-9854b7a_4e49c5512da4149987b14302291385a7da198278b570dcdaaea0669139dc9d3a.zip" +dotnet_runtime_osx_x64="dotnet-runtime-osx-x64/10.0.11-9854b7a_ff18c2e785cb5a27b4c1e721dd9383c1fb45cf4447c6079026b635eedb783d6c.zip" +dotnet_runtime_osx_arm64="dotnet-runtime-osx-arm64/10.0.11-9854b7a_b8bdcd10c63223ccd9d58f2714a30cb24e647e97af54af1c2d89c1243c313865.zip" +dotnet_sdk_linux="dotnet-sdk-linux-x64/10.0.303-9854b7a_5e248a0b4ee4147d4c1af05dc851fe93af8b08d0b0c0b1dddadea810e4fbf32b.7z" +dotnet_sdk_osx_x64="dotnet-sdk-osx-x64/10.0.303-9854b7a_29387043037d7a02fe59cb9c39b57bad96401e47aa6a0a36c053bdfd2bbaeb7d.7z" +dotnet_sdk_osx_arm64="dotnet-sdk-osx-arm64/10.0.303-9854b7a_178eab56d8dcd34270d820c985e78c3641b30498578c14744f7f09d75ef03ca9.7z" steve_artifact_return_value="" +uname=$(uname) +case ${uname} in + Darwin) + hashCmd="shasum -a 256" + ;; + Linux) + hashCmd=sha256sum + ;; + *) + echo "Error: unknown system: \"${uname}\""; + exit 1 + ;; +esac + download_or_return_steve_artifact() { local steve_name="$1" - #we could extend this to parse the Stevedore.conf, and to attempt downloads from multiple mirrors local steve_repo_url="$2" local unzip_dir_path="$HOME/.beebootstrap/$steve_name" @@ -52,39 +80,74 @@ download_or_return_steve_artifact() mkdir -p "$temporary_dir" downloaded_file="$temporary_dir/download.zip" - #--fail means that if the http result is not OK, curl will actually fail. if not it will write a beautiful 404 page into $downloaded_file and exit with code 0 - curl --fail --location "$download_link" --output "$downloaded_file" + dl_filename=$(basename "$download_link") + exp_sha256=$(echo "$dl_filename" | sed -e "s/.*_//" -e "s/\..*//") + + # A couple of short retries first, then back off to ~30s intervals; the + # last retry lands a bit past the 3 minute mark before giving up. + dl_success=0 + for sleepy_time in 0 2 2 30 30 30 30 30 30 30 + do + sleep $sleepy_time + + # --fail means that if the http result is not OK, curl will actually fail. + # If not it will write a beautiful 404 page into $downloaded_file and exit + # with code 0. + if curl --fail --location "$download_link" --output "$downloaded_file" + then + actual_sha256=$($hashCmd "$downloaded_file" | cut -d' ' -f1) + if [ "$actual_sha256" = "$exp_sha256" ] + then + dl_success=1 + break + fi + + echo "$downloaded_file sha256 mismatch (expected: $exp_sha256 actual: $actual_sha256), removing" + rm -f "$downloaded_file" + fi + done + + if [ "$dl_success" != 1 ] + then + echo "Error: bee bootstrap download failed" 1>&2 + exit 1 + fi - #after downloading into the temporary directory, we unzip it into the same temporary directory + # After downloading into the temporary directory, we unzip it into the same temporary directory. unzip -q -d "$temporary_dir" "$downloaded_file" - #then we delete the zip file, so only its contents remain + # then we delete the zip file, so only its contents remain. rm -f "$downloaded_file" - #we ensure the directory we need to put the final result in's parent directory exist. + # Ensure that the directory we need to put the final results in parent directory exists. mkdir -p "$(dirname "$unzip_dir_path")" - #and we move the temporary directory to the final location. We use a move, so that it's atomic, and a failed download, or a failed unzp cannot - #result in a situation where the targetdirectory did get created, but did not get properly populated. This way we can use the presence of the directory - #as indication + # And we move the temporary directory to the final location. We use a move, so that it's + # atomic, and a failed download, or a failed unzp cannot result in a situation where the + # targetdirectory did get created, but did not get properly populated. This way we can + # use the presence of the directory as indication. mv "$temporary_dir" "$unzip_dir_path" fi - #bash does not support return values, so we assign it to a global. yep. + # Bash does not support return values, so we assign it to a global. Yep. steve_artifact_return_value="$unzip_dir_path" } dotnet_runtime_steve="" +dotnet_sdk_steve="" determine_dotnet_runtime_steve() { type="$(uname)-$(uname -m)" case "$type" in Darwin-x86_64) dotnet_runtime_steve="$dotnet_runtime_osx_x64" + dotnet_sdk_steve="$dotnet_sdk_osx_x64" ;; Darwin-arm64) dotnet_runtime_steve="$dotnet_runtime_osx_arm64" + dotnet_sdk_steve="$dotnet_sdk_osx_arm64" ;; Linux-x86_64) dotnet_runtime_steve="$dotnet_runtime_linux" + dotnet_sdk_steve="$dotnet_sdk_linux" ;; *) echo "Unsupported system: $type" @@ -94,12 +157,16 @@ determine_dotnet_runtime_steve() { } if [ "$use_bee_from_steve" == "no" ]; then - #this script supports running as part of a full bee distribution. In this case, use_bee_from_steve is not set, - #and we find the path to the distribution by looking at where the script itself is. It shuold be placed in Standalone/Release of the distribution + # This script supports running as part of a full bee distribution. In this case, + # use_bee_from_steve is not set, and we find the path to the distribution by + # looking at where the script itself is. It shuold be placed in Standalone/Release + # of the distribution. standalone_release="$(dirname "$0")" else - #we also support downloading the bee distribution from a stevedore server. In this case use_bee_from_steve should be set to a stevedore artifact name. - #we'll download it and run it. In this mode, the only thing a user needs to version in their repo is this script. + # We also support downloading the bee distribution from a stevedore server. In + # this case use_bee_from_steve should be set to a stevedore artifact name. + # We'll download it and run it. In this mode, the only thing a user needs to + # version in their repo is this script. download_or_return_steve_artifact "$use_bee_from_steve" "$use_bee_from_steve_repo" "Bee distribution" standalone_release="$steve_artifact_return_value/Standalone/Release" fi @@ -110,232 +177,54 @@ if [ ! -e "$standalone_path" ]; then standalone_path="$standalone_release/Bee.StandaloneDriver.dll" fi - determine_dotnet_runtime_steve -download_or_return_steve_artifact "$dotnet_runtime_steve" "https://public-stevedore.unity3d.com/r/public" "Dotnet runtime" -dotnet_exe="$steve_artifact_return_value/dotnet" -#we assign BEE_DOTNET_MUXER env var here, so that the bee that is running knows how it can use this dotnet runtime to start other net5 framework dependent apps. -#it uses this to run the stevedore downloader program on net5. +if [ "${BEE_DOTNET_MUXER-}" != "" ]; then # If we are recursing, then use the dotnet.exe from outside. + dotnet_exe=$BEE_DOTNET_MUXER +else + download_or_return_steve_artifact "$dotnet_runtime_steve" "https://public-stevedore.unity3d.com/r/public" "Dotnet runtime" + dotnet_exe="$steve_artifact_return_value/dotnet" +fi + +# We assign BEE_DOTNET_MUXER env var here, so that the bee that is running knows how +# it can use this dotnet runtime to start other dotnet framework dependent apps. +# It uses this to run the stevedore downloader program. export BEE_DOTNET_MUXER="$dotnet_exe" export BEE_DISTRIBUTION_PATH="$distribution_path" +export DOTNET_MULTILEVEL_LOOKUP=0 + +if [ "${1:-default}" == "dotnet" ]; then + + if [ "${BEE_DOTNET_SDK_UNZIP_DIR_PATH-}" != "" ]; then # if there is a dotnet_sdk from outside + # Use the dotnet sdk from outside. + bee_dotnet_sdk_unzip_dir_path=$BEE_DOTNET_SDK_UNZIP_DIR_PATH + else + # Use bee steve to download and unpack the dotnet SDK + bee_dotnet_sdk_unzip_dir_path="$HOME/.beebootstrap/$dotnet_sdk_steve" + "$dotnet_exe" "$standalone_path" "steve" "internal-unpack" "public" "$dotnet_sdk_steve" "$bee_dotnet_sdk_unzip_dir_path" + + # Use the dotnet sdk we just downloaded. + export BEE_DOTNET_SDK_UNZIP_DIR_PATH=$bee_dotnet_sdk_unzip_dir_path + fi + + # Set the path so this dotnet SDK can be used by child processes. + PATH="$bee_dotnet_sdk_unzip_dir_path:$PATH" + + # DOTNET_ROOT lets child processes locate the .NET runtime directly. Without it, + # spawned processes (e.g. vstest test hosts) cannot find hostfxr and fail to start. + export DOTNET_ROOT=$bee_dotnet_sdk_unzip_dir_path + + # Invoke dotnet with the User's args. + shift 1; + dotnet_sdk_exe="$bee_dotnet_sdk_unzip_dir_path/dotnet" + $dotnet_sdk_exe "$@" + +else + + # Invoke bee. + "$dotnet_exe" "$standalone_path" "$@" +fi + -"$dotnet_exe" "$standalone_path" "$@" -# ReleaseNotes: - - -# Automatically generated by Yamato Job: https://unity-ci.cds.internal.unity3d.com/job/20992285 - -# ### Bee 2021 release notes -# -# ## Changes Requiring Changes To BuildCode -# * The type used for GUIDs in all CSharpProgram and ProjectFile members that handle them (`CSharpProgram.ExpilcitGuid`, `CSharpProgramReference` constructor, `IProjectFile.Guid`) was changed from string to System.GUID, to avoid bugs where you accidentally put curly braces in the GUID when you shouldn't, or the other way around. -# * Completely new Pram binding ([cross platform application deployment tooling](https://github.cds.internal.unity3d.com/unity/pram/#platform--runtime-application-manager-pram)) which is compatible with latest Pram version. The binding is located at `Bee.PramBinding.PramExecutable` and allows to set up application deploy and run actions. Note that Pram no longer handles application bundling and expects platform specific application formats. -# * StandaloneBeeDriver.BuildProgramProjectFile has been replaced by BuildProgramProjectFileFor(BuildProgramContext bpc). You can create a buildprogram context by having the very first thing you do in your buildprogram be: var context = new BuildProgramContext() -# * `EmscriptenExecutableFormat` extension defaults now to `html` -# * Buildprograms will by default be compiled using CSharpProgram2, targetting net5, and ran on the net5 runtime. This means you can use C# 9 in your buildprograms. In order to conveniently edit your buildprogram with Rider or VisualStudio you need to have the dotnet5 sdk installed. There is no requirement for any install if you just want to build. Transition should be smooth and automatic. Use UseCSharpSupport2=0 and RunBeeOnNet5=0 in bee.config to revert to the old behaviour. Inform #devs-bee if you have to do this, as the old behaviour will soon be removed completely. -# * The type `SerializableInvocation` is no longer nested inside StandaloneBeeDriver, but is now a standalone type in the `Bee` namespace. -# * If you have a normal bee deployment, and your buildprogram uses ConfigureCSharpAction(), you can remove that code, as the BuildProgramContext() constructor sets up csharp actions -# by default for everyone now. -# * `IDeployable.DeployTo` with (optional) `alreadyDeployed` is now deprecated, use new version with (optional) `destinationSubDirectory` parameter -# * `DeployableFile` as well as `BuiltNativeProgram` and inheritors are now `record`, meaning any derived class needs to be a `record` as well -# * Various deprecations in `BuiltNativeProgram` and inheritors, follow documentation provided by `Obsolete` attributes -# * `IDeployable.Path` is "soft-deprecated" (i.e. not yet marked with `Obsolete`), prefer depending on `IDeployable.Files` instead of relying in a single "representative file" -# * EmscriptenToolchain now only supports Emscripten2. You can get one using `ToolChain.Store.Emscripten().Sdk_2_0_19().Wasm()`. -# * Several Emscripten compiler flags were removed as they no longer exist in Emscripten2. WithSeparateAsm, PRECISE_F32. WithExtraExportedRuntimeMethods was renamed to WithExportedRuntimeMethods. -# * Fixed case 1378085 @zilys: -# Windows: when building generated IL2CPP C++ code with Visual Studio 2022, Unity will now correctly recognize it as "Visual Studio 2022" instead of "Visual Studio [Unknown]". -# * UWP executable native program no longer deploys `ucrtbased.dll` as part of its debug build -# * TinyProfiler2 ChromeTraceOptions external .traceevent files needs to be submitted as `Task` instead of `Func` -# -# ## Improvements -# * When a stevedore download fails due to a DNS failure, it will be attempted again. -# * On mac, stevedore will do unpacks using `ditto` so that we can preserve metadata like osx signing information. -# * Comparing `BagOfObjectFilesLibrary` instances is now faster. -# * Reduced lock contention in the backend, resulting in improved performance, especially when there is almost nothing to be built. -# * Make the backend flush its output buffers less frequently, for improved performance, especially when there is almost nothing to be built. -# * Add `CSharpProgram2.RestoreLogFilePaths`. This allows build code to make a `CSharpProgram2`'s restore depend on another `CSharpProgram2`, we use this in the Bee build code to avoid nuget conflicts in CI. -# * Increased the stevedore download timeout from 10s to 20s. -# * All `CSharpProgram2` instances have `WarningsAsErrors` set to true by default -# * If a node fails with non-zero exit code, its outputs (partial/incorrect results, logs, etc.) will not be deleted -# * Added `DynamicLinkerSettingsForLd` and `DynamicLinkerSettingsForXcode` native program extension methods -# * CSharpProgram2 exposes the .RollForward property to control the rollforward policy of netcore based runtimes -# * CSharpProgram2 exposes the .UseAppHost property to avoid creation of a framework dependent executable. -# * Preview feature: running bee and stevedore donwloader on an automatically downloaded .net5 runtime. opt in with `"RunBeeOnNet5" = true` in your bee.config file -# * Preview feature: bee can be consumed as 3 scripts. (bash, ps1, bat). They will download the full bee on demand. This avoid versioning the full bee distribution in your repo. -# * Bee executable supports newline seperated responsefiles -# * TundraBackend now has .AddEnvironmentVariableSignature("YOUR_VAR") that will cause the dag to be invalidated when the value of that environment variable changes in the future. -# * Add `CSharpProgram2.GenerateAssemblyInfo` to control whether an assemblyinfo file (containing attributes like AssemblyName, AssemblyVersion, etc) is automatically generated for the program or not. -# * Add `CSharpProgram2.RuntimeIdentifer` to control target platform (mostly relevant for using some Nuget packages with platform specific binaries) -# * Add `PackageReference.ReferenceSpecificPath` for situations where nuget chose a targetframework variation of a package and you're not happy with that choice. -# * Make dotnet publish command used by CSharpProgram2 be less verbose on CI -# * Custom buildprogram buildprograms have the ability to specify that their buildprogram supports providing help through `BuildProgramBuildProgramOutputData`. -# * .Net3.1 sdk is now available through `DotNetSdk.Sdk31` -# * Added infrastructure for cross platform application packaging -# * All normal bee deployments support `Backend.Current.AddCSharpAction()` out of the box. Try it, it's nice! -# * `CSharpProgram2` as well as the standalone driver will check for a user wide installed NuGet.Config that prevents any nuget operations from working. Choco unfortunately installs such a config. If one is found it's not changed or removed, but instead our dotnet restore commands use a local NuGet.Config that points to the default nuget package source. If a non-empty NuGet.Config is found system wide, we respect it and don't do anything. -# * ProjectFiles generated by CSharpProgram2 will use shorter paths in the artifacts directory for msbuild/IDE's to put their temporary files in. This makes it more likely your bee buildprogram -# runs properly on machines configured without EnableLongPaths=1 on windows machines. -# * Most `IDeployable`s (notably `BuiltNativeProgram` and `DeployableFile`) inherit now from a common base record class implementing a `DestinationSubDirectory`, allowing to specify subdirectories within `DeployTo`'s target -# * When using dynamic targetdirectories, you always had to have special code to deal with the situation where the targetDirectory you're trying to get the file listing -# from doesn't exist yet. There is a helper for this now: `mydir.FilesOrDummy("*.cs")`. It will return all cs files if the directory exists, and if it doesn't exist, will return a single cs file in that directory called `dummy.cs`. If you are using dynamic targetdirectories you probably have a helper function in your buildprogram that does exactly this that can now be removed. -# * `Add CSharpProgram.DefaultCompiler` to globally set a default csharp compiler to use for configurations that do not specify a custom one. -# * `Bee.Core.RunnableProgram` has a virtual `EnvironmentVariables` you can override for when your program needs to be run with specific env vars. -# * You can add `"BuildProgramPackageReferences": [{"Name": "Spectre.Console", "Version": "0.14.0"}]` to your `bee.config` to have your buildprogram use one or more package references. -# * BeeFiles in Bee.Config now supports wildcards -# * Emscripten toolchain constructor overload that takes a static library extension to make the migration from .bc static libs to .a easier. -# * Give a better error message when trying to run the ./bee bash script on windows. (Can happen when you use the Git Bash shell on windows) -# * Performance improvement: When a buildthread has no work, it will no eager start statting files for their timestamp for files where we know we're going to need their timestamp -# later in the build. It will also already start doing this while the dag is still being verified. -# * All bee libraries have adopted a new profiler implementation called TinyProfiler2. There are compatibility shims for the old `TinyProfiler.Section` calls that you might have in your -# buildprogram. This new implementation is much faster. It measures time based off the unix epoch, instead of from 0. If you are combining profile.json files with others, you might -# have to change the time basis of your other profiles in order for them to nicely line up. -# * Fixed bug where bee wouldn't work properly on M1 macs. -# * Added `Bee.NativeProgramSupport.CompilationDatabase` for generating LLVM-style JSON compilation databases, used by some code analysis tools. -# * CSharpProgram2 based Csc invocations are faster now that we compile them with /shared -# * `StevedoreArtifact.UnpackToUnusualLocation(NPath)` can now be used with any custom location, not just `artifacts` or `build`. -# * bee shell scripts now set an environment variable `BEE_DISTRIBUTION_PATH` that can be used to resolve the distribution in build programs. -# * Added new `Backend.AddDirectoriesToNeverScanByHash` to exclude directories from scanning input file contents by hash. This can be used a performance improvement for inputs which we don't expect to change. -# * Added `ProjectFile.TargetFrameworkMonikersOverride` to be able to use a different list of framework monikers in generated csproj than when building a CSharpProgram (allows to add .net 5.0 as a target framework on test projects targetting .net standard so that the IDE can run the tests directly) -# * Added ClangCompilerSettings.WithAbsoluteDiagnostics(bool) to allow buildprograms to choose when this should be used. Defaults to true for windows, otherwise false. -# * Added support for keeping action output files pristine. Keeping a file pristine means that if a user or another process modifies an output file of an action that action will run again next time bee is invoked. -# * TinyProfiler2 now accept .traceevents as `Task` which allows start writing report before .tracevents are available. .traceevents will be consumed in the order they are ready. -# * Dotnet that bee runs with is now available on the Backend, Bee no longer relies on DOTNET_ROOT env var outside of tests -# -# ## Bug Fixes -# * The backend will now correctly write stdout to the log for failures when failing with a signal on unix/mac systems. -# * The backend will now correctly write process status codes to the log file, instead of always writing 1 for failures. -# * The backend will no longer fail to write its database file if the folder it's supposed to write to does not exist yet. -# * A `CSharpProgram` that has an `Identifier` set will now have its LogicalName set in the generated .csproj. -# * Use 5 character temp dirs instead of 4 to avoid Windows reserved names (like COM0). -# * Upgraded reapi-cache-client with a fix for downloaded files being created with the wrong permissions. -# * `CSharpProgram2` will always emit portable pdb's, also when targetting net471 -# * `CSharpProgram2` SupportFiles now also works with MsBuild (i.e. when building/testing from Visual Studio/Rider) -# * Clang Sanitizers are now disabled for EmbeddedLinux (tests regarding them failed previously) -# * When using UseCSharpSupport2 feature, you can use StandaloneBeeDriver.BuildProgramProjectFileFor to create get a projectfile for the buildprogram. -# * Fix bug where if you used different bee's from different places in the same repo, instead of overwriting your only bee, the dag would not be regenerated. -# * Fix bug where when running on net5, bee.standalonedriver would try to download the cache client using mono instead of net5. -# * Fix memoryleak in the backend that could trigger the heap validation code to trigger when a buildgraph uses targetdirectories. -# * Many toolchains and the backend have seen fixes for environments where paths involved in the build contain unicode characters. -# * Fixed bug where all JavaCompile nodes were missing a dependency on their responsefile. -# * Fixed bug where bee.ps1 environment variable BEE_DOTNET_MUXER and BEE_DOTNET_NET5WRAPPER would persist in powershell sessions -# * Fix a windows only bug where a failure during the initial bee download will create a situation where all subsequent builds fail. -# * Fix bug where the --help system exposed to buildprograms didn't work when --help was passed as an early argument instead of a late one -# * Some bee libraries recently started shipping with version number 1.0.0.0 embedded in their metadata. Reverted back to the original 0.0.0.0 for binary compatibility -# * Fixed bug in Android platform ObjCopy function which caused it to use invalid path and fail -# * Added a warning to the bee bootstrap bash script when it detects that a curl snap package is being used. -# * Fixed bug where two CSharpPrograms being published into the same directory would give an error on the first build. -# * Fixed bug in CSharpProgram2 when referencing pre-built .NET assemblies that are generated by another node in the graph, such as a Stevedore download -# * Fix standalonedriver bug where bee library pdb's were not properly referenced, causing exceptions in buildprograms to not have linenumbers for stackframes inside bee libraries. -# * Fixed bug in CSharpProgram2 where on the first run the app returned from .SetupPublish() would not have its .DotNetAssemblies populated. -# * On Apple-M1 machines, BuildPrograms ran by the bee standalone were ran through a NET5 runtime which goes through rosetta. Now they run using a NET6 runtime which is much faster as it is native arm64. -# * On Apple-M1 machines, BuildPrograms reported X64 as the host platform, which can cause them to take incorrect decisions on what to build. -# * Certain NewtonSoft.Json usage in buildprograms could cause runtime errors (ala System.Security.Permissions). Thanks @joejo for the report. -# * EmscriptenSDK will use emcc.bat instead of invoking python manually to work around a bug in python's stdin handling that only shows up in unity player builds. -# * Fix windows bug where in certain shells (including cmd.exe), we would no longer output colors in the buildoutput -# * Fix Emscripten compilation failing on Linux due to relative EM_CONFIG variable -# * Fix Emscripten static linker not removing object files from previous build -# * Fix Emscripten dynamic linker not passing -shared for dynamic libraries -# * Fix CSharpProgram2 embedded resources having different identifiers when built from Rider/Visual Studio versus when built from Bee -# * Fix Stevedore manifest parsing failing on lines with spaces/tabs and trailing spaces/tabs -# * Stevedore environment variable lookup is now case insensitive on Windows (fb case 1376500) -# * `NPath.ResolveWithFileSystem()` will now return correct paths for Stevedore artifacts using `UnpackToUnusualLocation(NPath)`. -# requirements) -# * Attempt to force-disable color codes in build logs when building in Xcode, to avoid cases where Xcode can't parse the log for filenames. -# * `UnpackToUnusualLocation(NPath)` now properly handles custom artifact directories as well as `StevedoreSettings.UnpackPathPrefix`. Previously version files were hardcoded to be stored in "artifacts" causing steve to unpack them again if "artifacts" didn't exist or was removed. -# * Fix SDK Style project generation where CSharpProgram.CSProjBuildEvents were ignored -# * Fix SDK Style project generation where StartInfo was ignored -# * Remove output files before running static linkers, since many of them append to existing files. -# * Stevedore manifests are now registered as inputs to the build program. -# -# ## Other changes -# * The c++ program that does the actual build is renamed from tundra2.exe to bee_backend.exe. On linux it is no longer compiled with gcc but with clang. -# * Object files in a `BagOfObjectFilesLibrary` are now stored, and presented to the linker, in sorted order. -# * Platforms accessed through `ToolChain.Store` have been moved from `Bee.Core` to their relevant toolchain assemblies, which will make it easier to realize that you're missing an assembly reference now that Bee isn't shipped as a single assembly anymore. If the platform you're looking for is missing (say `ToolChain.Store.Windows()`), make sure you're referencing the relevant assembly (in this case, `Bee.Toolchain.Windows.dll`). -# * The "bee steve pack" command has been removed, since it no longer works with our infrastructure. See https://stevedore.unity3d.com/upload/ for alternative upload methods. -# * When publishing a `CSharpProgram2` with debug codegen, the generated PDB will always contain the source files. This is a workaround until we decide on a method to conditionally set properties -# * `CSharpProgram2.GenerateXmlDoc` has changed to a bool value, users can configure warnings via `WarningsAsErrors` and `IgnoredWarnings` -# * Add a preview-internal scaffolding in preparation for running bee on dotnet5 -# * Clang Sanitizers if enabled are now always linked statically -# * rebuilds (bee -l, jam -a) have been broken for a while. They are not fixed yet, but we print an error message now when you try this, instead of running endless loops of endless rebuilds. -# * The build (and rebuild) commands for the generated native program project files were not working, they were replaced with an informative message instructing the user to build them on the command line instead.* NDA bits are now distributed as Stevedore artifacts, declared in `bee_manifest.stevedore` files. -# * Relatedly, there is now a single bootstrap zip, which is the default way to consume Bee. -# * UseCSharpSupport2 and RunBeeOnNet5 are no longer options in bee.config, they are always considered turned on now. -# * TinyProfiler2 writes files without BOM - - -# ### Bee 2022 release notes -# -# ## Changes Requiring Changes To BuildCode -# -# * AppPackage.NativeLibraries condition input for `CollectionWithConditions` is no longer `(AppPackageConfiguration, Architecture)>` but `AppPackageConfigurationWithSpecificArchitecture` instead -# * Xcode toolchain classes for `Lipo` and `Dsymutil` now operate on `BuiltNativeProgram` instances, rather than `NPath` instances. -# -# ## Improvements -# -# * bee_backend on Windows now uses pipes instead of temp files for child process -# output. This might avoid problems due to antivirus software interfering with -# the build process. -# * `DownstreamCustomer` class now uses `CollectionWithConditions` to build command lines for `ApplySteps` and `VerifySteps`. This allows to write shell scripts with commands shared -# or specific to Operating systems. -# * WriteTextAction can now handle paths longer than 260 characters on Windows. -# * Putting the "same" CSharpProgram2 multiple times in the build graph targeting different frameworks is now possible. Prior to this fix an error was raised that the csproj was already registered with different content. -# * Added `bee steve download-manifest ` which can be used to download steve artifact files and place them next to the manifest. -# * BeeDriver2 library now exposes responsefiles on nodes, and a buildResult.WriteSummary function -# * DotNetSdk.Default is now has a public setter. -# * CSharpProgram2 now has a new `InvariantGlobalization` property which lets you enable Globalization Invariant Mode (https://github.com/dotnet/runtime/blob/main/docs/design/features/globalization-invariant-mode.md) and defaults to true. -# * Added `Backend.AddRPCAction` and related APIs to allow implementing actions in the process hosting the BeeDriver. -# * `cachewhy` now supports a `--target` parameter that can be used to specify the 'main' target that the requested target is part of, in order to support MultiDAG scenarios. -# * Updated to 13.0.1 of Newtonsoft.Json.dll -# * Added support for specifying SourceLink metadata for MSVC-based toolchains. -# * Text actions and response files can now have arbitrary text encodings -# * Add `CSharpProgram2.SetupPublish()` intended to replace SetupPublishFrameworkDependent(). It is a very different implementation that no longer uses Espionage technique to recreate the msbuild graph inside the bee graph, but instead just invokes "dotnet publish". A bit worse for incremental builds, but much more performant for full builds, and much less brittle. -# * CSharpProgram2 exposes now `WarningsNotAsErrors` -# * BuildProgram compilation no longer treats warnings from `[Obsolete]` as errors -# * `Backend` has now a `AddWriteTextAction` overload that takes a struct which allows to set arbitrary text encodings -# * You can now use the new `ActionArgs`-based APIs to configure actions: `Backend.AddCopyFileAction`, `Backend.AddShellCommandAction`, `Backend.AddCSharpAction`, `Backend.AddWriteTextAction`, etc. -# * on windows only CSharpProgram2.SetupPublish() now causes a "dotnet build-server shutdown" to be invoked at the end of the build so buildservers do not stay running -# * CSharpProgram2.SetupPublish() now takes a SetupPublishArgs so that future additional arguments will not become API breaks. New argument is TargetFramework for specifying a framework when publishing a library -# * You can now set the `BEE_TEST_TEMP_DIR` environment variable to override the directory used as a root when running tests -# * Added `Backend.RegisterAllOutputFilesInDirectoryToKeepPristine` to mark all output files in a directory as pristine. -# -# ## Bug Fixes -# -# * Updated reapi-cache-client to a version that converts absolute output paths -# in ActionResult's (which are forbidden by the REAPI spec) to relative paths. -# * BeeDriver2: Fix bug where profiler output task was never started, resulting in deadlocks when consuming apps would wait on it. -# * Fix bug where in certain conditions where the build would get cancelled it would first finish -# its entire queued workstack. -# * Fix bug where leaf input caching was broken. -# * Fix popping up command line windows for bee_backend when running in with StdOutMode.Stream (i.e. Unity editor player builds) -# * Fix Visual Studio Build Tools detection. Case 1414194 @martins -# * Fix issues with Console.WriteLine throwing exceptions when stdout is piped to a CI process. -# * Fixed bee.ps1 Move-Item flakiness by unpacking in place. -# * Windows RC.EXE invocations will now locate their `#include`d files as implicit inputs. -# * Fix BeeDriver2 bug where in some situations a scriptupdatable error did not get updated. fogbugz 1423317. -# * Response files for MSVC linking and compiling now use a UTF8 BOM header in order to fix locale issues -# * AppPackage setup now use much shorter paths. -# * Setting up multiple app package formats for the same package in the build graph no longer creates conflicts. -# * Fix bee_backend bug where it would hang if the the beedriver would cancel or crash before sending the dag verification message -# * Fix Leaf input Caching to be always enabled for Linux symbol related actions, even when Leaf input caching is disabled for the native program. -# * Fix bug where VisualStudioSolution solution folders were written out in a way that Rider/VS no longer respected -# * Fix BeeDriver2 bug where nodes that output a profiler output file wasn't being added to the build's combined profile.json file -# * BeeDriver2: stop using the same ipc pipename for multiple connections in a process -# * BeeDriver2: always wait for scriptupdaters to finish, regardless of if the buildprogram has to run again or not -# * BeeDriver2: give a better buildfailure message when a script updater implementor throws an IOException -# * Fix: Emscripten dynamic linker settings expose now Fetch -# -# ## Other changes -# * NPath.Files() and NPath.Directories() when used in buildprograms will now always return their results sorted. This was done to protect build program authors against -# different machines returning file listings in different order, which can cause compilers to produce different outputs. better to make sure we always do this the same way. -# * Change RunnableProgram API & Implementation in BeeDriver2 to be simpler and more robust -# * Deprecated BeeDriver2's ProfilerOutputFile on the buildrequest. Instead use buildRequest.ProfilerMode to indicate if you want a full chrometrace or trace events, and use the buildResult.ProfileOutputWritingTask -# to learn where the file is written -# * The bee standalone driver now runs on .NET6, which drops support to run the bee standalone driver on OSX 10.13. -# * CSharpProgram2's can now be built using the .NET6 sdk. Set the .Sdk property to DotNetSdk.Sdk6 -# * Added support for the `privately-redistributable` Stevedore repository. You can now vendor non-public Bee packages by running `./bee steve vendor privately-redistributable`. -# * Renamed AppPackageFormat.SetupManifest => AppPackageFormat.DefaultManifestSettingsFor and AppManifestSettings.Generate => AppManifestSettings.Setup. -# * On windows, if you start a bee build in a directory where another bee build is running, the 2nd build will now fail instead of waiting for the first one to finish -# * Fixed a crash printing status messages if the annotation contains a printf format specifiers. - -#MANIFEST public: bee/4614ca8dfa47_e30d39dc9e8f44f17d5d0e36aed8224ec76a26ca0702ca4a2bc2679504ef7520.zip \ No newline at end of file +# Automatically generated by Yamato Job: https://unity-ci.cds.internal.unity3d.com/job/77194119 +#MANIFEST public: bee/b649be378334_1a7f70b95e6ff016b5a75212ef827687532a7698eb826ea856f49f4162fe95ed.zip diff --git a/bee/bee.bat b/bee/bee.bat index 165a6aa96f4e0..e7cd997c6fd56 100644 --- a/bee/bee.bat +++ b/bee/bee.bat @@ -2,4 +2,9 @@ SETLOCAL SET distributionDir=%~dp0 SET beePowerShellScript=%distributionDir%bee.ps1 + +REM .NET can randomly hang on Windows while initializing ICU globalization data: +REM https://github.com/dotnet/runtime/issues/89194. Using NLS instead avoids it. +SET DOTNET_SYSTEM_GLOBALIZATION_USENLS=1 + PowerShell -NoProfile -ExecutionPolicy Bypass -File "%beePowerShellScript%" %* diff --git a/bee/bee.ps1 b/bee/bee.ps1 index 54e8a84bf74d0..45a41c1454979 100644 --- a/bee/bee.ps1 +++ b/bee/bee.ps1 @@ -15,12 +15,45 @@ trap $ErrorActionPreference = "Stop" +try { [Console]::OutputEncoding = [System.Text.Encoding]::UTF8 } catch {} + +# Fixup common environment variables that perl might have messed up +# They get corrupted if they contain certain unicode characters (use emojis for testing) +$envUserProfile = $env:USERPROFILE +$actualUserProfile = [Environment]::GetFolderPath('UserProfile'); + +if ([string]::IsNullOrEmpty($actualUserProfile)) { + # Protect ourselves in cases we're run not as a user + $actualUserProfile = $HOME +} + +$isCorruptedUserProfile = + -not [string]::IsNullOrEmpty($envUserProfile) -and + -not [string]::IsNullOrEmpty($actualUserProfile) -and + -not (Test-Path -LiteralPath $envUserProfile -PathType Container -ErrorAction Ignore) + +if ($isCorruptedUserProfile) { + $env:USERPROFILE = $actualUserProfile; + $env:HOME = $actualUserProfile; + + if (-not [string]::IsNullOrEmpty($env:LOCALAPPDATA) -and $env:LOCALAPPDATA.StartsWith($envUserProfile, [System.StringComparison]::OrdinalIgnoreCase)) { + $env:LOCALAPPDATA = $actualUserProfile + $env:LOCALAPPDATA.Substring($envUserProfile.Length) + } + + if (-not [string]::IsNullOrEmpty($env:APPDATA) -and $env:APPDATA.StartsWith($envUserProfile, [System.StringComparison]::OrdinalIgnoreCase)) { + $env:APPDATA = $actualUserProfile + $env:APPDATA.Substring($envUserProfile.Length) + } +} + Add-Type -AssemblyName System.IO.Compression.FileSystem +# Ensure some environment variables that can break the build are not set. +$env:MSBuildSDKsPath = $null # See BUILD-1219 + # These two values will be replaced by the buildprocess when this script is # used in a bootstrap distribution. When not replaced this script assumes it # lives next to the standalone driver. -$use_bee_from_steve="bee/4614ca8dfa47_e30d39dc9e8f44f17d5d0e36aed8224ec76a26ca0702ca4a2bc2679504ef7520.zip" +$use_bee_from_steve="bee/b649be378334_1a7f70b95e6ff016b5a75212ef827687532a7698eb826ea856f49f4162fe95ed.zip" $use_bee_from_steve_repo="https://public-stevedore.unity3d.com/r/public" if("$use_bee_from_steve_repo" -Match "testing") @@ -28,19 +61,120 @@ if("$use_bee_from_steve_repo" -Match "testing") Write-Host "Warning this bee bootstrap script is using a bee from the testing repository. This fine for testing, but not to use in production." } -# Stevedore artifact for the dotnet runtimes. these are produced by the -# yamato CI for our "netcorerun" repo. The zip file contains an info file +if ($false -eq $IsWindows) +{ + Write-Error "The bee powershell script is only supported on Windows, please run the ./bee script instead." + exit 1 +} + +# Stevedore artifact for the dotnet runtimes & SDKs. these are produced by the +# yamato CI UploadDotNetSdk. The zip file contains an info file # that describes from which git commit / yamato-id it was built. They are # plain upstream packages, just unzipped & rezipped with this extra # information added. -$dotnet_runtime_win_x64 = "dotnet-runtime-win-x64/a057d2d_16de45f31f424625fa1de7ec0cf4d5969c60412a701e17ab6083f83ea1bcb420.zip" #net-runtime 6.0.8 +$dotnet_runtime_win_x64 = "dotnet-runtime-win-x64/10.0.11-9854b7a_4df1e2d4126427eb1afba7c2871895815df610a66ae5f420499b82adfd98b7bd.zip" +$dotnet_runtime_win_arm64 = "dotnet-runtime-win-arm64/10.0.11-9854b7a_6e20639d28b10257b41b93be11b893867e76196aa9c18e5692b7c03f84a47f8a.zip" +$dotnet_sdk_win_x64="dotnet-sdk-win-x64/10.0.303-9854b7a_53ef5e78cf2ff8cbd31b6db17c420271efefb38f4c03ed453aef27bbf59560fe.7z" +$dotnet_sdk_win_arm64="dotnet-sdk-win-arm64/10.0.303-9854b7a_0c43200246662e26e934beda55bee3f7aed313460137d21245dc57e90b60789b.7z" + $global:steve_artifact_return_value = "" + +function Get-StevedorePackageInfo($steve_name) +{ + if (-not ($steve_name -match "^(?.*)/(?.*)_(?.*)\..*$")) { throw "Invalid stevedore artifact name: $steve_name" } + return @($matches.name, $matches.version, $matches.hash, "$($matches.name)_$(-join $matches.hash[0..3])") +} + +function Get-Args($scriptArgs) +{ + # Workaround for powershell parsing the command line arguments. Powershell will drop the `:` when passed things like `-p:Foo=true` which is a common + # way to pass arguments to msbuild when using dotnet. + + $kernel32_definition = @' +[DllImport("kernel32")] +public static extern IntPtr GetCommandLineW(); +[DllImport("kernel32")] +public static extern IntPtr LocalFree(IntPtr hMem); +'@ + + $kernel32 = Add-Type -IgnoreWarnings -MemberDefinition $kernel32_definition -Name 'Kernel32' -Namespace 'Win32' -PassThru + + $shell32_definition = @' +[DllImport("shell32.dll", SetLastError = true)] +public static extern IntPtr CommandLineToArgvW( +[MarshalAs(UnmanagedType.LPWStr)] string lpCmdLine, +out int pNumArgs); +'@ + + $shell32 = Add-Type -IgnoreWarnings -MemberDefinition $shell32_definition -Name 'Shell32' -Namespace 'Win32' -PassThru + + $raw_command_line = [System.Runtime.InteropServices.Marshal]::PtrToStringUni($kernel32::GetCommandLineW()) + + $parsed_arg_count = 0 + $parsed_arg_ptr = $shell32::CommandLineToArgvW($raw_command_line, [ref] $parsed_arg_count) + + $all_args = @( ) + try + { + for ($i = 0; $i -lt $parsed_arg_count; $i++) + { + $all_args += [System.Runtime.InteropServices.Marshal]::PtrToStringUni( + [System.Runtime.InteropServices.Marshal]::ReadIntPtr($parsed_arg_ptr, $i * [IntPtr]::Size) + ) + } + } + finally + { + $kernel32::LocalFree($parsed_arg_ptr) | Out-Null + } + + # Everything up to and including this script's own path (e.g. + # argv[0] + # argv[1] <-NoProfile> + # argv[2] <-ExecutionPolicy> + # argv[3] + # argv[4] <-File> + # argv[5] + # ) is PowerShell's own invocation preamble, not user args. We locate it by + # path instead of assuming a fixed token count (6), since that count depends + # on exactly how PowerShell was invoked and isn't guaranteed. + $script_index = -1 + for ($i = 0; $i -lt $all_args.Length; $i++) + { + if ($all_args[$i] -eq $PSCommandPath) + { + $script_index = $i + break + } + } + + if ($script_index -eq -1) + { + # Couldn't find our own script path in the raw command line (e.g. this + # script was dot-sourced or invoked directly in an interactive session + # rather than launched as its own process); fall back to PowerShell's + # own (colon-mangling) argument parsing. + return $scriptArgs + } + + if ($script_index -eq $all_args.Length - 1) + { + # No user args were passed. + return @( ) + } + + return $all_args[($script_index + 1)..($all_args.Length - 1)] +} + + function Get-Stevedore-Artifact($steve_name, $steve_repo_url, $output_name) { + $package_name, $package_version, $package_hash, $package_unique_path = Get-StevedorePackageInfo $steve_name + # We could extend this to parse the Stevedore.conf, and to attempt # downloads from multiple mirrors... - $unzip_dir_path = "$HOME\.beebootstrap\$steve_name" + $unzip_dir_path = "$actualUserProfile\.beebootstrap\$package_unique_path" $unpacked_marker = "$unzip_dir_path\.UNPACKED" if(![System.IO.File]::Exists($unpacked_marker)) { @@ -51,7 +185,7 @@ function Get-Stevedore-Artifact($steve_name, $steve_repo_url, $output_name) $download_link = "$steve_repo_url/$steve_name" $random = Get-Random - $temporary_dir = "$HOME/.beebootstrap/download_$random" + $temporary_dir = "$actualUserProfile/.beebootstrap/download_$random" New-Item -ItemType Directory -Force -Path $temporary_dir | Out-Null $downloaded_file = "$temporary_dir/download.zip" @@ -63,8 +197,40 @@ function Get-Stevedore-Artifact($steve_name, $steve_repo_url, $output_name) Write-Host "Downloading $output_name" - # Download into the temporary directory - Invoke-WebRequest $download_link -outfile "$downloaded_file" + # A couple of short retries first, then back off to ~30s intervals; the + # last retry lands a bit past the 3 minute mark before giving up. + $dl_success = $false + $retry_delays_seconds = @(0, 2, 2, 30, 30, 30, 30, 30, 30, 30) + foreach ($sleep_seconds in $retry_delays_seconds) + { + Start-Sleep -Seconds $sleep_seconds + + try { Invoke-WebRequest $download_link -outfile "$downloaded_file" } + catch + { + Write-Host "Download of `"$download_link`" failed: $($_.Exception.Message)" + continue + } + + if (Get-Command Get-FileHash -errorAction SilentlyContinue) + { + $actual_sha256 = (Get-FileHash "$downloaded_file" -Algorithm SHA256).Hash + if ($actual_sha256 -ne $package_hash) + { + Write-Host "$downloaded_file sha256 mismatch: expected $package_hash actual $actual_sha256 - removing" + Remove-Item "$downloaded_file" -Force + continue + } + } + + $dl_success = $true + break + } + if (!$dl_success) + { + Write-Host "Failed to download `"$download_link`"" + exit 1 + } # Make sure the parent directory of the target directory already exists $parent_dir = Split-Path -Parent $unzip_dir_path @@ -82,7 +248,7 @@ function Get-Stevedore-Artifact($steve_name, $steve_repo_url, $output_name) Remove-Item "$temporary_dir" -Recurse -Force } - # We assign to a global instead of returning a value, since returning + # We assign to a global instead of returning a value, since returning # a value is somehow very brittle, as any command in this function # that isn't piped to Out-Null might actually pollute the return value. $global:steve_artifact_return_value = $unzip_dir_path @@ -110,29 +276,97 @@ if(![System.IO.File]::Exists($standalone_path)) { $standalone_path = "$standalone_release/Bee.StandaloneDriver.dll" } -$dotnet_steve_artifact = "$dotnet_runtime_win_x64" -Get-Stevedore-Artifact $dotnet_steve_artifact "https://public-stevedore.unity3d.com/r/public" "Dotnet runtime" -$dotnet_exe = "$global:steve_artifact_return_value\dotnet.exe" +if([Environment]::GetEnvironmentVariable("PROCESSOR_ARCHITECTURE", 'Machine') -eq "ARM64") +{ + $dotnet_runtime_steve_artifact = "$dotnet_runtime_win_arm64" + $dotnet_sdk_steve_artifact = "$dotnet_sdk_win_arm64" +} +else +{ + $dotnet_runtime_steve_artifact = "$dotnet_runtime_win_x64" + $dotnet_sdk_steve_artifact = "$dotnet_sdk_win_x64" +} -# We assign BEE_DOTNET_MUXER env var here, so that the bee that is running -# knows how it can use this dotnet runtime to start other net5 framework -# dependent apps. It uses this to run the stevedore downloader program on net5. +if($Null -ne $env:BEE_DOTNET_MUXER)# if we are recursing, then use the dotnet.exe from outside. +{ + $dotnet_exe = $env:BEE_DOTNET_MUXER +} +else +{ + Get-Stevedore-Artifact $dotnet_runtime_steve_artifact "https://public-stevedore.unity3d.com/r/public" "Dotnet runtime" + $dotnet_exe = "$global:steve_artifact_return_value\dotnet.exe" +} + +$parsed_args = Get-Args $args + +#we assign BEE_DOTNET_MUXER env var here, so that the bee that is running knows how it can use this dotnet runtime to start other dotnet framework dependent apps. +#it uses this to run the stevedore downloader program. try { $env:BEE_DISTRIBUTION_PATH = $distribution_path $env:BEE_DOTNET_MUXER = $dotnet_exe - $env:DOTNET_MULTILEVEL_LOOKUP=0 + $DOTNET_MULTILEVEL_LOOKUP_backup = $env:DOTNET_MULTILEVEL_LOOKUP + $env:DOTNET_MULTILEVEL_LOOKUP = 0 + + # Decide the branch using PowerShell's own $args (reliable) rather than the + # raw-reparsed $parsed_args, since the latter's alignment depends on how this + # script was invoked. $parsed_args is only needed below to preserve `:` in + # arguments like `-p:Foo=true` when passing them on to the dotnet SDK. + if($args.Length -gt 0 -and $args[0] -eq 'dotnet') + { + if($Null -ne $env:BEE_DOTNET_SDK_UNZIP_DIR_PATH) # if there is a dotnet_sdk from outside + { # use the dotnet sdk from outside. + $bee_dotnet_sdk_unzip_dir_path = $env:BEE_DOTNET_SDK_UNZIP_DIR_PATH + } + else + { + # Use bee steve to download and unpack the dotnet SDK + $package_name, $package_version, $package_hash, $package_unique_path = Get-StevedorePackageInfo $dotnet_sdk_steve_artifact + $bee_dotnet_sdk_unzip_dir_path = "$actualUserProfile\.beebootstrap\$package_unique_path" + & $dotnet_exe $standalone_path "steve", "internal-unpack", "public", "$dotnet_sdk_steve_artifact", "$bee_dotnet_sdk_unzip_dir_path" + + # use the dotnet sdk we just downloaded. + $env:BEE_DOTNET_SDK_UNZIP_DIR_PATH = $bee_dotnet_sdk_unzip_dir_path + } + + # Set the path so this dotnet SDK can be used by children. + $PATH_backup = $env:PATH + $env:PATH = "$bee_dotnet_sdk_unzip_dir_path;$env:PATH" + + # DOTNET_ROOT lets child processes locate the .NET runtime directly. Without it, + # spawned processes (e.g. vstest test hosts) cannot find hostfxr.dll and fail to start. + $env:DOTNET_ROOT = $bee_dotnet_sdk_unzip_dir_path - # Run the wrapper and pass on any user args - & $dotnet_exe $standalone_path $args + # Run the wrapper and pass on any user args + $dotnet_sdk_exe = "$bee_dotnet_sdk_unzip_dir_path\dotnet.exe" + if ($parsed_args.Length -gt 1) + { + & $dotnet_sdk_exe $parsed_args[1..($parsed_args.Length - 1)] + } + else + { + & $dotnet_sdk_exe + } + } else { + + # Run the wrapper and pass on any user args + & $dotnet_exe $standalone_path $args + } + } finally { $env:BEE_DISTRIBUTION_PATH = $Null $env:BEE_DOTNET_MUXER = $Null + $env:DOTNET_MULTILEVEL_LOOKUP = $DOTNET_MULTILEVEL_LOOKUP_backup + if($Null -ne $PATH_backup) + { + $env:PATH = $PATH_backup + } } + # Ensure exit code is preserved when running from another script. exit $LastExitCode -# ReleaseNotes: -# Automatically generated by Yamato Job: https://unity-ci.cds.internal.unity3d.com/job/20992285 -#MANIFEST public: bee/4614ca8dfa47_e30d39dc9e8f44f17d5d0e36aed8224ec76a26ca0702ca4a2bc2679504ef7520.zip \ No newline at end of file + +# Automatically generated by Yamato Job: https://unity-ci.cds.internal.unity3d.com/job/77194119 +#MANIFEST public: bee/b649be378334_1a7f70b95e6ff016b5a75212ef827687532a7698eb826ea856f49f4162fe95ed.zip