diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 9f76b5e0..18c7d681 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,6 @@ +=== Unreleased +- Windows: a DLL is packed only where the loader looks for it. Since 1.4.5 the same DLL could land in up to three places - bin, the Ruby archdir and the devkit's msys64\ucrt64\bin (e.g. libcrypto-3-x64.dll, libssl-3-x64.dll, zlib1.dll) - adding ~14 MB to a 46 MB --output-dir package. RubyInstaller ships the DLLs of an extension beside it as a private SxS assembly (openssl.so depends on openssl.so-assembly, whose manifest lists libcrypto-3-x64.dll and libssl-3-x64.dll), which the loader resolves to that directory only: such a DLL now stays beside its extension and no longer gets a copy in bin, which it never used. Other DLLs - companions without an assembly, and DLLs the dependency run loaded through the search order from the devkit or from outside the prefix (OpenSSL loads ZLIB1.dll by name) - go into bin once per basename, and no longer also to their original location in a Ruby arch dir or msys64; a basename ruby_builtin_dlls provides is not packed again. DLLs in bin and in a gem's tree (e.g. tiny_tds' FreeTDS) keep their location. A RubyInstaller 4.0 test app using openssl, psych, zlib, fiddle and sqlite3, built under ridk/bundle exec, shrinks from 30.5 MB to 24.0 MB and runs with a minimal PATH. The decision is Direction.windows_dll_placement, unit tested on every platform. + === 1.5.1 - Releases carry the three `--static-runtime` runtimes (Windows zip, Linux and macOS tarballs) next to the gems: the release workflow calls the runtime workflows, which build, test and attest them, and attaches what they built (a published release is immutable, so they cannot be added afterwards). A manual run of the release workflow is a dry run that builds and tests everything a release would contain and publishes nothing. The runtime workflows run on branch pushes only; for a tag the release workflow runs them. - The macOS runtime and macOS executables are reproducible: the runtime build sets ZERO_AR_DATE=1 (Apple's ar, libtool and ranlib wrote each member's modification time into the static libraries, and ld the objects' times into the executable's debug map), and OCRAN links macOS executables with -Wl,-S, without a debug map naming objects compiled in a temporary directory. The runtime workflows package the smoke application twice and require the same bytes. diff --git a/lib/ocran/direction.rb b/lib/ocran/direction.rb index 5fafc651..392fe783 100644 --- a/lib/ocran/direction.rb +++ b/lib/ocran/direction.rb @@ -50,6 +50,84 @@ class Direction # ships a prebuilt sqlite3_native.so inside its lib directory. Both the # gem directory and the extension directory (where RubyGems puts the # products of a source build) are scanned. + # Decides where each Windows DLL candidate is packed, so that a DLL is + # packed only where the loader looks for it. + # + # RubyInstaller ships the DLLs of an extension next to it as a private + # SxS assembly: openssl.so embeds a dependency on "openssl.so-assembly", + # whose manifest beside it (openssl.so-assembly.manifest) lists + # libcrypto-3-x64.dll and libssl-3-x64.dll as its files. The loader + # resolves those names to the assembly's directory and nowhere else - a + # copy in bin does not help openssl.so, and it fails to load without the + # copy beside it. ruby_builtin_dlls is the same mechanism for ruby.exe's + # own assembly, active for the whole process. Every other DLL is found + # through the search order, whose only packed directory is bin (the + # application directory of ruby.exe). So: + # - a DLL listed by an assembly manifest in its own directory stays + # where it is and gets no copy in bin; one in bindir goes into bin, + # beside the packed ruby.exe, even where bindir is not + # exec_prefix/bin; + # - a basename that ruby_builtin_dlls provides is not packed again; + # - any other DLL - a companion without an assembly, one the dependency + # run loaded from the devkit's msys64\ucrt64\bin or from outside the + # prefix (OpenSSL loads ZLIB1.dll by name at run time) - is copied into + # bin once per basename, preferring the copy from the Ruby tree. A DLL + # from a gem's tree (e.g. FreeTDS under tiny_tds' ports/) also keeps its + # original location, where the gem may load it by path; one from a + # Ruby arch dir or msys64 does not, the bin copy serves it. + # + # +candidates+ are [path, kind] pairs in detection order. +builtin_dir+ + # is RubyInstaller's ruby_builtin_dlls (packed whole elsewhere), or nil. + # Returns [source, target, kind] triples, target relative to the + # extraction root. + def self.windows_dll_placement(candidates, exec_prefix:, bindir:, builtin_dir: nil, arch_dirs: []) + msys_dir = exec_prefix / "msys64" + rank = lambda do |dll| + if dll.subpath?(bindir) then 0 + elsif dll.subpath?(msys_dir) then 2 + elsif dll.subpath?(exec_prefix) then 1 + else 3 + end + end + assembly_files = Hash.new do |cache, dir| + cache[dir] = dir.glob("*.manifest").flat_map do |manifest| + File.read(manifest, mode: "rb").scan(/]*?\bname\s*=\s*["']([^"']+)["']/i).flatten + rescue SystemCallError + [] + end.to_set { |name| name.downcase } + end + + in_bin = Set.new + builtin_dir&.children&.each { |path| in_bin << path.basename.to_s.downcase if path.extname?(".dll") } + targets = Set.new + placement = [] + place = lambda do |dll, target, kind| + placement << [dll, target, kind] if targets.add?(target.to_s.downcase) + end + + candidates.each_with_index.sort_by { |(dll, _), i| [rank.(dll), i] }.each do |(dll, kind), _| + name = dll.basename.to_s.downcase + next if builtin_dir && dll.subpath?(builtin_dir) + + if dll.subpath?(bindir) + in_bin << name if dll.dirname.path_key == bindir.path_key + place.(dll, BINDIR / dll.relative_path_from(bindir), kind) + next + end + if dll.subpath?(exec_prefix) && assembly_files[dll.dirname].include?(name) + place.(dll, dll.relative_path_from(exec_prefix), kind) + next + end + next unless in_bin.add?(name) + + place.(dll, BINDIR / dll.basename, kind) + if dll.subpath?(exec_prefix) && !dll.subpath?(msys_dir) && arch_dirs.none? { |dir| dll.subpath?(dir) } + place.(dll, dll.relative_path_from(exec_prefix), kind) + end + end + placement + end + def self.gem_native_binaries(spec) ext_dir = begin @@ -555,32 +633,26 @@ def construct(builder) end end - # Windows-only: DLLs bundled into bin, next to ruby.exe, by basename - dlls_in_bin = Set.new - add_dll_to_bin = proc do |dll| - next if dll.subpath?(bindir) || !dlls_in_bin.add?(dll.basename.to_s.downcase) - - say "Adding detected DLL #{dll} to bin" - builder.copy_to_bin(dll, dll.basename) - end + # Windows-only: DLL candidates, packed by .windows_dll_placement below + windows_dlls = [] # Windows-only: Add detected DLLs if Gem.win_platform? && @option.auto_detect_dlls? && !@option.static_runtime? # The Windows loader resolves the imports of a native extension from - # the extension's own directory, the application directory of the - # packed ruby.exe (bin) plus its SxS assembly (ruby_builtin_dlls), - # and the system directories. PATH is not consulted on hardened - # systems, and the AddDllDirectory mechanism gems use through - # ruby_installer/runtime is not available in a packed app. A detected - # DLL that lives anywhere else - a gem's bundled library (e.g. - # FreeTDS under tiny_tds' ports/), a devkit's msys64 tree inside the - # Ruby prefix, or a directory outside the prefix entirely - is packed - # only at a location the loader never searches, and the application - # dies with LoadError on machines where nothing masks the gap. So - # additionally bundle a copy of every such DLL into bin, next to - # ruby.exe, mirroring what the Linux branch below achieves with - # LD_LIBRARY_PATH. DLLs from the Windows directory always come from - # the target system and are never bundled. + # its own SxS assembly (RubyInstaller's .so-assembly.manifest + # beside it), the application directory of the packed ruby.exe (bin) + # plus its assembly (ruby_builtin_dlls), and the system directories. + # PATH is not consulted on hardened systems, and the AddDllDirectory + # mechanism gems use through ruby_installer/runtime is not available + # in a packed app. A detected DLL that lives anywhere else - a gem's + # bundled library (e.g. FreeTDS under tiny_tds' ports/), a devkit's + # msys64 tree inside the Ruby prefix, or a directory outside the + # prefix entirely - is packed only at a location the loader never + # searches, and the application dies with LoadError on machines + # where nothing masks the gap. So bundle every such DLL into bin, + # next to ruby.exe, mirroring what the Linux branch below achieves + # with LD_LIBRARY_PATH. DLLs from the Windows directory always come + # from the target system and are never bundled. windows_dir = Pathname(ENV["SystemRoot"] || "C:/Windows") detect_dlls.each do |dll| @@ -589,17 +661,9 @@ def construct(builder) # (and any MSVC-built DLL a gem loads) fails to start with # 0xC0000135 where the VC++ Redistributable is not installed. # Microsoft permits deploying it app-locally, next to ruby.exe. - if dll.basename.to_s.match?(VC_RUNTIME_DLL_RE) - add_dll_to_bin.call(dll) - next - end - next if dll.subpath?(windows_dir) + next if dll.subpath?(windows_dir) && !dll.basename.to_s.match?(VC_RUNTIME_DLL_RE) - if dll.subpath?(exec_prefix) - say "Adding detected DLL #{dll}" - builder.duplicate_to_exec_prefix(dll) - end - add_dll_to_bin.call(dll) + windows_dlls << [dll, "detected"] end # Proactively include companion DLLs for loaded native extensions. @@ -607,18 +671,15 @@ def construct(builder) # directory (e.g., libssl-3-x64.dll alongside openssl.so) that are # loaded lazily on first use. Scanning .so directories ensures those # DLLs are bundled even when the extension is required but not - # exercised during the OCRAN dependency scan. They also go into bin: - # a copy in archdir only helps extensions in archdir itself, while - # the same extension packed at a gem path (openssl and psych are - # gems since Ruby 3.x) resolves its imports from bin. + # exercised during the OCRAN dependency scan. Those an extension's + # assembly manifest lists stay beside it, where the loader looks for + # them; others go into bin (see .windows_dll_placement). features.select { |f| f.extname?(".so") && f.subpath?(exec_prefix) } .map(&:dirname).uniq { |dir| dir.path_key } .each do |dir| dir.each_child do |path| next unless path.file? && path.extname?(".dll") - say "Adding companion DLL #{path}" - builder.duplicate_to_exec_prefix(path) - add_dll_to_bin.call(path) + windows_dlls << [path, "companion"] end end end @@ -632,13 +693,15 @@ def construct(builder) # PE import tables, so they are packed with --no-autodll too, and for # extensions the dependency run did not exercise. if Gem.win_platform? - windows_imported_dlls(features).each do |dll| - if dll.subpath?(bindir) - verbose "Adding imported DLL #{dll}" - builder.duplicate_to_exec_prefix(dll) - else - add_dll_to_bin.call(dll) - end + windows_dlls.concat(windows_imported_dlls(features).map { |dll| [dll, "imported"] }) + + arch_dirs = RbConfig::CONFIG.values_at("archdir", "sitearchdir", "vendorarchdir") + .compact.map { |dir| Pathname(dir) } + Direction.windows_dll_placement(windows_dlls, exec_prefix: exec_prefix, bindir: bindir, + builtin_dir: ruby_builtin_manifest&.dirname, + arch_dirs: arch_dirs).each do |dll, target, kind| + say "Adding #{kind} DLL #{dll} as #{target}" + builder.cp(dll, target) end end diff --git a/test/test_windows_dll_placement.rb b/test/test_windows_dll_placement.rb new file mode 100644 index 00000000..59ae8098 --- /dev/null +++ b/test/test_windows_dll_placement.rb @@ -0,0 +1,163 @@ +# frozen_string_literal: true +require "minitest/autorun" +require "tmpdir" +require "pathname" +require_relative "../lib/ocran/direction" + +# Windows packages carried the same DLL in up to three places (bin, the +# Ruby archdir and the devkit's msys64\ucrt64\bin). A DLL must be packed +# where the loader looks for it - beside its extension when the extension's +# SxS assembly lists it, in bin otherwise - and nowhere else. The decision +# is path and manifest logic, so it is tested on any platform. +class TestWindowsDllPlacement < Minitest::Test + def setup + @dir = Dir.mktmpdir + @prefix = Pathname(@dir) / "Ruby40-x64" + @bindir = @prefix / "bin" + @builtin = @bindir / "ruby_builtin_dlls" + @archdir = @prefix / "lib/ruby/4.0.0/x64-mingw-ucrt" + @msys = @prefix / "msys64/ucrt64/bin" + @fiddle = @prefix / "lib/ruby/gems/4.0.0/extensions/x64-mingw-ucrt/4.0.0/fiddle-1.1.8" + [@builtin, @archdir, @msys, @fiddle].each(&:mkpath) + %w[libwinpthread-1.dll libgmp-10.dll].each { |f| (@builtin / f).write("") } + assembly(@builtin, "ruby_builtin_dlls", "libgmp-10.dll", "libwinpthread-1.dll") + # RubyInstaller 4.0's layout + assembly(@archdir, "openssl.so-assembly", "libcrypto-3-x64.dll", "libssl-3-x64.dll") + assembly(@archdir, "psych.so-assembly", "libyaml-0-2.dll") + assembly(@archdir, "zlib.so-assembly", "zlib1.dll") + assembly(@archdir, "date_core.so-assembly", "libwinpthread-1.dll") + assembly(@fiddle, "fiddle.so-assembly", "libffi-8.dll") + end + + def assembly(dir, name, *files) + (dir / "#{name}.manifest").write(<<~XML) + + + + + #{files.map { |f| %() }.join} + + XML + end + + def teardown + FileUtils.remove_entry(@dir) + end + + def place(candidates, builtin_dir: @builtin) + Ocran::Direction.windows_dll_placement(candidates, exec_prefix: @prefix, bindir: @bindir, + builtin_dir: builtin_dir, arch_dirs: [@archdir]) + .map { |source, target, _| [source.relative_path_from(@prefix).to_s, target.to_s] } + end + + def test_rubyinstaller_layout + candidates = [ + [@archdir / "libcrypto-3-x64.dll", "detected"], + [@archdir / "libssl-3-x64.dll", "detected"], + [@msys / "ZLIB1.dll", "detected"], + [@archdir / "libyaml-0-2.dll", "detected"], + [@archdir / "zlib1.dll", "detected"], + [@fiddle / "libffi-8.dll", "detected"], + [@archdir / "libcrypto-3-x64.dll", "companion"], + [@archdir / "libssl-3-x64.dll", "companion"], + [@archdir / "libwinpthread-1.dll", "companion"], + [@archdir / "libyaml-0-2.dll", "companion"], + [@archdir / "zlib1.dll", "companion"], + [@fiddle / "libffi-8.dll", "companion"], + [@msys / "libwinpthread-1.dll", "detected"], + ] + + # Assembly files stay beside their extension, and only there; ZLIB1.dll, + # which OpenSSL loads by name from the devkit, goes into bin once. + assert_equal [ + ["lib/ruby/4.0.0/x64-mingw-ucrt/libcrypto-3-x64.dll", "lib/ruby/4.0.0/x64-mingw-ucrt/libcrypto-3-x64.dll"], + ["lib/ruby/4.0.0/x64-mingw-ucrt/libssl-3-x64.dll", "lib/ruby/4.0.0/x64-mingw-ucrt/libssl-3-x64.dll"], + ["lib/ruby/4.0.0/x64-mingw-ucrt/libyaml-0-2.dll", "lib/ruby/4.0.0/x64-mingw-ucrt/libyaml-0-2.dll"], + ["lib/ruby/4.0.0/x64-mingw-ucrt/zlib1.dll", "lib/ruby/4.0.0/x64-mingw-ucrt/zlib1.dll"], + [@fiddle.relative_path_from(@prefix).join("libffi-8.dll").to_s, @fiddle.relative_path_from(@prefix).join("libffi-8.dll").to_s], + ["lib/ruby/4.0.0/x64-mingw-ucrt/libwinpthread-1.dll", "lib/ruby/4.0.0/x64-mingw-ucrt/libwinpthread-1.dll"], + ["msys64/ucrt64/bin/ZLIB1.dll", "bin/ZLIB1.dll"], + ], place(candidates) + end + + def test_dll_without_an_assembly_goes_into_bin_once + (@archdir / "date_core.so-assembly.manifest").delete + candidates = [ + [@msys / "libffi-8.dll", "detected"], + [@archdir / "libfoo-1.dll", "companion"], + [@msys / "libfoo-1.dll", "detected"], + ] + assert_equal [ + ["lib/ruby/4.0.0/x64-mingw-ucrt/libfoo-1.dll", "bin/libfoo-1.dll"], + ["msys64/ucrt64/bin/libffi-8.dll", "bin/libffi-8.dll"], + ], place(candidates) + end + + def test_builtin_dlls_are_not_packed_again + (@archdir / "date_core.so-assembly.manifest").delete + candidates = [ + [@builtin / "libgmp-10.dll", "detected"], + [@msys / "libgmp-10.dll", "detected"], + [@archdir / "libwinpthread-1.dll", "companion"], + ] + assert_empty place(candidates) + end + + def test_without_ruby_builtin_dlls_the_dll_goes_into_bin + assert_equal [["msys64/ucrt64/bin/libwinpthread-1.dll", "bin/libwinpthread-1.dll"]], + place([[@msys / "libwinpthread-1.dll", "detected"]], builtin_dir: nil) + end + + def test_dll_in_bindir_stays_and_wins_over_other_copies + candidates = [ + [@msys / "libffi-8.dll", "detected"], + [@bindir / "libffi-8.dll", "imported"], + [@bindir / "libffi-8.dll", "detected"], + ] + assert_equal [["bin/libffi-8.dll", "bin/libffi-8.dll"]], place(candidates) + end + + def test_bindir_outside_exec_prefix_bin_maps_to_bin + bindir = @prefix / "usr/bin" + candidates = [[bindir / "libgmp-10.dll", "imported"], [@msys / "libgmp-10.dll", "detected"]] + placement = Ocran::Direction.windows_dll_placement(candidates, exec_prefix: @prefix, bindir: bindir) + assert_equal [["usr/bin/libgmp-10.dll", "bin/libgmp-10.dll"]], + placement.map { |source, target, _| [source.relative_path_from(@prefix).to_s, target.to_s] } + end + + def test_basenames_are_compared_case_insensitively + candidates = [ + [@archdir / "ZLIB1.dll", "companion"], + [@msys / "zlib1.dll", "detected"], + [@prefix / "lib/ruby/site_ruby/libbar.dll", "detected"], + [@msys / "LIBBAR.dll", "detected"], + ] + assert_equal [ + ["lib/ruby/4.0.0/x64-mingw-ucrt/ZLIB1.dll", "lib/ruby/4.0.0/x64-mingw-ucrt/ZLIB1.dll"], + ["lib/ruby/site_ruby/libbar.dll", "bin/libbar.dll"], + ["lib/ruby/site_ruby/libbar.dll", "lib/ruby/site_ruby/libbar.dll"], + ["msys64/ucrt64/bin/zlib1.dll", "bin/zlib1.dll"], + ], place(candidates) + end + + def test_gem_dll_also_keeps_its_original_location + gem_dll = @prefix / "lib/ruby/gems/4.0.0/gems/tiny_tds-3.0.0/ports/bin/libsybdb-5.dll" + assert_equal [ + ["lib/ruby/gems/4.0.0/gems/tiny_tds-3.0.0/ports/bin/libsybdb-5.dll", "bin/libsybdb-5.dll"], + ["lib/ruby/gems/4.0.0/gems/tiny_tds-3.0.0/ports/bin/libsybdb-5.dll", + "lib/ruby/gems/4.0.0/gems/tiny_tds-3.0.0/ports/bin/libsybdb-5.dll"], + ], place([[gem_dll, "detected"]]) + end + + def test_dll_outside_the_prefix_goes_into_bin_after_ruby_tree_copies + candidates = [ + [Pathname(@dir) / "freetds/bin/libiconv-2.dll", "detected"], + [@archdir / "libiconv-2.dll", "companion"], + [Pathname(@dir) / "freetds/bin/libsybdb-5.dll", "detected"], + ] + assert_equal [ + ["lib/ruby/4.0.0/x64-mingw-ucrt/libiconv-2.dll", "bin/libiconv-2.dll"], + ["../freetds/bin/libsybdb-5.dll", "bin/libsybdb-5.dll"], + ], place(candidates) + end +end