diff --git a/.github/workflows/linux-packages.yml b/.github/workflows/linux-packages.yml index f66a3d6f1a..9ceadf37f0 100644 --- a/.github/workflows/linux-packages.yml +++ b/.github/workflows/linux-packages.yml @@ -220,12 +220,15 @@ jobs: dist="$(find _packages/sources -name 'tablepro-*.tar.xz' ! -name '*-vendor.tar.xz')" vendor="$(find _packages/sources -name 'tablepro-*-vendor.tar.xz')" linux/build-aux/packaging/build-rpm.sh "$dist" "$vendor" _packages/rpm - - name: Keep the Meson test log + - name: Keep the RPM failure logs if: failure() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: linux-rpm-testlog-${{ runner.arch }} - path: _packages/rpm/testlog.txt + path: | + _packages/rpm/testlog.txt + _packages/rpm/disk-usage.tsv + _packages/rpm/disk-usage.txt if-no-files-found: ignore retention-days: 7 - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 diff --git a/linux/build-aux/packaging/build-rpm.sh b/linux/build-aux/packaging/build-rpm.sh index 7cd381de4d..64fc8de939 100755 --- a/linux/build-aux/packaging/build-rpm.sh +++ b/linux/build-aux/packaging/build-rpm.sh @@ -15,7 +15,55 @@ out="$(cd "$3" && pwd)" name="$(basename "$1" .tar.xz)" top="$(mktemp -d)" -trap 'rm -rf "$top"' EXIT +disk_samples="$out/disk-usage.tsv" +disk_summary="$out/disk-usage.txt" +disk_monitor="" + +sample_disk() { + df -Pk "$top" | awk -v timestamp="$(date +%s)" ' + NR == 2 { printf "%s\t%s\t%s\t%s\n", timestamp, $2, $3, $4 } + END { if (NR != 2) exit 1 } + ' >> "$disk_samples" +} + +cleanup() { + build_status=$? + trap - EXIT HUP INT TERM + if [ -n "$disk_monitor" ]; then + kill "$disk_monitor" 2>/dev/null || true + wait "$disk_monitor" 2>/dev/null || echo "RPM disk sampler stopped early; measurements may be incomplete." >&2 + fi + # The archive and debug packages can peak before the build tree is removed. + sample_disk || true + if awk ' + NR == 2 { start = $4; used = $3; peak = $3; minimum = $4 } + NR > 1 { if ($3 > peak) peak = $3; if ($4 < minimum) minimum = $4 } + END { + if (NR < 2) exit 1 + print "RPM filesystem disk usage (sampled every 1 second, KiB):" + printf "start_available\t%.0f\nminimum_available\t%.0f\n", start, minimum + printf "peak_used\t%.0f\npeak_increase\t%.0f\n", peak, peak - used + } + ' "$disk_samples" > "$disk_summary"; then + cat "$disk_summary" || true + fi + rm -rf "$top" || true + exit "$build_status" +} +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM +printf 'unix_seconds\tsize_kib\tused_kib\tavailable_kib\n' > "$disk_samples" +sample_disk +( + trap - EXIT + trap 'exit 0' HUP INT TERM + while sleep 1; do + sample_disk + done +) & +disk_monitor=$! mkdir "$top/SOURCES" "$top/SPECS" cp "$1" "$2" "$top/SOURCES/" @@ -35,9 +83,12 @@ dnf builddep -y --setopt=install_weak_deps=False "$srpm" # reads the vendored crates from the config %cargo_prep writes. # Meson prints only the end of a failed suite's log, so the whole log is # kept beside the packages. -if ! CARGO_NET_OFFLINE=true rpmbuild --define "_topdir $top" --rebuild "$srpm"; then - find "$top/BUILD" -path '*/meson-logs/testlog.txt' -exec cp {} "$out/" \; - exit 1 +if CARGO_NET_OFFLINE=true rpmbuild --define "_topdir $top" --rebuild "$srpm"; then + : +else + build_status=$? + find "$top/BUILD" -path '*/meson-logs/testlog.txt' -exec cp {} "$out/" \; || true + exit "$build_status" fi cp "$srpm" "$top"/RPMS/*/*.rpm "$out/" diff --git a/linux/build-aux/rpm/tablepro.spec b/linux/build-aux/rpm/tablepro.spec index 9369b9814a..fdd09449a5 100644 --- a/linux/build-aux/rpm/tablepro.spec +++ b/linux/build-aux/rpm/tablepro.spec @@ -2,6 +2,9 @@ # Full debuginfo adds to what each rustc of the app and the final link # need, and an 8 GB builder ran out of memory with it. %global rustflags_debuginfo 1 +# Native objects and their archive coexist; full DuckDB DWARF can fill the builder before linking. +%global _pkg_extra_cflags -g1 +%global _pkg_extra_cxxflags -g1 # Fedora's one codegen unit puts the whole app crate through LLVM as one # module, and its unit test build passed 7 GB that way. Cargo's release # default of 16 splits the work, and thin LTO still optimizes across it. diff --git a/linux/build-aux/tests/test_rpm_disk_usage.py b/linux/build-aux/tests/test_rpm_disk_usage.py new file mode 100644 index 0000000000..55d0bdfeef --- /dev/null +++ b/linux/build-aux/tests/test_rpm_disk_usage.py @@ -0,0 +1,77 @@ +import csv +import io +import os +import subprocess +import tempfile +import tarfile +import unittest +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parent.parent / "packaging" / "build-rpm.sh" + + +class RpmDiskUsageTest(unittest.TestCase): + def test_a_failed_build_keeps_its_exit_code_and_stops_the_sampler(self): + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + tools = work / "bin" + tools.mkdir() + dnf = tools / "dnf" + dnf.write_text("#!/bin/sh\nexit 0\n") + dnf.chmod(0o755) + rpmbuild = tools / "rpmbuild" + rpmbuild.write_text("""#!/bin/sh +set -eu +build_top=${2#_topdir } +if [ "$3" = -bs ]; then + mkdir -p "$build_top/SRPMS" "$build_top/BUILD/failed/meson-logs" + touch "$build_top/SRPMS/tablepro.src.rpm" +else + printf 'package test failed\\n' > "$build_top/BUILD/failed/meson-logs/testlog.txt" + sleep 2 + exit 37 +fi +""") + rpmbuild.chmod(0o755) + temporary = work / "temporary" + temporary.mkdir() + source = work / "tablepro-0.1.0.tar.xz" + vendor = work / "tablepro-0.1.0-vendor.tar.xz" + with tarfile.open(source, "w:xz") as archive: + spec = b"Name: tablepro\n" + entry = tarfile.TarInfo("tablepro-0.1.0/linux/build-aux/rpm/tablepro.spec") + entry.size = len(spec) + archive.addfile(entry, io.BytesIO(spec)) + vendor.touch() + output = work / "output" + environment = dict(os.environ, PATH=f"{tools}:{os.environ['PATH']}", TMPDIR=str(temporary)) + + # An orphan sampler holds the captured pipes open after the build exits. + result = subprocess.run( + ["sh", str(SCRIPT), str(source), str(vendor), str(output)], + env=environment, + capture_output=True, + text=True, + timeout=10, + check=False, + ) + + self.assertEqual(result.returncode, 37, result.stderr) + self.assertEqual(list(temporary.iterdir()), []) + self.assertEqual((output / "testlog.txt").read_text(), "package test failed\n") + with (output / "disk-usage.tsv").open() as samples: + rows = list(csv.DictReader(samples, delimiter="\t")) + self.assertGreaterEqual(len(rows), 3) + self.assertTrue(all(int(row["size_kib"]) > 0 for row in rows)) + self.assertTrue(all(int(row["available_kib"]) >= 0 for row in rows)) + summary = (output / "disk-usage.txt").read_text() + self.assertIn("sampled every 1 second", summary) + self.assertIn(summary, result.stdout) + measurements = dict(line.split("\t") for line in summary.splitlines()[1:]) + self.assertEqual(int(measurements["start_available"]), int(rows[0]["available_kib"])) + self.assertEqual(int(measurements["minimum_available"]), min(int(row["available_kib"]) for row in rows)) + self.assertEqual(int(measurements["peak_used"]), max(int(row["used_kib"]) for row in rows)) + + +if __name__ == "__main__": + unittest.main()