diff --git a/.github/scripts/test_sysupgrade.sh b/.github/scripts/test_sysupgrade.sh index 9995928c2..3a072ffb6 100755 --- a/.github/scripts/test_sysupgrade.sh +++ b/.github/scripts/test_sysupgrade.sh @@ -64,6 +64,7 @@ sed -e 's|"$1/etc/os-release"|"${1:-@SB@}@OSRELEASE@"|g' \ -e "s|/proc/cmdline|@SB@/proc/cmdline|g" \ -e "s|/proc/mounts|@SB@/proc/mounts|g" \ -e "s|/proc/meminfo|@SB@/proc/meminfo|g" \ + -e "s|/proc/sys/vm/min_free_kbytes|@SB@/proc/min_free_kbytes|g" \ -e "s|/proc/sys/vm/drop_caches|@SB@/tmp/drop_caches|g" \ -e "s|/etc/init.d/|@SB@/etc/init.d/|g" \ -e "s|/var/run/vigil-supervise.pid|@SB@/tmp/vigil-supervise.pid|g" \ @@ -92,6 +93,9 @@ else echo "S95majestic $1" >> "$FLASH_LOG" fi [ "$1" = stop ] && [ -n "$STUB_STOP_FAILS" ] && exit 1 +# A stopped majestic holds no file any more, the watchdog included. +[ "$1" = stop ] && [ -n "$STUB_MAJESTIC_PID" ] && [ -n "$WDOG_PROC" ] && + rm -rf "$WDOG_PROC/$STUB_MAJESTIC_PID" if [ "$1" = stop ] && [ "$STUB_MEM_AFTER_STOP" = unreadable ]; then : > "$STUB_MEMINFO" elif [ "$1" = stop ] && [ -n "$STUB_MEM_AFTER_STOP" ]; then @@ -587,6 +591,7 @@ reset_env() { unset STUB_PREPARE STUB_ABORT STUB_UMOUNT_BUSY STUB_UMOUNT_HIDES RUN_ENV FOLLOW_LOG STUB_UBI_SLOW rm -rf "$SB/sys" printf '::sysinit:/etc/init.d/rcS\n::shutdown:/bin/umount -a -f\n::restart:/sbin/init\n' > "$SB/etc/inittab" + echo 540 > "$SB/proc/min_free_kbytes" rm -f "$SB"/tmp/*.gk7205v500 set_meminfo : > "$SDMOUNTS" @@ -888,17 +893,121 @@ run -z --archive="$SB/tmp/fw.tgz" # image the cases above have to catch, so refuse while the numbers are still # knowable. #2425 frees the archive after the unpack, which cannot help the # unpack itself. +# A kernel that reports neither MemAvailable nor the fields to estimate it +# leaves no RAM figure, so the cap stands: nothing says it may be raised. reset_env make_archive "$K" "$R" stub df 'echo "Filesystem 1K-blocks Used Available Use% Mounted on"; echo "tmpfs 13564 13560 4 99% /tmp"' +printf 'MemTotal: 131072 kB\nMemFree: 65536 kB\n' > "$SB/proc/meminfo" run -z --archive="$SB/tmp/fw.tgz" -if [ "$RC" -ne 0 ] && nothing_wrote && printf '%s' "$OUT" | grep -q "No room to unpack"; then +if [ "$RC" -ne 0 ] && nothing_wrote && printf '%s' "$OUT" | grep -q "No room to unpack" && + ! grep -q "remount.*size=" "$SB/tmp/flash.log"; then ok "an unpack that cannot fit is refused before it truncates an image" else bad "full /tmp -> expected a refusal, rc=$RC log='$(cat "$SB/tmp/flash.log")'" fi rm -f "$SB/bin/df" +# ...but a cap is not memory. tmpfs takes pages only as they are written, and +# defaults to half of RAM: on an XM510 at mem=18M that is 7804 KB against the +# 8341 KB an archive and its images need together, so no archive could be +# installed there at all (#2618). Where RAM was measured to hold the unpack, +# the cap is raised for the run instead. +reset_env +make_archive "$K" "$R" +stub df 'echo "Filesystem 1K-blocks Used Available Use% Mounted on"; echo "tmpfs 13564 13560 4 99% /tmp"' +run -z --archive="$SB/tmp/fw.tgz" +if [ "$RC" -eq 0 ] && flashed /dev/mtd3 && rebooted && + grep -q "^remount -o remount,size=[0-9]*k .*/tmp$" "$SB/tmp/flash.log" && + printf '%s' "$OUT" | grep -q "Raised the .*/tmp cap from 13564 KB to [0-9]* KB"; then + ok "a /tmp short only by its cap is raised when RAM can back it (#2618)" +else + bad "/tmp cap with RAM to spare -> expected the cap raised and a flash, rc=$RC out='$OUT' log='$(cat "$SB/tmp/flash.log")'" +fi +rm -f "$SB/bin/df" + +# An archive in /tmp that cannot be unpacked beside majestic gets majestic +# stopped by check_unpack_ram anyway; asking it into upgrade mode first is +# what an XM510 at mem=18M could not afford -- majestic, a minute old, was +# OOM-killed during prepare and the run died with it (#2618). So it is +# stopped up front instead, and never asked to prepare. +reset_env +make_archive "$K" "$R" +set_meminfo 100 # the sandbox archive unpacks to about 20 KB +STUB_MAJESTIC_PID=901 STUB_MEM_AFTER_STOP=60000 +run -z --archive="$SB/tmp/fw.tgz" +if [ "$RC" -eq 0 ] && flashed /dev/mtd3 && grep -q "S95majestic stop" "$SB/tmp/flash.log" && + ! grep -q "majestic_api prepare" "$SB/tmp/flash.log" && + printf '%s' "$OUT" | grep -q "Short of memory by [0-9]* KB with majestic running"; then + ok "an archive that cannot unpack beside majestic stops it before prepare (#2618)" +else + bad "short archive -> expected majestic stopped instead of prepared, rc=$RC out='$OUT' log='$(cat "$SB/tmp/flash.log")'" +fi +# majestic owning the watchdog does not keep it from that stop: its own stop +# releases the device with the magic close. But the ownership is recorded +# first, so the RAM-phase keeper still arms in case a driver ignores it. +mkdir -p "$SB/wdproc/901/fd" +ln -sf /dev/null "$SB/wdproc/901/fd/3" +reset_env +make_archive "$K" "$R" +set_meminfo 100 +STUB_MAJESTIC_PID=901 STUB_MEM_AFTER_STOP=60000 +WDOG=/dev/null WDOG_PROC="$SB/wdproc" +run -z --archive="$SB/tmp/fw.tgz" +if grep -q "S95majestic stop" "$SB/tmp/flash.log" && ! grep -q "majestic_api prepare" "$SB/tmp/flash.log" && + printf '%s' "$OUT" | grep -q "majestic fed it and was stopped for memory; taken over at the flash"; then + ok "an owning majestic is stopped early too, and its ownership still arms the keeper" +else + bad "owning majestic, short archive -> expected an early stop with the keeper armed, rc=$RC out='$OUT' log='$(cat "$SB/tmp/flash.log")'" +fi +rm -rf "$SB/wdproc" + +# A stop that does not happen falls back to prepare, as if nothing were short. +reset_env +make_archive "$K" "$R" +set_meminfo 100 +STUB_MAJESTIC_PID=901 STUB_STOP_FAILS=1 STUB_MEM_AFTER_STOP=60000 +run -z --archive="$SB/tmp/fw.tgz" +if grep -q "majestic_api prepare" "$SB/tmp/flash.log"; then + ok "...and a majestic that will not stop is still asked to prepare" +else + bad "failed early stop -> expected the prepare fallback, rc=$RC log='$(cat "$SB/tmp/flash.log")'" +fi + +# ...and one with room leaves majestic to prepare, as before. +reset_env +make_archive "$K" "$R" +STUB_MAJESTIC_PID=901 +run -z --archive="$SB/tmp/fw.tgz" +if [ "$RC" -eq 0 ] && grep -q "majestic_api prepare" "$SB/tmp/flash.log" && + ! grep -q "S95majestic stop" "$SB/tmp/flash.log"; then + ok "...while an archive with room leaves majestic to prepare" +else + bad "archive with room -> expected prepare, not a stop, rc=$RC log='$(cat "$SB/tmp/flash.log")'" +fi + +# A kernel before 3.14 has no MemAvailable. It is estimated the way +# si_mem_available() computes it -- here from what a lab XM510 (3.0.101) +# reported: free 1516 - low watermark 675 (540 min_free + a quarter), file +# cache 7684 - 675, reclaimable slab 904 - 452 = 8302 KB. +reset_env +printf 'MemTotal: 15608 kB\nMemFree: 1516 kB\nBuffers: 2232 kB\nCached: 5548 kB\nActive(file): 1936 kB\nInactive(file): 5748 kB\nShmem: 96 kB\nSReclaimable: 904 kB\n' > "$SB/proc/meminfo" +run -z --kernel="$K" --rootfs="$R" +if printf '%s' "$OUT" | grep -q "RAM left to flash from: 8302 KB"; then + ok "no MemAvailable (kernel < 3.14): estimated as si_mem_available() would" +else + bad "MemAvailable estimate -> expected 8302 KB, out='$(printf '%s' "$OUT" | grep -i 'RAM left')'" +fi +# ...and a meminfo missing a field it needs is still no opinion. +reset_env +printf 'MemTotal: 15608 kB\nMemFree: 1516 kB\nActive(file): 1936 kB\nInactive(file): 5748 kB\n' > "$SB/proc/meminfo" +run -z --kernel="$K" --rootfs="$R" +if [ "$RC" -eq 0 ] && ! printf '%s' "$OUT" | grep -q "RAM left to flash from"; then + ok "...and a meminfo it cannot estimate from is no opinion, not a refusal" +else + bad "partial meminfo -> expected no RAM figure and a flash, rc=$RC out='$OUT'" +fi + # Fails open, like every other measurement in this script: a df that will not # answer is not grounds to refuse an upgrade. reset_env diff --git a/general/overlay/usr/sbin/sysupgrade b/general/overlay/usr/sbin/sysupgrade index 0661c84ca..fb2ed256d 100755 --- a/general/overlay/usr/sbin/sysupgrade +++ b/general/overlay/usr/sbin/sysupgrade @@ -1199,6 +1199,18 @@ above could not see this file. The download is incomplete. Nothing was written." done } +# What unpacking archive $1 puts in /tmp, in KB: the part of it this run keeps, +# sized from the gzip trailer. Fails when the trailer will not read. +archive_unpack_kb() { + local sz need + sz=$(stat -c %s "$1" 2>&3) || return 1 + [ -n "$sz" ] && [ "$sz" -gt 4 ] 2>&3 || return 1 + # The last four bytes of a gzip stream are the uncompressed size. + need=$(od -An -tu4 -j $((sz - 4)) -N4 "$1" 2>&3 | awk '{print $1; exit}') + [ -n "$need" ] && [ "$need" -gt 0 ] 2>&3 || return 1 + unpacked_part_kb $((need / 1024)) unpacked +} + # Will the unpack fit in /tmp? # # Asked only on an archive route, where the .tgz and everything inside it have @@ -1216,13 +1228,9 @@ above could not see this file. The download is incomplete. Nothing was written." # Refuses only on arithmetic it actually has: a gzip trailer that will not read # or a df that will not parse means proceed. check_unpack_room() { - local a=$1 sz need free - sz=$(stat -c %s "$a" 2>&3) || return 0 - [ -n "$sz" ] && [ "$sz" -gt 4 ] 2>&3 || return 0 - # The last four bytes of a gzip stream are the uncompressed size. - need=$(od -An -tu4 -j $((sz - 4)) -N4 "$a" 2>&3 | awk '{print $1; exit}') - [ -n "$need" ] && [ "$need" -gt 0 ] 2>&3 || return 0 - need=$(($(unpacked_part_kb $((need / 1024)) unpacked) * 1024)) + local a=$1 need free + need=$(archive_unpack_kb "$a") || return 0 + need=$((need * 1024)) # The same archive has to fit twice over: once in the tmpfs, and once in the # RAM that tmpfs is made of. They are different numbers -- see # check_unpack_ram -- and either can be the one that binds. @@ -1232,10 +1240,33 @@ check_unpack_room() { free=$(df -k /tmp 2>&3 | awk 'END { print $(NF - 2) }') [ -n "$free" ] && [ "$free" -gt 0 ] 2>&3 || return 0 [ "$((need / 1024))" -lt "$free" ] && return 0 + grow_tmp "$((need / 1024))" "$free" && return 0 die "No room to unpack $a in /tmp: needs $((need / 1024)) KB, $free KB free. Use 'sysupgrade --channel nightly' instead -- it streams and stages no second copy." } +# Raise /tmp's own cap when that cap is all that is short. +# +# tmpfs defaults to half of RAM, and a cap is not memory: it is a ceiling on +# pages taken as they are written. On an XM510 at its stock mem=18M that ceiling +# is 7804 KB, and an archive plus its images is 8341 KB, so every archive +# upgrade was refused (#2618) while the RAM behind /tmp could hold it. Raised by +# what is missing plus the tools' reserve, and only when RAM was measured to +# hold that much -- check_unpack_ram has already counted it and stopped majestic +# if that was needed. A ramfs has no cap, and a RAM figure nobody could read is +# no grounds to raise one. Put back by restore_resources on a run that stays up. +grow_tmp() { + local need=$1 free=$2 size + tmpfs_is_ramfs && return 1 + usable_ram || return 1 + [ "$((need + UNPACK_RESERVE_KB))" -lt "$ram_kb" ] || return 1 + size=$(df -k /tmp 2>&3 | awk 'END { print $(NF - 4) }') + [ -n "$size" ] && [ "$size" -gt 0 ] 2>&3 || return 1 + mount -o remount,size=$((size + need - free + UNPACK_RESERVE_KB))k /tmp 2>&3 || return 1 + tmp_cap_kb=$size + echo "Raised the /tmp cap from $size KB to $((size + need - free + UNPACK_RESERVE_KB)) KB for the unpack: $ram_kb KB of RAM is there to back it." +} + # What the run still needs for itself while that tmpfs is full, in KB. # # The measurement is not free of the measurer: curl, gzip and tar are forked @@ -1254,13 +1285,41 @@ Use 'sysupgrade --channel nightly' instead -- it streams and stages no second co # Overridable so the test harness can pin the boundary. UNPACK_RESERVE_KB=${UNPACK_RESERVE_KB:-512} -# What /tmp can still be given, in KB, into $ram_kb: MemAvailable, less the -# free CMA where tmpfs is ramfs (see check_unpack_ram), whose amount is then -# left in $ram_cma_kb. Fails when MemAvailable cannot be read, which both -# callers take as no opinion. +# What MemAvailable would say, on a kernel too old to say it: 3.14 added the +# line, so XM510 (3.0), XM530 (3.10) and hi3516cv100 (3.0.8) never had it, and +# every RAM guard here stood down on them -- an XM510 then lost its rootfs +# mid-erase (#2647) and could not install an archive at all (#2618). The same +# arithmetic si_mem_available() does: free pages above the low watermark, plus +# the file page cache and reclaimable slab, each less the part reclaim cannot +# have (half, capped at the watermark). The low watermark is min_free_kbytes +# plus a quarter, as the kernel derives it. Fails when a field is missing. +mem_available_estimate() { + local minfree + minfree=$(cat /proc/sys/vm/min_free_kbytes 2>&3) + [ -n "$minfree" ] && [ "$minfree" -ge 0 ] 2>&3 || return 1 + awk -v minfree="$minfree" ' + /^MemFree:/ { free = $2; n++ } + /^Active\(file\):/ { pc += $2; n++ } + /^Inactive\(file\):/ { pc += $2; n++ } + /^SReclaimable:/ { sl = $2; n++ } + END { + if (n < 4) exit 1 + wl = int(minfree * 5 / 4) + a = free - wl + h = int(pc / 2); a += pc - (h < wl ? h : wl) + h = int(sl / 2); a += sl - (h < wl ? h : wl) + print (a > 0 ? a : 0) + }' /proc/meminfo 2>&3 +} + +# What /tmp can still be given, in KB, into $ram_kb: MemAvailable (estimated +# where the kernel does not report it), less the free CMA where tmpfs is ramfs +# (see check_unpack_ram), whose amount is then left in $ram_cma_kb. Fails when +# neither can be read, which both callers take as no opinion. usable_ram() { local cma ram_kb=$(awk '/^MemAvailable:/ { print $2; exit }' /proc/meminfo 2>&3) + [ -n "$ram_kb" ] || ram_kb=$(mem_available_estimate) ram_cma_kb= [ -n "$ram_kb" ] && [ "$ram_kb" -gt 0 ] 2>&3 || return 1 tmpfs_is_ramfs || return 0 @@ -1270,6 +1329,22 @@ usable_ram() { ram_cma_kb=$cma } +# Whether the archive this run unpacks will not fit in RAM beside majestic, +# by $majestic_short_kb. Only an archive already staged in /tmp counts -- one +# behind the old root or still to be downloaded is measured where it lands -- +# and a RAM figure nobody can read is no reason to stop anything. +majestic_short_for_archive() { + local need + majestic_short_kb= + case "$archive" in /tmp/*) ;; *) return 1 ;; esac + pidof majestic >/dev/null 2>&1 || return 1 + need=$(archive_unpack_kb "$archive") || return 1 + usable_ram || return 1 + need=$((need + UNPACK_RESERVE_KB)) + [ "$need" -lt "$ram_kb" ] && return 1 + majestic_short_kb=$((need - ram_kb)) +} + # Stop majestic to give the upgrade the memory it holds, when the upgrade is # short of it. $1 is the shortfall in KB, for the log. # @@ -1291,7 +1366,14 @@ stop_majestic_for_memory() { local pid [ "1" = "$majestic_stopped" ] && return 1 pid=$(pidof majestic 2>/dev/null) || return 1 - if [ -n "$wdog_owner_pid" ]; then + # "releases": free_resources' early stop, before upgrade mode. majestic's + # own stop gives its watchdog back with the magic close, so stopping the + # owner there leaves nothing armed; and its ownership is already recorded, + # so where a driver ignores the magic close the RAM-phase keeper takes the + # device over from the pivot -- the unpack before it is seconds against + # majestic's 300 s. Later stops come after upgrade mode let go of the + # watchdog, and keep the guard. + if [ -n "$wdog_owner_pid" ] && [ "releases" != "$2" ]; then case " $pid " in *" $wdog_owner_pid "*) return 1 ;; esac fi echo_c 33 "\nShort of memory by $1 KB with majestic running: stopping it, which gives back what it holds in RAM." @@ -1814,7 +1896,22 @@ free_resources() { # (restore_resources). A majestic too old to have the endpoint gets the # SIGQUIT it always got. if [ "1" != "$web_update" ]; then - if majestic_api prepare; then + # An archive already in /tmp that cannot be unpacked beside majestic + # gets majestic stopped by check_unpack_ram anyway -- and asking it + # into upgrade mode first is what this costs on a camera that tight. + # On an XM510 at mem=18M, a majestic started a minute earlier, with + # the 4 MB archive in a tmpfs nothing can reclaim, faulted its own + # pages back after drop_caches during prepare and was OOM-killed; + # the run died with it, leaving the lock and no majestic (#2618). + # + # The watchdog owner is recorded first, so the RAM-phase keeper still + # arms if majestic was it. A stop that is refused or fails falls back + # to prepare, as if nothing had been measured. + wdog_owner_pid=$(watchdog_owner) && wdog_userspace_owned=1 + if majestic_short_for_archive && + stop_majestic_for_memory "$majestic_short_kb" releases; then + : + elif majestic_api prepare; then majestic_prepared=1 echo "majestic stopped its video for the upgrade" else @@ -1892,10 +1989,14 @@ free_resources() { # Said out loud either way: a camera that comes back from a flash with a # corrupted rootfs needs this line to tell an unfed-dog reset apart from a # bad image, and it costs one scan of /proc on the normal system. + # Asked again, now that the services are down; an owner recorded before an + # early majestic stop above keeps the keeper armed. wdog_owner_pid=$(watchdog_owner) && wdog_userspace_owned=1 echo_c 34 "\nWatchdog:" - if [ "1" = "$wdog_userspace_owned" ]; then + if [ "1" = "$wdog_userspace_owned" ] && [ -n "$wdog_owner_pid" ]; then echo "$WDOG: pid $wdog_owner_pid feeds it; taken over if that dies" + elif [ "1" = "$wdog_userspace_owned" ]; then + echo "$WDOG: majestic fed it and was stopped for memory; taken over at the flash" else [ -c "$WDOG" ] && echo "$WDOG: nobody feeds it; left alone" || echo "none" fi @@ -1914,6 +2015,8 @@ free_resources() { # # Started low-number-first, the reverse of the order they were stopped in. restore_resources() { + # Best effort: a /tmp still holding more than the old cap keeps the new one. + [ -n "$tmp_cap_kb" ] && mount -o remount,size=${tmp_cap_kb}k /tmp 2>&3 [ "1" = "$services_stopped" ] || return 0 echo_c 37 "\nRestarting the services stopped for the upgrade" local s @@ -3147,8 +3250,10 @@ reboot_system() { # Same reasoning as die(): this run stays up, so what free_resources # stopped has to come back rather than wait for a reboot that was # explicitly declined. + # The images first, as die() does: they are of no further use, and + # while they sit in /tmp a cap raised for them cannot be put back. + discard_unpacked restore_resources - rm -f "$unpacked_list" rm $LOCK_FILE restore_majestic return 0