From 3b3433616ee4c374c1e2508551b997f4d1a695a2 Mon Sep 17 00:00:00 2001 From: glennmichael123 Date: Wed, 16 Sep 2026 09:38:51 -0700 Subject: [PATCH] perf(android): ship libcraft.so without its DWARF, and keep the DWARF for symbols Every generated app's APK carries the Zig JNI library, and at ReleaseSafe each was about 5.5 MB, most of it debug info nothing read. AGP would strip it with the NDK's strip, but a generated project configures no NDK, so it never did. Release builds now run the NDK's llvm-objcopy after linking: --only-keep-debug writes zig-out/android-symbols//libcraft.so.debug, and --strip-debug --add-gnu-debuglink produces the library the generator installs. Only the DWARF goes: the symbol table stays so a tombstone still names functions, and ndk-stack or Play Console can take the .debug files. Debug builds install the library as built. Not Zig's own ObjCopy step: ELF to ELF copying is fatal("unimplemented") in zig objcopy at 0.17.0-dev.1963, for every strip mode. The NDK is already required to build these libraries at all. The Android E2E runner now refuses to start when the x86_64 library still carries any .debug_ section, has no .gnu_debuglink, or has no symbols file holding .debug_info beside it. It reads the ELF section table itself, and protocol.test.ts covers the shape #204 found. Closes #204 --- .github/workflows/README.md | 5 +- packages/android/README.md | 14 +++++ packages/zig/build.zig | 92 +++++++++++++++++++++++++---- scripts/mobile-e2e/android.ts | 18 +++++- scripts/mobile-e2e/protocol.test.ts | 61 ++++++++++++++++++- scripts/mobile-e2e/protocol.ts | 59 ++++++++++++++++++ 6 files changed, 235 insertions(+), 14 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 590233f4..f96d0e26 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -55,7 +55,10 @@ into the APK and once without: The Zig Android library links bionic, so building it needs the NDK — `sdkmanager --install "ndk;26.1.10909125"`. Without it `build-android` refuses -rather than producing a library that installs and then fails to load. +rather than producing a library that installs and then fails to load. The NDK's +`llvm-objcopy` also strips each release library's DWARF into +`zig-out/android-symbols/`, and the suite refuses to start on a `libcraft.so` +that still carries any. ```bash cd packages/zig && zig build build-android-all -Doptimize=ReleaseSafe -Dandroid-ndk="$ANDROID_NDK_HOME" && cd - diff --git a/packages/android/README.md b/packages/android/README.md index 47f7eff1..a406e1af 100644 --- a/packages/android/README.md +++ b/packages/android/README.md @@ -611,6 +611,20 @@ For automated uploads, integrate with fastlane: fastlane supply --aab ./android/app/build/outputs/bundle/release/app-release.aab ``` +### Native crash symbols + +`zig build build-android-all` strips the DWARF out of each release +`libcraft.so` before the generator copies it into `jniLibs`. With it, the +library was about 5.5 MB per ABI in every APK. The symbol table stays, so a +native crash tombstone still names functions. The DWARF goes to +`zig-out/android-symbols//libcraft.so.debug`, linked from the library by +`.gnu_debuglink`. + +To symbolicate a crash, point `ndk-stack -sym` at that directory. To have Play +Console symbolicate for you, upload the `.debug` files as the release's native +debug symbols. A `-Doptimize=Debug` build keeps the debug info in the library +itself. + ## License MIT diff --git a/packages/zig/build.zig b/packages/zig/build.zig index 5e7fe584..747bc5f9 100644 --- a/packages/zig/build.zig +++ b/packages/zig/build.zig @@ -2471,11 +2471,9 @@ pub fn build(b: *std.Build) void { }); linkAndroidLibc(b, android_arm64_jni, ndk, "aarch64-linux-android"); - const android_arm64_jni_install = b.addInstallArtifact(android_arm64_jni, .{ - .dest_dir = .{ .override = .{ .custom = "android/arm64-v8a" } }, - }); - build_android.dependOn(&android_arm64_jni_install.step); - build_android_all.dependOn(&android_arm64_jni_install.step); + const android_arm64_jni_install = installAndroidJni(b, android_arm64_jni, ndk, "arm64-v8a", debug_build); + build_android.dependOn(android_arm64_jni_install); + build_android_all.dependOn(android_arm64_jni_install); } else { // Not built at all rather than built without bionic. The library would // compile, install, ship in the APK and then fail at `dlopen` with two @@ -2521,11 +2519,9 @@ pub fn build(b: *std.Build) void { // `x86_64`, matching the jniLibs directory the loader looks in — not // `x86`, which is the 32-bit ABI and would be silently ignored. - const android_x86_jni_install = b.addInstallArtifact(android_x86_jni, .{ - .dest_dir = .{ .override = .{ .custom = "android/x86_64" } }, - }); - build_android_x86.dependOn(&android_x86_jni_install.step); - build_android_all.dependOn(&android_x86_jni_install.step); + const android_x86_jni_install = installAndroidJni(b, android_x86_jni, ndk, "x86_64", debug_build); + build_android_x86.dependOn(android_x86_jni_install); + build_android_all.dependOn(android_x86_jni_install); } const android_x86_lib = b.addLibrary(.{ @@ -2618,6 +2614,82 @@ fn androidSysroot(b: *std.Build, ndk: []const u8) ?[]const u8 { return null; } +/// A tool from the NDK's LLVM toolchain, or null when this NDK has none for +/// this host. Found the way `androidSysroot` finds the sysroot, and for the +/// same reason. +fn androidTool(b: *std.Build, ndk: []const u8, name: []const u8) ?[]const u8 { + const io = b.graph.io; + for ([_][]const u8{ "darwin-x86_64", "darwin-arm64", "linux-x86_64", "windows-x86_64" }) |host| { + const exe = if (std.mem.startsWith(u8, host, "windows")) b.fmt("{s}.exe", .{name}) else name; + const candidate = b.pathJoin(&.{ ndk, "toolchains/llvm/prebuilt", host, "bin", exe }); + std.Io.Dir.cwd().access(io, candidate, .{}) catch continue; + return candidate; + } + return null; +} + +/// Install a JNI library where the Android generator looks for it, without its +/// DWARF, and install the DWARF beside it for symbolication (#204). +/// +/// Every generated app's APK carries this library, and at `ReleaseSafe` each +/// one was about 5.5 MB, most of it debug info nothing read: AGP strips native +/// libraries by running the NDK's strip, and a generated project configures no +/// NDK, so it never did. So it happens here, with the same tool AGP would use. +/// +/// `--strip-debug` and not `--strip-all`. The symbol table stays, which is what +/// turns a native crash tombstone's addresses into function names, and it is a +/// small fraction of what the DWARF was. The DWARF goes to +/// `android-symbols//libcraft.so.debug`, linked from the stripped library +/// by `.gnu_debuglink`, which is the shape `ndk-stack` and Play Console's +/// native debug symbols both take. +/// +/// `llvm-objcopy` rather than Zig's own ObjCopy step, because ELF to ELF +/// copying is `fatal("unimplemented")` in `zig objcopy` at 0.17.0-dev.1963; +/// every strip mode fails. The NDK is already required to build these at all. +/// +/// A Debug build installs the library as built: whoever asked for Debug wants +/// the debug info in place. +fn installAndroidJni( + b: *std.Build, + lib: *std.Build.Step.Compile, + ndk: []const u8, + abi: []const u8, + keep_debug_info: bool, +) *std.Build.Step { + const dest = b.fmt("android/{s}", .{abi}); + if (keep_debug_info) { + const install = b.addInstallArtifact(lib, .{ .dest_dir = .{ .override = .{ .custom = dest } } }); + return &install.step; + } + + const objcopy = androidTool(b, ndk, "llvm-objcopy") orelse { + return &b.addFail(b.fmt( + "-Dandroid-ndk={s} has no toolchains/llvm/prebuilt//bin/llvm-objcopy, " ++ + "which strips libcraft.so's debug info for release builds. " ++ + "Point it at an NDK root, or build with -Doptimize=Debug to keep it.", + .{ndk}, + )).step; + }; + + const keep = b.addSystemCommand(&.{ objcopy, "--only-keep-debug" }); + keep.addFileArg(lib.getEmittedBin()); + const debug_info = keep.addOutputFileArg("libcraft.so.debug"); + + const strip = b.addSystemCommand(&.{ objcopy, "--strip-debug" }); + strip.addPrefixedFileArg("--add-gnu-debuglink=", debug_info); + strip.addFileArg(lib.getEmittedBin()); + const stripped = strip.addOutputFileArg("libcraft.so"); + + const install = b.addInstallFileWithDir(stripped, .{ .custom = dest }, "libcraft.so"); + const symbols = b.addInstallFileWithDir( + debug_info, + .{ .custom = b.fmt("android-symbols/{s}", .{abi}) }, + "libcraft.so.debug", + ); + install.step.dependOn(&symbols.step); + return &install.step; +} + /// Link one Android artifact against the NDK's bionic. /// /// A `--libc` file, not include and library paths. Paths alone are what diff --git a/scripts/mobile-e2e/android.ts b/scripts/mobile-e2e/android.ts index fd9172d0..f34a8470 100644 --- a/scripts/mobile-e2e/android.ts +++ b/scripts/mobile-e2e/android.ts @@ -1,8 +1,8 @@ import type { LegOutcome, RunnerOptions } from './types' -import { copyFileSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { copyFileSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs' import { join } from 'node:path' import { init } from '../../packages/android/src/index' -import { androidDeclines, awaitedNeeds, DISMISS_SHARE_MENU, evaluateRun, hasTerminated, runtimePermissionGranted, shareMenuInFront } from './protocol' +import { androidDeclines, awaitedNeeds, DISMISS_SHARE_MENU, elfSectionNames, evaluateRun, hasTerminated, runtimePermissionGranted, shareMenuInFront, strippedLibraryProblems } from './protocol' import { command, driverPage, waitForFile } from './support' /** @@ -321,6 +321,20 @@ export async function runAndroid(options: RunnerOptions): Promise ) } + // What every generated app will ship, checked before any leg uses it. The + // runtime leg asserts the library loads and answers; this asserts it is the + // release shape #204 settled on, stripped with its DWARF kept beside it, + // because a regression there costs every APK megabytes and fails no case. + const symbolsPath = join(runtimeDir, '..', 'android-symbols', 'x86_64', 'libcraft.so.debug') + const libraryProblems = strippedLibraryProblems( + elfSectionNames(new Uint8Array(readFileSync(abi))), + existsSync(symbolsPath) ? elfSectionNames(new Uint8Array(readFileSync(symbolsPath))) : null, + ) + if (libraryProblems.length) + throw new Error(`${abi}: ${libraryProblems.join('; ')}`) + const megabytes = (path: string) => `${(statSync(path).size / 1024 / 1024).toFixed(2)} MB` + console.log(`x86_64/libcraft.so ships at ${megabytes(abi)}; its symbols are ${megabytes(symbolsPath)} beside it`) + const outcomes: LegOutcome[] = [] for (const leg of legs(runtimeDir)) { try { diff --git a/scripts/mobile-e2e/protocol.test.ts b/scripts/mobile-e2e/protocol.test.ts index 77241719..5347afb2 100644 --- a/scripts/mobile-e2e/protocol.test.ts +++ b/scripts/mobile-e2e/protocol.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'bun:test' import { readFileSync } from 'node:fs' import { join } from 'node:path' -import { ANDROID_DECLINE_PHRASES, androidDeclines, awaitedNeeds, deepLinkProblems, deepLinkResults, DISMISS_SHARE_MENU, evaluateRun, hasTerminated, parseDriverOutput, REQUIRED_CASES, requiredCaseProblems, runtimePermissionGranted, shareMenuInFront, ZIG_REFUSED_ACTIONS, ZIG_SERVED_ACTIONS, ZIG_TESTED_ACTIONS, zigDispatchedActions, zigHandBacks, zigRefusals } from './protocol' +import { ANDROID_DECLINE_PHRASES, androidDeclines, awaitedNeeds, deepLinkProblems, deepLinkResults, DISMISS_SHARE_MENU, elfSectionNames, evaluateRun, hasTerminated, parseDriverOutput, REQUIRED_CASES, requiredCaseProblems, runtimePermissionGranted, shareMenuInFront, strippedLibraryProblems, ZIG_REFUSED_ACTIONS, ZIG_SERVED_ACTIONS, ZIG_TESTED_ACTIONS, zigDispatchedActions, zigHandBacks, zigRefusals } from './protocol' const ESC = String.fromCharCode(27) @@ -372,6 +372,65 @@ describe('cold-start deep links', () => { }) }) +/** A minimal ELF64 little-endian file whose section header table names `sections`. */ +function elfWith(sections: string[]): Uint8Array { + const names = ['', ...sections, '.shstrtab'] + const strtab = new TextEncoder().encode(`${names.join('\0')}\0`) + const headerSize = 64 + const entrySize = 64 + const tableOffset = headerSize + strtab.length + const bytes = new Uint8Array(tableOffset + entrySize * names.length) + const view = new DataView(bytes.buffer) + bytes.set([0x7F, 0x45, 0x4C, 0x46, 2, 1, 1], 0) + view.setBigUint64(0x28, BigInt(tableOffset), true) + view.setUint16(0x3A, entrySize, true) + view.setUint16(0x3C, names.length, true) + view.setUint16(0x3E, names.length - 1, true) + bytes.set(strtab, headerSize) + let nameOffset = 0 + names.forEach((name, index) => { + const at = tableOffset + entrySize * index + view.setUint32(at, nameOffset, true) + nameOffset += new TextEncoder().encode(name).length + 1 + if (name === '.shstrtab') { + view.setBigUint64(at + 0x18, BigInt(headerSize), true) + view.setBigUint64(at + 0x20, BigInt(strtab.length), true) + } + }) + return bytes +} + +describe('the shipped Android library', () => { + const stripped = ['.dynsym', '.text', '.symtab', '.gnu_debuglink'] + const debug = ['.debug_info', '.debug_line', '.symtab'] + + it('reads section names out of an ELF64 file, and refuses anything else', () => { + expect(elfSectionNames(elfWith(['.text', '.debug_info']))).toEqual(['', '.text', '.debug_info', '.shstrtab']) + expect(elfSectionNames(new Uint8Array([0x7F, 0x45, 0x4C, 0x46]))).toBeNull() + expect(elfSectionNames(new TextEncoder().encode('not an elf file at all, but long enough to have a header, surely'))).toBeNull() + }) + + it('passes a stripped library with its DWARF moved beside it', () => { + expect(strippedLibraryProblems(elfSectionNames(elfWith(stripped)), elfSectionNames(elfWith(debug)))).toEqual([]) + }) + + // #204 as found: the release library still carrying its debug info. + it('fails a library that still ships DWARF', () => { + const unstripped = elfSectionNames(elfWith(['.dynsym', '.text', '.debug_info', '.debug_str', '.symtab'])) + expect(strippedLibraryProblems(unstripped, elfSectionNames(elfWith(debug)))).toEqual([ + 'libcraft.so still carries .debug_info, .debug_str; release builds ship without DWARF (#204)', + 'libcraft.so has no .gnu_debuglink, so nothing ties it to its symbols file', + ]) + }) + + it('fails a strip that threw the DWARF away instead of keeping it', () => { + expect(strippedLibraryProblems(elfSectionNames(elfWith(stripped)), null)) + .toEqual(['android-symbols has no libcraft.so.debug beside the stripped library']) + expect(strippedLibraryProblems(elfSectionNames(elfWith(stripped)), elfSectionNames(elfWith(['.symtab'])))) + .toEqual(['libcraft.so.debug holds no .debug_info; the strip discarded the DWARF instead of moving it']) + }) +}) + describe('runtime permission state', () => { // As `dumpsys package` prints a coarse-only grant on Android 14. const dumpsys = [ diff --git a/scripts/mobile-e2e/protocol.ts b/scripts/mobile-e2e/protocol.ts index 3ff67efa..1bc3306a 100644 --- a/scripts/mobile-e2e/protocol.ts +++ b/scripts/mobile-e2e/protocol.ts @@ -441,6 +441,65 @@ export function runtimePermissionGranted(dumpsysPackage: string, permission: str return match ? match[1] === 'true' : undefined } +/** + * The section names in an ELF file, or null when `bytes` is not a 64-bit + * little-endian ELF. + * + * Read directly rather than through `readelf`, which a macOS host does not + * have and a Linux runner has only by accident. Every Android ABI the harness + * ships (arm64-v8a, x86_64) is ELF64 little-endian, so that is all this reads. + */ +export function elfSectionNames(bytes: Uint8Array): string[] | null { + const ELF_MAGIC = [0x7F, 0x45, 0x4C, 0x46] + if (bytes.length < 64 || ELF_MAGIC.some((byte, index) => bytes[index] !== byte)) return null + if (bytes[4] !== 2 || bytes[5] !== 1) return null + + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) + const sectionTable = Number(view.getBigUint64(0x28, true)) + const entrySize = view.getUint16(0x3A, true) + const count = view.getUint16(0x3C, true) + const namesIndex = view.getUint16(0x3E, true) + if (entrySize < 0x28 || sectionTable + entrySize * count > bytes.length || namesIndex >= count) return null + + const header = (index: number) => sectionTable + entrySize * index + const namesOffset = Number(view.getBigUint64(header(namesIndex) + 0x18, true)) + const namesSize = Number(view.getBigUint64(header(namesIndex) + 0x20, true)) + if (namesOffset + namesSize > bytes.length) return null + + const names: string[] = [] + for (let index = 0; index < count; index++) { + const start = namesOffset + view.getUint32(header(index), true) + let end = start + while (end < namesOffset + namesSize && bytes[end] !== 0) end++ + names.push(new TextDecoder().decode(bytes.subarray(start, end))) + } + return names +} + +/** + * Why a shipped `libcraft.so` and its symbols file are not what #204 settled + * on, one line per reason. + * + * The library must carry no DWARF: every generated app's APK includes it, and + * with DWARF it was about 5.5 MB per ABI. It must carry `.gnu_debuglink`, so a + * crash can be matched to the symbols file. And the symbols file must actually + * hold the DWARF, or the strip threw it away rather than moving it. + */ +export function strippedLibraryProblems(library: string[] | null, symbols: string[] | null): string[] { + const problems: string[] = [] + if (!library) return ['libcraft.so is not an ELF64 little-endian file'] + const dwarf = library.filter(name => name.startsWith('.debug_')) + if (dwarf.length) + problems.push(`libcraft.so still carries ${dwarf.join(', ')}; release builds ship without DWARF (#204)`) + if (!library.includes('.gnu_debuglink')) + problems.push('libcraft.so has no .gnu_debuglink, so nothing ties it to its symbols file') + if (!symbols) + problems.push('android-symbols has no libcraft.so.debug beside the stripped library') + else if (!symbols.includes('.debug_info')) + problems.push('libcraft.so.debug holds no .debug_info; the strip discarded the DWARF instead of moving it') + return problems +} + /** * The actions the Zig dispatcher saw, by name. *