diff --git a/Cargo.lock b/Cargo.lock index 75851af..17e8f55 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,9 +21,9 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.19.0" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b796ba3c974e5133f6ca1a2c4e97c5941ea1a46d202df83344afaa18219f1a62" +checksum = "cd4ef186188fed2d4f8177d712ad623c751c6e84f45b31da694cc53f483f320b" dependencies = [ "actix-codec", "actix-service", @@ -122,9 +122,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.17.0" +version = "4.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67756b0fc99e07b8702b9a856e3d6ec036387243c2ce6310070d1e8f02680bb0" +checksum = "697d10a06f7a550791dcb963e3090aa5c65c4d94c029114814a4bd652f30f0b4" dependencies = [ "actix-codec", "actix-http", @@ -164,9 +164,9 @@ dependencies = [ [[package]] name = "actix-web-codegen" -version = "4.4.0" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96b09c4878563f8ab4a5fd0c59f9f0d6e0e9f60eb9b748526a0b9604fd89c50" +checksum = "f7c9f4c52b954a73c76757a3952d3437fca00a2fbcb49704f8a6ea9f767e73f0" dependencies = [ "actix-router", "proc-macro2", @@ -610,9 +610,9 @@ dependencies = [ [[package]] name = "ctutils" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" +checksum = "38d28fb6aa7dd680b3ffd1a3e64e115c177bdba63c0ae7f9cd48129ff05c4fa0" dependencies = [ "cmov", ] @@ -847,7 +847,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1528,21 +1528,6 @@ version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" -[[package]] -name = "libfs" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b79c3145dcaad30b6d7b1e30c9d2d23799b882e9a2e9d1fc7bcc0699c5279ff9" -dependencies = [ - "cfg-if", - "libc", - "linux-raw-sys", - "log", - "rustix", - "thiserror 2.0.21", - "xattr", -] - [[package]] name = "libredox" version = "0.1.25" @@ -1763,7 +1748,7 @@ dependencies = [ [[package]] name = "omnect-cli" -version = "0.28.5" +version = "0.29.0" dependencies = [ "actix-web", "anyhow", @@ -1782,7 +1767,7 @@ dependencies = [ "hmac", "httpmock", "keyring-core", - "libfs", + "libc", "log", "mbrman", "num_cpus", @@ -1793,10 +1778,13 @@ dependencies = [ "regex", "reqwest 0.13.5", "ring", + "roxmltree", + "rustix", "serde", "serde_json", "serde_path_to_error", "sha2 0.11.1", + "signal-hook", "stdext", "strum", "strum_macros", @@ -2222,6 +2210,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "roxmltree" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9da7f59328b09405b181fee1018925fad50f157afb43c0c0f5c522c256dd5907" +dependencies = [ + "memchr", +] + [[package]] name = "rustc_version" version = "0.4.1" @@ -2250,7 +2247,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2451,6 +2448,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f7447afd772ca79ad17ae0a3972a6b0df0cb4b80990fd82707593e9e012a74" +dependencies = [ + "libc", + "signal-hook-registry", +] + [[package]] name = "signal-hook-registry" version = "1.4.8" @@ -2644,7 +2651,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index c7db19b..d9077bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ license = "MIT OR Apache-2.0" name = "omnect-cli" readme = "README.md" repository = "https://github.com/omnect/omnect-cli" -version = "0.28.5" +version = "0.29.0" [dependencies] actix-web = { version = "4.17", default-features = false, features = ["macros"] } @@ -31,7 +31,7 @@ hmac = { version = "0.13", default-features = false } mbrman = { version = "0.6", default-features = false } omnect-crypto = { git = "https://github.com/omnect/omnect-crypto.git", tag = "0.5.0" } keyring-core = { version = "1.0", default-features = false } -libfs = { version = "0.9", default-features = false } +libc = { version = "0.2", default-features = false } log = { version = "0.4", default-features = false } num_cpus = { version = "1.17", default-features = false } oauth2 = { version = "5.0", default-features = false, features = [ @@ -44,10 +44,13 @@ reqwest = { version = "0.13", default-features = false, features = [ "json", "native-tls", ] } +roxmltree = { version = "0.21", default-features = false, features = ["std"] } +rustix = { version = "1.1", default-features = false, features = ["fs", "std"] } serde = { version = "1.0", default-features = false, features = ["derive"] } serde_json = { version = "1.0", default-features = false } serde_path_to_error = { version = "0.1", default-features = false } sha2 = { version = "0.11", default-features = false } +signal-hook = { version = "0.4", default-features = false, features = ["iterator"] } stdext = { version = "0.3", default-features = false } strum = { version = "0.28", default-features = false } strum_macros = { version = "0.28", default-features = false } @@ -81,5 +84,5 @@ tar = "0.4" # metadata for building with cargo-deb (https://crates.io/crates/cargo-deb) [package.metadata.deb] -depends = "bmap-tools, e2tools, keychain, libc6 (>= 2.39), libmagic1, libssl3 (>= 3.0.0), mtools" +depends = "e2tools, keychain, libc6 (>= 2.39), libmagic1, libssl3 (>= 3.0.0), mtools" revision = "" diff --git a/Dockerfile b/Dockerfile index add2cbf..91ba58a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,6 @@ COPY ${debian_dir}/omnect-cli_${omnect_cli_version}_amd64.deb omnect-cli_${omnec RUN apt-get update && \ apt-get install -y --no-install-recommends \ - bmap-tools \ ca-certificates \ e2tools \ fdisk \ @@ -34,14 +33,11 @@ RUN < - **Note2**: The ssh tunnel option requires some additional settings. See [here](Usage-with-docker) for more details.
- **Note3**: The docker inject command is not supported by omnect-cli docker image.
. + **Note1**: The ssh tunnel option requires some additional settings. See [here](Usage-with-docker) for more details.
+ **Note2**: The docker inject command is not supported by omnect-cli docker image.
. # Build from sources @@ -184,6 +183,22 @@ omnect-cli file copy-to-image --help - File permissions: inject `systemd-tmpfiles.d` - Wifi: inject `wpa_supplicant-wlan0.conf` +## Output image + +A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gzip`), e.g. `my-image.wic.xz` for `my-image.wic`. An image that already has an extension of the compression keeps its name, except that `.bzip2` becomes `.bz2`, because `bmaptool` does not read `.bzip2`. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. With `-p` its name must end in an extension of the compression that `bmaptool` reads (`.xz`, `.bz2`, `.gzip` or `.gz`), and without `-p` it must not have a compression extension. + +The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. An existing output image must be a regular file, so `-o` cannot name a device or a directory; to write a device, flash the image with `bmaptool copy`. It also fails before it starts if the output image or its bmap exists and is not writable for the user, or if the directory of the new file is not writable. The output image is written to a new file in its directory and then renamed, so the directory needs write access and space for one more image. An existing output image in a directory you cannot write is staged in `TMPDIR` instead and copied into it. Until the rename, the new file is in a hidden `.omnect-cli-*` directory. The command removes it when it ends or is stopped with Ctrl-C, `SIGTERM` or `SIGHUP`. It stays after a `SIGKILL` or a power loss, and you can delete it. A rename cannot replace a mount point, e.g. an image mounted with `docker run -v "$(pwd)/my-image.wic":/source/my-image.wic`, so the new image is staged in `TMPDIR` and copied into it. If this copy fails or is stopped, the new image stays, and the log names it. If the copy fails, the new bmap of `-b` stays next to it. With `-b` the command fails for such a mounted image, because the bmap would be written next to the mount point and not next to the image. Mount the directory instead. The new file belongs to the user who runs `omnect-cli`. `copy-from-image` only reads the image and writes nothing. + +A command copies each partition it reads or changes to `TMPDIR` (default `/tmp`). A command that changes an image, or reads a compressed one, also copies the uncompressed image there, and for an output image that is a mount point or in a directory you cannot write also the new image. So `TMPDIR` needs space for all of them. Ctrl-C, `SIGTERM` or `SIGHUP` removes them too. If `/tmp` is a small tmpfs, set `TMPDIR` to a directory with more space. + +## bmap files + +Commands that change an image create a bmap file with `-b` (`--output-bmap`). It is written next to the output image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic` and `my-image.wic.xz.bmap` for `my-image.wic.xz`, and replaces a file with this name. An image changed in place keeps the name of its bmap: the command replaces the bmap that `bmaptool` finds for it, e.g. `my-image.wic.bmap` for `-i my-image.wic.xz -p xz`. `bmaptool` finds the bmap there without `--bmap`. The command fails if `bmaptool` would use this file also for an image that stays (`` or `.` next to it, for which no other bmap comes first), because it would not match that image. E.g. `my-image.wic.bmap` when `-i my-image.wic.xz` writes `my-image.wic`, also if the bmap does not exist yet, or when `-i my-image.wic` changes the image in place and `my-image.wic.xz` without its own bmap is next to it. Choose another output with `-o` then. + +The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else the bmap next to the image that `bmaptool` would use: `.bmap`, else the same with one extension of the image removed at a time, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image and sha256 range checksums that match it, otherwise the command fails. A command without `-b` fails if such a bmap exists for the output image (e.g. `my-image.wic.bmap` for `my-image.wic.xz`), because `bmaptool` would use it and it would not match the image. So when you run several commands on one image, use `-b` on each of them. `--no-input-bmap` skips a bmap next to the image that does not match, and the created bmap then maps the whole image. The created bmap maps every range of the input bmap, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). It also maps every block that the image file stores as data, which includes the blocks `omnect-cli` changed. The image file itself stays sparse. + +Without an input bmap, the created bmap maps the whole image, and a flash writes every block. + ## ssh tunnel ### Inject ssh tunnel credentials diff --git a/project-context.md b/project-context.md index f1dd9ea..b683176 100644 --- a/project-context.md +++ b/project-context.md @@ -50,7 +50,11 @@ - `src/docker.rs` — `docker pull --platform` + `docker save` for multi-arch images - `src/image.rs` — firmware image architecture detection (ARM32/ARM64/x86_64) - `src/file/mod.rs` — high-level image operations: identity config, certs, hostname patching -- `src/file/functions.rs` — partition read/modify/write via dd + e2cp/mcopy +- `src/file/functions.rs` — sparse partition read, modify via e2cp/mcopy, write back of changed blocks only +- `src/file/bmap.rs` — input bmap lookup and checksum check; writes the output bmap: input ranges (or the whole image) plus data blocks +- `src/file/output.rs` — output image name, conflict checks, staged write and commit of the output image and bmap +- `src/file/sparse.rs` — data/hole ranges via `SEEK_DATA`/`SEEK_HOLE`, sparse range copy +- `src/file/signal.rs` — `RegisteredTempDir`: a temp dir that SIGINT/SIGTERM/SIGHUP (unless ignored) also removes - `src/file/partition.rs` — GPT/MBR partition table parsing - `src/file/compression.rs` — xz/bzip2/gzip compress/decompress with auto-detection - `src/validators/` — validation for identity config (TOML), device-update config (JSON), SSH keys @@ -58,7 +62,7 @@ ## 4. Repository-Specific Constraints -- External tools `e2cp`, `e2mkdir`, `mcopy`, `mmd`, `dd`, `fallocate`, `ssh-keygen`, `fdisk` must be available at runtime (Dockerfile copies them explicitly). +- External tools `e2cp`, `e2mkdir`, `mcopy`, `mmd`, `ssh-keygen`, `fdisk` must be available at runtime (Dockerfile copies them explicitly). - Partition enum maps partition names to numbers differently for GPT vs MBR — see `file/functions.rs`. - OAuth2 callback binds to `127.0.0.1:4000` and `[::1]:4000`; container mode overrides to `0.0.0.0`. - `conf/` directory uses `.gitignore` to track only `*.template` files — actual configs are generated, never committed. diff --git a/src/cli.rs b/src/cli.rs index 746e253..629e833 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -2,12 +2,35 @@ use crate::file::{ compression::Compression, functions::{FileCopyFromParams, FileCopyToParams, Partition}, }; -use clap::Parser; +use clap::{Args, Parser}; use std::path::PathBuf; use url::Url; const COPYRIGHT: &str = "Copyright © 2021 by conplement AG"; +#[derive(Args, Debug)] +pub struct ImageOutput { + /// optional: create a bmap of the output image + #[arg(short = 'b', long = "output-bmap")] + pub output_bmap: bool, + /// optional: bmap of the input image; default: the bmap next to the image that bmaptool would use + #[arg(long = "input-bmap", requires = "output_bmap")] + pub input_bmap: Option, + /// optional: do not use a bmap next to the image, so the bmap maps the whole image + #[arg( + long = "no-input-bmap", + requires = "output_bmap", + conflicts_with = "input_bmap" + )] + pub no_input_bmap: bool, + /// optional: write the output image to this path, ending in .xz, .bz2, .gzip or .gz with --pack-image, else without a compression extension; default: the input image, without the compression extension, plus the --pack-image extension + #[arg(short = 'o', long = "output-image")] + pub output_image: Option, + /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') + #[arg(short = 'p', long = "pack-image", value_enum)] + pub compress_image: Option, +} + // ToDo: command completion #[derive(Parser, Debug)] #[command(after_help = COPYRIGHT)] @@ -34,12 +57,8 @@ pub enum Docker { /// destination path of the docker image in the firmware image (must end in ".tar.gz") #[clap(short = 'e', long = "dest")] dest: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, } @@ -55,12 +74,8 @@ pub enum File { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// copy files from image CopyFromImage { @@ -88,12 +103,8 @@ pub enum IdentityConfig { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// EXPERIMENTAL: set transparent gateway config.toml file and additional certificates and keys SetIotedgeGatewayConfig { @@ -112,12 +123,8 @@ pub enum IdentityConfig { /// path to device identity certificate key file #[arg(short = 'k', long = "device_identity_key")] device_identity_key: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// EXPERIMENTAL: set leaf device config.toml file and additional certificate SetIotLeafSasConfig { @@ -130,12 +137,8 @@ pub enum IdentityConfig { /// path to root ca certificate file #[arg(short = 'r', long = "root_ca")] root_ca: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// set certificates in order to support X.509 based DPS provisioning and certificate renewal via EST SetDeviceCertificate { @@ -154,12 +157,8 @@ pub enum IdentityConfig { /// period of validity in days #[arg(short = 'D', long = "days")] days: u32, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// set certificates in order to support X.509 based DPS provisioning WITHOUT certificate renewal via EST SetDeviceCertificateNoEst { @@ -172,12 +171,8 @@ pub enum IdentityConfig { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// generate and set bootstrap certificate for edge ca issuance/renewal. SetEdgeCaCertificate { @@ -196,12 +191,8 @@ pub enum IdentityConfig { /// period of validity in days #[arg(short = 'D', long = "days")] days: u32, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, } @@ -217,12 +208,8 @@ pub enum IotHubDeviceUpdate { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// import update to azure iot-hub ImportUpdate { @@ -336,12 +323,8 @@ pub enum SshConfig { /// path to public key of the ssh root ca #[arg(short = 'r', long = "root_ca")] root_ca: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// set ssh connection parameters (see diff --git a/src/file/bmap.rs b/src/file/bmap.rs new file mode 100644 index 0000000..4dd1453 --- /dev/null +++ b/src/file/bmap.rs @@ -0,0 +1,592 @@ +use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, sparse}; +use anyhow::{Context, Result}; +use sha2::{Digest, Sha256}; +use std::{ + fs, + io::Write, + ops::{Range, RangeInclusive}, + os::unix::fs::FileExt, + path::{Path, PathBuf}, +}; + +const BMAP_EXTENSION: &str = "bmap"; +const SHA256_CHECKSUM_TYPE: &str = "sha256"; +const SHA256_HEX_LEN: usize = 64; +const BMAP_VERSION: &str = "2.0"; +const BMAP_MAJOR_VERSION: u32 = 2; +const BMAP_ROOT: &str = "bmap"; + +/// The block ranges of an image that a bmap file maps. +#[derive(Debug)] +pub struct Bmap { + pub block_size: u64, + pub image_size: u64, + pub ranges: Vec>, + /// The sha256 of each range, if the bmap has them. + pub sha256: Option>, +} + +impl Bmap { + pub fn read(path: &Path) -> Result { + let text = fs::read_to_string(path) + .context(format!("Bmap::read: cannot read {}", path.display()))?; + let doc = roxmltree::Document::parse(&text) + .context(format!("Bmap::read: invalid xml in {}", path.display()))?; + let root = doc.root_element(); + anyhow::ensure!( + root.has_tag_name(BMAP_ROOT), + "Bmap::read: {} is not a bmap", + path.display() + ); + let major_version = root + .attribute("version") + .and_then(|v| v.split('.').next()?.parse::().ok()) + .context(format!( + "Bmap::read: no valid version in {}", + path.display() + ))?; + // like bmaptool, refuse a bmap of a newer major version + anyhow::ensure!( + major_version <= BMAP_MAJOR_VERSION, + "Bmap::read: {} has version {major_version}.x, only up to {BMAP_MAJOR_VERSION}.x is \ + supported", + path.display() + ); + let child = |name: &str| { + root.children() + .find(|n| n.has_tag_name(name)) + .context(format!("Bmap::read: no {name} in {}", path.display())) + }; + let number = |name: &str| -> Result { + child(name)? + .text() + .unwrap_or_default() + .trim() + .parse() + .context(format!("Bmap::read: invalid {name} in {}", path.display())) + }; + + let block_size = number("BlockSize")?; + anyhow::ensure!(block_size > 0, "Bmap::read: BlockSize is 0"); + let image_size = number("ImageSize")?; + let range_nodes: Vec<_> = child("BlockMap")? + .children() + .filter(|n| n.has_tag_name("Range")) + .collect(); + let ranges = range_nodes + .iter() + .map(|n| parse_range(n.text().unwrap_or_default())) + .collect::>()?; + let sha256 = child("ChecksumType") + .ok() + .and_then(|n| n.text()) + .is_some_and(|t| t.trim() == SHA256_CHECKSUM_TYPE) + .then(|| { + range_nodes + .iter() + .map(|n| n.attribute("chksum").map(str::to_owned)) + .collect::>() + }) + .flatten(); + + Ok(Self { + block_size, + image_size, + ranges, + sha256, + }) + } +} + +fn parse_range(text: &str) -> Result> { + let text = text.trim(); + let (first, last) = text.split_once('-').unwrap_or((text, text)); + let parse = |s: &str| { + s.trim() + .parse::() + .context(format!("parse_range: invalid range {text}")) + }; + let (first, last) = (parse(first)?, parse(last)?); + anyhow::ensure!(first <= last, "parse_range: invalid range {text}"); + + Ok(first..=last) +} + +/// Returns `explicit`, else the bmap `bmaptool` uses for `image`: `.bmap`, else the same +/// with one extension of `image` removed at a time, whichever exists first. +pub fn find_input_bmap(image: &Path, explicit: Option<&Path>) -> Result> { + if let Some(path) = explicit { + anyhow::ensure!( + path.is_file(), + "find_input_bmap: {} not found", + path.display() + ); + return Ok(Some(path.to_path_buf())); + } + + Ok(bmaptool_lookup(image) + .into_iter() + .find(|bmap| bmap.is_file())) +} + +/// The paths where `bmaptool` looks for the bmap of `image`, in its order: `.bmap`, then +/// the same with one extension of `image` removed at a time. +pub fn bmaptool_lookup(image: &Path) -> Vec { + let mut image = image.to_path_buf(); + let mut bmaps = vec![bmap_path(&image)]; + while image.extension().is_some() { + image.set_extension(""); + bmaps.push(bmap_path(&image)); + } + bmaps +} + +pub fn bmap_path(image: &Path) -> PathBuf { + image.with_added_extension(BMAP_EXTENSION) +} + +/// Fails if a range of `image_file` does not match its checksum in `bmap`, so the bmap of +/// another image with the same size is not used. +pub fn verify_checksums(image_file: &Path, bmap: &Bmap) -> Result<()> { + let sha256 = bmap + .sha256 + .as_ref() + .context("verify_checksums: the bmap has no sha256 range checksums")?; + let image = fs::File::open(image_file).context(format!( + "verify_checksums: cannot open {}", + image_file.display() + ))?; + let image_len = image + .metadata() + .context("verify_checksums: cannot get image size")? + .len(); + check_layout(bmap, image_len)?; + + let mut buf = vec![0u8; COPY_CHUNK_SIZE]; + for (range, expected) in bmap.ranges.iter().zip(sha256) { + let bytes = + range.start() * bmap.block_size..((range.end() + 1) * bmap.block_size).min(image_len); + let actual = range_sha256(&image, bytes, &mut buf)?; + anyhow::ensure!( + actual.eq_ignore_ascii_case(expected.trim()), + "verify_checksums: range {range:?} of the image does not match the bmap, so it is the \ + bmap of another image or of an older version of this image" + ); + } + + Ok(()) +} + +/// Writes a bmap of `image_file` to `out`. It maps the ranges of `input`, or the whole +/// image without one, and every block that the image file stores as data. A range of `input` +/// can be all zeros and a hole in the image, and a bmap flash still has to write it, else the +/// device keeps what an earlier installation left there. +pub fn write_bmap(image_file: &Path, input: Option<&Bmap>, out: &mut impl Write) -> Result<()> { + let image = fs::File::open(image_file) + .context(format!("write_bmap: cannot open {}", image_file.display()))?; + let image_len = image + .metadata() + .context("write_bmap: cannot get image size")? + .len(); + let block_size = IMAGE_BLOCK_SIZE as u64; + + let ranges = mapped_ranges(&image, image_len, input)?; + + let mut buf = vec![0u8; COPY_CHUNK_SIZE]; + let mut block_map = String::new(); + for range in &ranges { + let bytes = range.start() * block_size..((range.end() + 1) * block_size).min(image_len); + let sha256 = range_sha256(&image, bytes, &mut buf)?; + block_map.push_str(&format!( + " {}-{} \n", + range.start(), + range.end() + )); + } + let blocks = image_len.div_ceil(block_size); + let mapped: u64 = ranges.iter().map(|r| r.end() - r.start() + 1).sum(); + + // like bmaptool, the file checksum is taken with zeros in place of itself + let zeros = "0".repeat(SHA256_HEX_LEN); + let xml = format!( + "\n\ + \n \ + {image_len} \n \ + {block_size} \n \ + {blocks} \n \ + {mapped} \n \ + {SHA256_CHECKSUM_TYPE} \n \ + {zeros} \n \ + \n{block_map} \n\ + \n" + ); + let xml = xml.replacen(&zeros, &hex(&Sha256::digest(xml.as_bytes())), 1); + + out.write_all(xml.as_bytes()) + .context("write_bmap: cannot write the bmap") +} + +fn mapped_ranges( + image: &fs::File, + image_len: u64, + input: Option<&Bmap>, +) -> Result>> { + let block_size = IMAGE_BLOCK_SIZE as u64; + let mut ranges: Vec> = match input { + Some(bmap) => { + check_layout(bmap, image_len)?; + bmap.ranges + .iter() + .map(|r| r.start() * bmap.block_size..(r.end() + 1) * bmap.block_size) + .collect() + } + None => vec![Range { + start: 0, + end: image_len, + }], + }; + ranges.extend(sparse::data_ranges(image, 0, image_len)?); + Ok(sparse::merge( + ranges + .into_iter() + .map(|r| { + r.start / block_size * block_size..r.end.min(image_len).next_multiple_of(block_size) + }) + .filter(|r| !r.is_empty()) + .collect(), + ) + .into_iter() + .map(|r| r.start / block_size..=r.end / block_size - 1) + .collect()) +} + +fn range_sha256(image: &fs::File, bytes: Range, buf: &mut [u8]) -> Result { + let mut hasher = Sha256::new(); + let mut pos = bytes.start; + while pos < bytes.end { + let n = usize::try_from((bytes.end - pos).min(buf.len() as u64))?; + image + .read_exact_at(&mut buf[..n], pos) + .context("range_sha256: cannot read image")?; + hasher.update(&buf[..n]); + pos += n as u64; + } + + Ok(hex(&hasher.finalize())) +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// Fails if `bmap` is for an image of another size or maps blocks outside the image. +fn check_layout(bmap: &Bmap, image_len: u64) -> Result<()> { + anyhow::ensure!( + bmap.image_size == image_len, + "check_layout: bmap is for an image of {} bytes, the image has {image_len}", + bmap.image_size + ); + let image_blocks = image_len.div_ceil(bmap.block_size); + if let Some(range) = bmap.ranges.iter().find(|r| *r.end() >= image_blocks) { + anyhow::bail!("check_layout: range {range:?} is outside the image"); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use crate::file::bmap::*; + + const BLOCK_SIZE: u64 = IMAGE_BLOCK_SIZE as u64; + const IMAGE_BLOCKS: u64 = 16; + const DATA_BLOCK: u64 = 0; + const ZERO_BLOCKS: RangeInclusive = 2..=3; + const UNMAPPED_BLOCK: u64 = 5; + const SMALL_BLOCK_SIZE: u64 = 512; + const PARTIAL_BLOCK_LEN: usize = 100; + const DATA_BYTE: u8 = 0x5a; + // content an earlier installation left on the device + const OLD_CONTENT_BYTE: u8 = 0xa5; + const YOCTO_BMAP: &str = "testfiles/image.wic.bmap"; + const YOCTO_IMAGE_SIZE: u64 = 30408704; + const YOCTO_MAPPED_BLOCKS: u64 = 77; + + fn bmap_xml(image_size: u64, block_map: &str) -> String { + format!( + "{image_size}\ + {BLOCK_SIZE}{block_map}" + ) + } + + fn write_input_bmap(path: &Path, image_size: u64, ranges: &[RangeInclusive]) { + let block_map: String = ranges + .iter() + .map(|r| format!("{}-{}", r.start(), r.end())) + .collect(); + fs::write(path, bmap_xml(image_size, &block_map)).expect("write bmap"); + } + + /// A sparse image with data in `DATA_BLOCK` and holes everywhere else. + fn sparse_image(dir: &Path) -> PathBuf { + let path = dir.join("image.wic"); + let image = fs::File::create(&path).expect("create image"); + image + .set_len(IMAGE_BLOCKS * BLOCK_SIZE) + .expect("set image size"); + image + .write_all_at(&[DATA_BYTE; IMAGE_BLOCK_SIZE], DATA_BLOCK * BLOCK_SIZE) + .expect("write data block"); + path + } + + fn created_bmap(image: &Path, input: Option<&Bmap>) -> Bmap { + let path = image.with_extension(BMAP_EXTENSION); + let mut file = fs::File::create(&path).expect("create bmap"); + write_bmap(image, input, &mut file).expect("write bmap"); + Bmap::read(&path).expect("read bmap") + } + + fn is_mapped(bmap: &Bmap, block: u64) -> bool { + bmap.ranges.iter().any(|r| r.contains(&block)) + } + + /// Writes the blocks `bmap` maps onto a device that still holds old content, as a bmap + /// flash does. + fn bmap_flash(image: &Path, bmap: &Bmap, device: &Path) { + let image = fs::read(image).expect("read image"); + let mut content = vec![OLD_CONTENT_BYTE; image.len()]; + for range in &bmap.ranges { + let start = usize::try_from(range.start() * bmap.block_size).expect("start"); + let end = usize::try_from((range.end() + 1) * bmap.block_size).expect("end"); + let end = end.min(image.len()); + content[start..end].copy_from_slice(&image[start..end]); + } + fs::write(device, content).expect("write device"); + } + + #[test] + fn read_parses_yocto_bmap() { + let bmap = Bmap::read(Path::new(YOCTO_BMAP)).expect("read bmap"); + + assert_eq!(bmap.block_size, BLOCK_SIZE); + assert_eq!(bmap.image_size, YOCTO_IMAGE_SIZE); + let mapped: u64 = bmap.ranges.iter().map(|r| r.end() - r.start() + 1).sum(); + assert_eq!(mapped, YOCTO_MAPPED_BLOCKS); + assert_eq!( + bmap.sha256.map(|c| c.len()), + Some(bmap.ranges.len()), + "one checksum per range" + ); + } + + #[test] + fn read_parses_single_blocks_and_spans() { + let dir = tempfile::tempdir().expect("create temp dir"); + let path = dir.path().join("image.wic.bmap"); + let block_map = " 3 5-7 "; + fs::write(&path, bmap_xml(BLOCK_SIZE, block_map)).expect("write bmap"); + + let bmap = Bmap::read(&path).expect("read bmap"); + assert_eq!(bmap.ranges, vec![3..=3, 5..=7]); + } + + #[test] + fn read_has_no_checksums_without_sha256_checksum_type() { + let dir = tempfile::tempdir().expect("create temp dir"); + let path = dir.path().join("image.wic.bmap"); + fs::write(&path, bmap_xml(BLOCK_SIZE, "0")).expect("write bmap"); + + assert_eq!(Bmap::read(&path).expect("read bmap").sha256, None); + } + + #[test] + fn read_rejects_invalid_bmaps() { + let dir = tempfile::tempdir().expect("create temp dir"); + let path = dir.path().join("image.wic.bmap"); + let invalid = [ + "1".to_string(), + bmap_xml(BLOCK_SIZE, "7-5"), + bmap_xml(BLOCK_SIZE, "x"), + "".to_string(), + bmap_xml(BLOCK_SIZE, "") + .replace("", ""), + bmap_xml(BLOCK_SIZE, "").replace("version=\"2.0\"", "version=\"3.0\""), + bmap_xml(BLOCK_SIZE, "").replace(" version=\"2.0\"", ""), + ]; + + for xml in invalid { + fs::write(&path, &xml).expect("write bmap"); + assert!(Bmap::read(&path).is_err(), "accepted {xml}"); + } + } + + #[test] + fn find_input_bmap_takes_explicit_path() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + let explicit = dir.path().join("other.bmap"); + + assert!(find_input_bmap(&image, Some(&explicit)).is_err()); + fs::write(&explicit, "").expect("write bmap"); + assert_eq!( + find_input_bmap(&image, Some(&explicit)).expect("find bmap"), + Some(explicit) + ); + } + + #[test] + fn find_input_bmap_looks_next_to_the_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic.xz"); + let uncompressed_bmap = dir.path().join("image.wic.bmap"); + let compressed_bmap = dir.path().join("image.wic.xz.bmap"); + + assert_eq!(find_input_bmap(&image, None).expect("find bmap"), None); + fs::write(&uncompressed_bmap, "").expect("write bmap"); + assert_eq!( + find_input_bmap(&image, None).expect("find bmap"), + Some(uncompressed_bmap) + ); + fs::write(&compressed_bmap, "").expect("write bmap"); + assert_eq!( + find_input_bmap(&image, None).expect("find bmap"), + Some(compressed_bmap) + ); + } + + #[test] + fn find_input_bmap_strips_extensions_like_bmaptool() { + let dir = tempfile::tempdir().expect("create temp dir"); + let bmap = dir.path().join("image.bmap"); + fs::write(&bmap, "").expect("write bmap"); + + for image in ["image.wic", "image.wic.xz"] { + assert_eq!( + find_input_bmap(&dir.path().join(image), None).expect("find bmap"), + Some(bmap.clone()) + ); + } + } + + #[test] + fn write_bmap_maps_zero_ranges_of_the_input_bmap() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + let input_bmap = dir.path().join("input.bmap"); + write_input_bmap(&input_bmap, IMAGE_BLOCKS * BLOCK_SIZE, &[ZERO_BLOCKS]); + let input = Bmap::read(&input_bmap).expect("read bmap"); + + let bmap = created_bmap(&image, Some(&input)); + assert!(ZERO_BLOCKS.clone().all(|b| is_mapped(&bmap, b))); + assert!( + is_mapped(&bmap, DATA_BLOCK), + "data outside the input ranges" + ); + assert!(!is_mapped(&bmap, UNMAPPED_BLOCK)); + let file = fs::File::open(&image).expect("open image"); + assert_eq!( + sparse::data_ranges(&file, 0, IMAGE_BLOCKS * BLOCK_SIZE).expect("data ranges"), + vec![DATA_BLOCK * BLOCK_SIZE..(DATA_BLOCK + 1) * BLOCK_SIZE], + "image is not sparse any more" + ); + + let device = dir.path().join("device"); + bmap_flash(&image, &bmap, &device); + let device = fs::read(&device).expect("read device"); + let zero_start = usize::try_from(ZERO_BLOCKS.start() * BLOCK_SIZE).expect("start"); + let zero_end = usize::try_from((ZERO_BLOCKS.end() + 1) * BLOCK_SIZE).expect("end"); + assert!(device[zero_start..zero_end].iter().all(|b| *b == 0)); + } + + #[test] + fn write_bmap_without_input_maps_whole_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + + assert_eq!( + created_bmap(&image, None).ranges, + vec![0..=IMAGE_BLOCKS - 1] + ); + } + + #[test] + fn write_bmap_maps_whole_blocks_for_a_smaller_bmap_block_size() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + // starts in DATA_BLOCK and ends in the hole after it + let first = (DATA_BLOCK + 1) * BLOCK_SIZE / SMALL_BLOCK_SIZE - 1; + let input = Bmap { + block_size: SMALL_BLOCK_SIZE, + image_size: IMAGE_BLOCKS * BLOCK_SIZE, + ranges: vec![RangeInclusive::new(first, first + 2)], + sha256: None, + }; + + assert!(is_mapped( + &created_bmap(&image, Some(&input)), + DATA_BLOCK + 1 + )); + } + + #[test] + fn verify_checksums_accepts_bmap_of_the_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + // a mapped last block that is shorter than the block size + fs::OpenOptions::new() + .write(true) + .open(&image) + .expect("open image") + .write_all_at(&[DATA_BYTE; PARTIAL_BLOCK_LEN], IMAGE_BLOCKS * BLOCK_SIZE) + .expect("write partial block"); + + verify_checksums(&image, &created_bmap(&image, None)).expect("verify checksums"); + } + + #[test] + fn verify_checksums_rejects_changed_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + let bmap = created_bmap(&image, None); + fs::OpenOptions::new() + .write(true) + .open(&image) + .expect("open image") + .write_all_at(&[!DATA_BYTE], DATA_BLOCK * BLOCK_SIZE) + .expect("change data block"); + + let err = verify_checksums(&image, &bmap).expect_err("changed image accepted"); + assert!(err.to_string().contains("does not match the bmap"), "{err}"); + } + + #[test] + fn verify_checksums_rejects_bmap_without_checksums() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + let mut bmap = created_bmap(&image, None); + bmap.sha256 = None; + + assert!(verify_checksums(&image, &bmap).is_err()); + } + + #[test] + fn bmap_of_another_image_is_rejected() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + let mut other_size = created_bmap(&image, None); + other_size.image_size += BLOCK_SIZE; + let outside = Bmap { + block_size: BLOCK_SIZE, + image_size: IMAGE_BLOCKS * BLOCK_SIZE, + ranges: vec![RangeInclusive::new(IMAGE_BLOCKS, IMAGE_BLOCKS)], + sha256: Some(vec![String::new()]), + }; + + for bmap in [&other_size, &outside] { + assert!(verify_checksums(&image, bmap).is_err()); + assert!(write_bmap(&image, Some(bmap), &mut Vec::new()).is_err()); + } + } +} diff --git a/src/file/compression.rs b/src/file/compression.rs index 16f1862..1761b8c 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -1,13 +1,21 @@ +use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, is_zero}; use anyhow::{Context, Result}; -use filemagic::Magic; +use filemagic::{Flags, Magic}; use log::debug; -use std::env; -use std::fs::File; -use std::path::PathBuf; -use std::str::FromStr; +use std::{ + env, + fs::File, + io::{BufWriter, IntoInnerError, Read, Seek, SeekFrom, Write}, + os::unix::fs::FileExt, + path::{Path, PathBuf}, + str::FromStr, +}; use strum::IntoEnumIterator; use strum_macros::EnumIter; +/// The extensions of the compressed images that `bmaptool` reads. +pub const BMAPTOOL_COMPRESSIONS: [&str; 7] = ["gz", "gzip", "bz2", "xz", "lzo", "lz4", "zst"]; + #[derive(Clone, Debug, EnumIter)] #[allow(non_camel_case_types)] pub enum Compression { @@ -46,15 +54,17 @@ impl Compression { source: &mut std::fs::File, destination: &mut std::fs::File, ) -> std::io::Result { - let mut enc: Box = match &self { - Compression::bzip2 => Box::new(bzip2::write::BzEncoder::new( - destination, - bzip2::Compression::best(), - )), - Compression::gzip => Box::new(flate2::write::GzEncoder::new( - destination, - flate2::Compression::best(), - )), + // only `finish` reports an error while writing the trailer + let (bytes_read, _) = match &self { + Compression::bzip2 => { + let mut enc = bzip2::write::BzEncoder::new(destination, bzip2::Compression::best()); + (std::io::copy(source, &mut enc)?, enc.finish()?) + } + Compression::gzip => { + let mut enc = + flate2::write::GzEncoder::new(destination, flate2::Compression::best()); + (std::io::copy(source, &mut enc)?, enc.finish()?) + } Compression::xz { compression_level: level, } => { @@ -62,13 +72,11 @@ impl Compression { .threads(num_cpus::get() as u32) .preset(*level) .encoder()?; - Box::new(xz2::write::XzEncoder::new_stream(destination, stream)) + let mut enc = xz2::write::XzEncoder::new_stream(destination, stream); + (std::io::copy(source, &mut enc)?, enc.finish()?) } }; - - let bytes_written = std::io::copy(source, &mut enc)?; - enc.flush()?; - Ok(bytes_written) + Ok(bytes_read) } pub fn decompress( @@ -76,15 +84,18 @@ impl Compression { source: &mut std::fs::File, destination: &mut std::fs::File, ) -> std::io::Result { - let mut dec: Box = match &self { - Compression::bzip2 => Box::new(bzip2::write::BzDecoder::new(destination)), - Compression::gzip => Box::new(flate2::write::GzDecoder::new(destination)), - Compression::xz { .. } => Box::new(xz2::write::XzDecoder::new(destination)), + // the multi decoders also read concatenated streams, as pbzip2 writes them + let mut decoder: Box = match &self { + Compression::bzip2 => Box::new(bzip2::read::MultiBzDecoder::new(source)), + Compression::gzip => Box::new(flate2::read::MultiGzDecoder::new(source)), + Compression::xz { .. } => Box::new(xz2::read::XzDecoder::new_multi_decoder(source)), }; - - let bytes_written = std::io::copy(source, &mut dec)?; - dec.write_all(&[])?; - dec.flush()?; + let mut sparse = BufWriter::with_capacity(COPY_CHUNK_SIZE, SparseWriter::new(destination)?); + let bytes_written = std::io::copy(&mut decoder, &mut sparse)?; + sparse + .into_inner() + .map_err(IntoInnerError::into_error)? + .finish()?; Ok(bytes_written) } @@ -96,17 +107,47 @@ impl Compression { } } - fn extension(&self) -> &'static str { + pub fn extension(&self) -> &'static str { + self.extensions()[0] + } + + /// The first one is used for a compressed image that omnect-cli creates. + fn extensions(&self) -> &'static [&'static str] { match &self { - Compression::bzip2 => "bzip2", - Compression::gzip => "gzip", - Compression::xz { .. } => "xz", + Compression::bzip2 => &["bz2", "bzip2"], + Compression::gzip => &["gzip", "gz"], + Compression::xz { .. } => &["xz"], } } + /// `path` with the extension omnect-cli gives a compressed image appended. + pub fn compressed_path(&self, path: &Path) -> PathBuf { + path.with_added_extension(self.extension()) + } + + pub fn matches_extension(&self, path: &Path) -> bool { + path.extension() + .and_then(|e| e.to_str()) + .is_some_and(|e| self.extensions().contains(&e)) + } + + /// Whether `path` has an extension of `self` that `bmaptool` reads. + pub fn matches_output_extension(&self, path: &Path) -> bool { + self.matches_extension(path) + && path + .extension() + .and_then(|e| e.to_str()) + .is_some_and(|e| BMAPTOOL_COMPRESSIONS.contains(&e)) + } + + pub fn has_extension(path: &Path) -> bool { + Compression::iter().any(|c| c.matches_extension(path)) + } + pub fn from_file(image_file_name: &PathBuf) -> Result> { - let detector = Magic::open(Default::default()) - .context("image::compression: failed to open libmagic")?; + // a symlink, e.g. the Yocto deploy link to the latest image, is checked by its target + let detector = + Magic::open(Flags::SYMLINK).context("image::compression: failed to open libmagic")?; detector .load::(&[]) @@ -126,34 +167,331 @@ impl Compression { } } -pub fn decompress(image_file_name: &PathBuf, compression: &Compression) -> Result { - let mut new_image_file = PathBuf::from(image_file_name); +/// Skips zero blocks, so they stay holes in the file. +struct SparseWriter<'a> { + file: &'a mut File, + pos: u64, +} + +impl<'a> SparseWriter<'a> { + fn new(file: &'a mut File) -> std::io::Result { + let pos = file.stream_position()?; + // skipped zero blocks must read back as zeros + file.set_len(pos)?; + Ok(Self { file, pos }) + } + + fn finish(self) -> std::io::Result<()> { + // extends the file over zero blocks at the end + self.file.set_len(self.pos)?; + self.file.seek(SeekFrom::Start(self.pos))?; + Ok(()) + } +} + +impl Write for SparseWriter<'_> { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let block_offset = (self.pos % IMAGE_BLOCK_SIZE as u64) as usize; + + // one call handles a run of blocks that are all zero or all data + let mut end = buf.len().min(IMAGE_BLOCK_SIZE - block_offset); + let zero = is_zero(&buf[..end]); + while end < buf.len() { + let next = buf.len().min(end + IMAGE_BLOCK_SIZE); + if is_zero(&buf[end..next]) != zero { + break; + } + end = next; + } - if new_image_file - .extension() - .is_some_and(|ext| ext == compression.extension()) - { - new_image_file.set_extension(""); + let written = if zero { + end + } else { + self.file.write_at(&buf[..end], self.pos)? + }; + self.pos += written as u64; + Ok(written) } - let mut destination = File::create(&new_image_file)?; - let mut source = File::open(image_file_name)?; + fn flush(&mut self) -> std::io::Result<()> { + self.file.flush() + } +} + +/// Decompresses `image_file_name` into `dest_dir`, named without the compression extension. +pub fn decompress( + image_file_name: &Path, + compression: &Compression, + dest_dir: &Path, +) -> Result { + anyhow::ensure!( + compression.matches_extension(image_file_name), + "decompress: {} has no {:?} file extension", + image_file_name.display(), + compression.extensions() + ); + let new_image_file = dest_dir.join( + image_file_name + .file_stem() + .context("decompress: cannot get image file name")?, + ); + + let mut source = File::open(image_file_name).context(format!( + "decompress: cannot open {}", + image_file_name.display() + ))?; + let mut destination = File::create(&new_image_file).context(format!( + "decompress: cannot create {}", + new_image_file.display() + ))?; debug!("decompress {image_file_name:?} to {new_image_file:?}"); let bytes_written = compression.decompress(&mut source, &mut destination)?; debug!("image::decompress: copied {} bytes.", bytes_written); Ok(new_image_file) } -pub fn compress(image_file_name: &PathBuf, compression: &Compression) -> Result { - let new_image_file = PathBuf::from(format!( - "{}.{}", - image_file_name.to_str().unwrap(), - compression.extension() - )); - let mut destination = File::create(&new_image_file)?; - let mut source = File::open(image_file_name)?; - debug!("compress {image_file_name:?} to {new_image_file:?}"); - let bytes_written = compression.compress(&mut source, &mut destination)?; +pub fn compress( + image_file_name: &Path, + compression: &Compression, + destination: &mut File, +) -> Result<()> { + let mut source = File::open(image_file_name).context(format!( + "compress: cannot open {}", + image_file_name.display() + ))?; + debug!("compress {image_file_name:?}"); + let bytes_written = compression.compress(&mut source, destination)?; debug!("image::compress: copied {} bytes.", bytes_written); - Ok(new_image_file) + Ok(()) +} + +#[cfg(test)] +mod tests { + use crate::file::compression::*; + use std::{fs, io::Read, os::unix::fs::MetadataExt}; + + const XZ_TEST_IMAGE: &str = "testfiles/image.wic.xz"; + + // unit of `st_blocks` + const STAT_BLOCK_SIZE: u64 = 512; + const DATA_OFFSET: usize = 1024 * 1024 + 100; + const DATA: &[u8] = b"abc"; + const IMAGE_LEN: usize = 3 * 1024 * 1024 + 123; + const XZ_TEST_LEVEL: u32 = 1; + // cuts into the trailer, which every format checks only at the end of the stream + const TRUNCATED_BYTES: u64 = 4; + const OLD_CONTENT_BYTE: u8 = 0xa5; + const OLD_CONTENT_EXTRA_LEN: usize = 4096; + + fn all_compressions() -> [Compression; 3] { + [ + Compression::bzip2, + Compression::gzip, + Compression::xz { + compression_level: XZ_TEST_LEVEL, + }, + ] + } + + fn test_image() -> Vec { + let mut image = vec![0u8; IMAGE_LEN]; + image[DATA_OFFSET..DATA_OFFSET + DATA.len()].copy_from_slice(DATA); + image + } + + fn compress_image(compression: &Compression, image: &[u8]) -> File { + let mut source = tempfile::tempfile().expect("create source"); + source.write_all(image).expect("write source"); + source.rewind().expect("rewind source"); + + let mut compressed = tempfile::tempfile().expect("create compressed"); + compression + .compress(&mut source, &mut compressed) + .expect("compress"); + compressed.rewind().expect("rewind compressed"); + compressed + } + + #[test] + fn from_file_follows_a_symlink() { + let dir = tempfile::tempdir().expect("create temp dir"); + let link = dir.path().join("image.wic.xz"); + std::os::unix::fs::symlink( + Path::new(env!("CARGO_MANIFEST_DIR")).join(XZ_TEST_IMAGE), + &link, + ) + .expect("create symlink"); + + assert!(matches!( + Compression::from_file(&link).expect("detect compression"), + Some(Compression::xz { .. }) + )); + } + + fn read_all(file: &mut File) -> Vec { + let mut content = Vec::new(); + file.rewind().expect("rewind"); + file.read_to_end(&mut content).expect("read"); + content + } + + #[test] + fn decompress_keeps_zero_blocks_as_holes() { + let image = test_image(); + + for compression in all_compressions() { + let mut compressed = compress_image(&compression, &image); + let mut decompressed = tempfile::tempfile().expect("create decompressed"); + compression + .decompress(&mut compressed, &mut decompressed) + .expect("decompress"); + + assert!( + read_all(&mut decompressed) == image, + "{compression:?}: content differs" + ); + let meta = decompressed.metadata().expect("metadata"); + assert!( + meta.blocks() * STAT_BLOCK_SIZE < meta.len(), + "{compression:?}: no holes" + ); + } + } + + #[test] + fn decompress_replaces_old_content() { + let image = test_image(); + + for compression in all_compressions() { + let mut compressed = compress_image(&compression, &image); + let mut decompressed = tempfile::tempfile().expect("create decompressed"); + decompressed + .write_all(&vec![OLD_CONTENT_BYTE; IMAGE_LEN + OLD_CONTENT_EXTRA_LEN]) + .expect("write old content"); + decompressed.rewind().expect("rewind decompressed"); + + compression + .decompress(&mut compressed, &mut decompressed) + .expect("decompress"); + + assert!( + read_all(&mut decompressed) == image, + "{compression:?}: content differs" + ); + } + } + + #[test] + fn decompress_rejects_truncated_input() { + let image = test_image(); + + for compression in all_compressions() { + let mut compressed = compress_image(&compression, &image); + let len = compressed.metadata().expect("metadata").len(); + compressed + .set_len(len - TRUNCATED_BYTES) + .expect("truncate compressed"); + + let mut decompressed = tempfile::tempfile().expect("create decompressed"); + assert!( + compression + .decompress(&mut compressed, &mut decompressed) + .is_err(), + "{compression:?}: truncated input accepted" + ); + } + } + + #[test] + fn decompress_reads_concatenated_streams() { + let image = test_image(); + let (first, second) = image.split_at(DATA_OFFSET); + + for compression in all_compressions() { + let mut concatenated = tempfile::tempfile().expect("create concatenated"); + for part in [first, second] { + let mut compressed = compress_image(&compression, part); + concatenated + .write_all(&read_all(&mut compressed)) + .expect("write stream"); + } + concatenated.rewind().expect("rewind concatenated"); + + let mut decompressed = tempfile::tempfile().expect("create decompressed"); + compression + .decompress(&mut concatenated, &mut decompressed) + .expect("decompress"); + assert!( + read_all(&mut decompressed) == image, + "{compression:?}: content differs" + ); + } + } + + #[test] + fn decompress_file_strips_every_extension_of_the_compression() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = test_image(); + + for compression in all_compressions() { + for extension in compression.extensions() { + let path = dir.path().join(format!("image.wic.{extension}")); + let mut compressed = compress_image(&compression, &image); + fs::write(&path, read_all(&mut compressed)).expect("write compressed"); + + let decompressed = decompress(&path, &compression, dir.path()).expect("decompress"); + assert_eq!(decompressed, dir.path().join("image.wic")); + assert!( + fs::read(&decompressed).expect("read decompressed") == image, + "{compression:?}: content of .{extension} differs" + ); + assert!(Compression::has_extension(&path)); + } + } + } + + #[test] + fn compressed_path_has_an_extension_bmaptool_reads() { + let image = Path::new("image.wic"); + let expected = ["image.wic.bz2", "image.wic.gzip", "image.wic.xz"]; + + for (compression, expected) in all_compressions().iter().zip(expected) { + assert_eq!(compression.compressed_path(image), Path::new(expected)); + } + } + + #[test] + fn decompress_file_rejects_unknown_extension() { + let dir = tempfile::tempdir().expect("create temp dir"); + let compression = Compression::gzip; + let path = dir.path().join("image.wic"); + let mut compressed = compress_image(&compression, &test_image()); + let content = read_all(&mut compressed); + fs::write(&path, &content).expect("write compressed"); + + assert!(decompress(&path, &compression, dir.path()).is_err()); + assert!( + fs::read(&path).expect("read compressed") == content, + "source was changed" + ); + } + + #[test] + fn compress_fails_when_the_trailer_cannot_be_written() { + let dir = tempfile::tempdir().expect("create temp dir"); + // an empty source makes `finish` the first write to the destination + let source = dir.path().join("empty.wic"); + fs::write(&source, []).expect("write source"); + for compression in all_compressions() { + let mut source = fs::File::open(&source).expect("open source"); + let mut full = fs::OpenOptions::new() + .write(true) + .open("/dev/full") + .expect("open /dev/full"); + assert!( + compression.compress(&mut source, &mut full).is_err(), + "{compression:?}: write error was ignored" + ); + } + } } diff --git a/src/file/functions.rs b/src/file/functions.rs index 4b8cf95..b852145 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -1,11 +1,19 @@ +use crate::file::{ + COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, dir_of, partition::SECTOR_SIZE, signal::RegisteredTempDir, + sparse, +}; use anyhow::{Context, Result}; use log::{debug, warn}; -use std::collections::HashMap; -use std::fmt::{self, Display}; -use std::fs; -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::str::FromStr; +use std::{ + collections::HashMap, + fmt::{self, Display}, + fs, + ops::Range, + os::unix::fs::FileExt, + path::{Path, PathBuf}, + process::Command, + str::FromStr, +}; use stdext::function_name; use uuid::Uuid; @@ -184,12 +192,9 @@ macro_rules! try_exec_cmd { } pub fn copy_to_image(file_copy_params: &[FileCopyToParams], image_file: &Path) -> Result<()> { - // we use the folder the image is located in - // the caller is responsible to create a /tmp/ directory if needed - let working_dir = image_file - .parent() - .context("copy_to_image: cannot get directory of image")? - .to_path_buf(); + // the partition files get their own dir, so they never collide with the image + let working_dir = + RegisteredTempDir::create().context("copy_to_image: cannot create a temp dir")?; let image_file = image_file.to_str().unwrap(); let mut partition_map: HashMap<&Partition, Vec<(&PathBuf, &PathBuf)>> = HashMap::new(); @@ -204,7 +209,7 @@ pub fn copy_to_image(file_copy_params: &[FileCopyToParams], image_file: &Path) - // 1. for each involved partition for partition in partition_map.keys() { - let mut partition_file = working_dir.clone(); + let mut partition_file = working_dir.path().to_path_buf(); let partition_info = get_partition_info(image_file, partition)?; partition_file.push(Path::new(&format!("{}.img", partition_info.num))); @@ -265,17 +270,29 @@ pub fn copy_to_image(file_copy_params: &[FileCopyToParams], image_file: &Path) - Ok(()) } +/// Fails if the directory of `out_file` does not exist. A path without a directory is in the +/// current directory. +fn ensure_output_dir_exists(out_file: &Path) -> Result<()> { + let dir = dir_of(out_file); + anyhow::ensure!( + dir.is_dir(), + "copy_from_image: output dir {} does not exist", + dir.display() + ); + Ok(()) +} + pub fn copy_from_image(file_copy_params: &[FileCopyFromParams], image_file: &Path) -> Result<()> { - // we use the folder the image is located in - // the caller is responsible to create a /tmp/ directory if needed - let working_dir = image_file - .parent() - .context("copy_to_image: cannot get directory of image")? - .to_path_buf(); + for param in file_copy_params { + ensure_output_dir_exists(¶m.out_file)?; + } + // the partition files get their own dir, so they never collide with the image + let working_dir = + RegisteredTempDir::create().context("copy_from_image: cannot create a temp dir")?; let image_file = image_file.to_str().unwrap(); for param in file_copy_params.iter() { - let mut partition_file = working_dir.clone(); + let mut partition_file = working_dir.path().to_path_buf(); let partition_info = get_partition_info(image_file, ¶m.partition)?; let in_file = param.in_file.to_str().unwrap(); @@ -285,19 +302,9 @@ pub fn copy_from_image(file_copy_params: &[FileCopyFromParams], image_file: &Pat read_partition(image_file, partition_file, &partition_info)?; - anyhow::ensure!( - param - .out_file - .parent() - .unwrap() - .try_exists() - .is_ok_and(|exists| exists), - "copy_from_image: output dir does not exist." - ); - // copy if param.partition == Partition::boot { - let mut tmp_out_file = working_dir.clone(); + let mut tmp_out_file = working_dir.path().to_path_buf(); // mcopy deadlocks when target file is not residing in workingdir so we copy to a temp file tmp_out_file.push(format!( "{}-{}", @@ -412,20 +419,15 @@ fn read_partition( return Ok(()); } - let mut dd = Command::new("dd"); - dd.arg(format!("if={image_file}")) - .arg(format!("of={partition_file}")) - .arg("bs=512") - .arg(format!("skip={}", partition_info.start)) - .arg(format!("count={}", partition_info.count)) - .arg("conv=sparse") - .arg("status=none"); - exec_cmd!(dd); - - let mut sync = Command::new("sync"); - exec_cmd!(sync); + let image = + fs::File::open(image_file).context(format!("read_partition: cannot open {image_file}"))?; - Ok(()) + sparse::copy_to_new_file( + &image, + partition_info.start * SECTOR_SIZE, + Path::new(partition_file), + partition_info.count * SECTOR_SIZE, + ) } fn write_partition( @@ -433,34 +435,277 @@ fn write_partition( partition_file: &str, partition_info: &PartitionInfo, ) -> Result<()> { - let mut dd = Command::new("dd"); - dd.arg(format!("if={partition_file}")) - .arg(format!("of={image_file}")) - .arg("bs=512") - .arg(format!("seek={}", partition_info.start)) - .arg(format!("count={}", partition_info.count)) - .arg("conv=notrunc,sparse") - .arg("status=none"); - exec_cmd!(dd); - - let mut fallocate = Command::new("fallocate"); - fallocate.arg("-d").arg(image_file); - exec_cmd!(fallocate); - - let mut sync = Command::new("sync"); - exec_cmd!(sync); + let partition = fs::File::open(partition_file) + .context(format!("write_partition: cannot open {partition_file}"))?; + let image = fs::OpenOptions::new() + .read(true) + .write(true) + .open(image_file) + .context(format!("write_partition: cannot open {image_file}"))?; + + let image_offset = partition_info.start * SECTOR_SIZE; + let len = partition + .metadata() + .context("write_partition: cannot get partition file size")? + .len() + .min(partition_info.count * SECTOR_SIZE); + + // Only write blocks that differ: unchanged blocks keep their allocation, so a bmap + // created from the result still maps every block the source image mapped. A zero block + // that a tool wrote into a hole is data in the partition file and must become data in the + // image too, else a bmap flash keeps the old content there. + let partition_data = sparse::data_ranges(&partition, 0, len)?; + let image_data = sparse::data_ranges(&image, image_offset, image_offset + len)?; + // a block that is a hole in both files is zero in both, so it never needs a write + let block_size = IMAGE_BLOCK_SIZE as u64; + let spans = sparse::merge( + partition_data + .iter() + .cloned() + .chain( + image_data + .iter() + .map(|r| r.start - image_offset..r.end - image_offset), + ) + .map(|r| r.start / block_size * block_size..r.end.next_multiple_of(block_size).min(len)) + .collect(), + ); + let mut src = vec![0u8; COPY_CHUNK_SIZE]; + let mut dst = vec![0u8; COPY_CHUNK_SIZE]; + for span in spans { + write_changed_blocks( + &partition, + &image, + image_offset, + span, + (&partition_data, &image_data), + (&mut src, &mut dst), + )?; + } Ok(()) } -pub fn generate_bmap_file(image_file: &str) -> Result<()> { - let mut bmaptool = Command::new("bmaptool"); - bmaptool - .arg("create") - .arg("-o") - .arg(format!("{image_file}.bmap")) - .arg(image_file); - exec_cmd!(bmaptool); +fn write_changed_blocks( + partition: &fs::File, + image: &fs::File, + image_offset: u64, + span: Range, + (partition_data, image_data): (&[Range], &[Range]), + (src, dst): (&mut [u8], &mut [u8]), +) -> Result<()> { + let mut pos = span.start; + while pos < span.end { + let n = usize::try_from((span.end - pos).min(COPY_CHUNK_SIZE as u64))?; + partition + .read_exact_at(&mut src[..n], pos) + .context("write_partition: cannot read partition file")?; + image + .read_exact_at(&mut dst[..n], image_offset + pos) + .context("write_partition: cannot read image")?; + + let blocks = src[..n] + .chunks(IMAGE_BLOCK_SIZE) + .zip(dst[..n].chunks(IMAGE_BLOCK_SIZE)); + for (i, (s, d)) in blocks.enumerate() { + let block_pos = pos + (i * IMAGE_BLOCK_SIZE) as u64; + let block = block_pos..block_pos + s.len() as u64; + let image_block = image_offset + block.start..image_offset + block.end; + if s != d || (has_data(partition_data, &block) && has_hole(image_data, &image_block)) { + image + .write_all_at(s, image_block.start) + .context("write_partition: cannot write image")?; + } + } + pos += n as u64; + } Ok(()) } + +fn has_data(data_ranges: &[Range], block: &Range) -> bool { + let i = data_ranges.partition_point(|r| r.end <= block.start); + data_ranges.get(i).is_some_and(|r| r.start < block.end) +} + +fn has_hole(data_ranges: &[Range], block: &Range) -> bool { + let i = data_ranges.partition_point(|r| r.end <= block.start); + !data_ranges + .get(i) + .is_some_and(|r| r.start <= block.start && block.end <= r.end) +} + +#[cfg(test)] +mod tests { + use crate::file::functions::*; + use std::os::unix::fs::MetadataExt; + + const PART_NUM: u32 = 1; + const PART_START: u64 = 8; + const PART_SECTORS: u64 = 64; + const OLD_CONTENT_BYTE: u8 = 0xa5; + const ZEROED_BLOCK: usize = 2; + // the first partition of an old MBR layout + const UNALIGNED_PART_START: u64 = 63; + + fn partition_info() -> PartitionInfo { + PartitionInfo { + num: PART_NUM, + start: PART_START, + count: PART_SECTORS, + } + } + + #[test] + fn ensure_output_dir_exists_takes_a_bare_name() { + let dir = tempfile::tempdir().expect("create temp dir"); + + ensure_output_dir_exists(Path::new("out.txt")).expect("current dir"); + ensure_output_dir_exists(&dir.path().join("out.txt")).expect("existing dir"); + assert!(ensure_output_dir_exists(&dir.path().join("missing/out.txt")).is_err()); + } + + #[test] + fn read_partition_fails_for_partition_past_the_image_end() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + let partition = dir.path().join("partition.img"); + fs::File::create(&image) + .expect("create image") + .set_len((PART_START + PART_SECTORS - 1) * SECTOR_SIZE) + .expect("set image size"); + + assert!( + read_partition( + image.to_str().expect("image path"), + partition.to_str().expect("partition path"), + &partition_info(), + ) + .is_err() + ); + } + + #[test] + fn write_partition_writes_block_changed_to_zeros() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + let partition = dir.path().join("partition.img"); + let image_len = usize::try_from((PART_START + PART_SECTORS) * SECTOR_SIZE).expect("size"); + let part_offset = usize::try_from(PART_START * SECTOR_SIZE).expect("offset"); + + fs::write(&image, vec![OLD_CONTENT_BYTE; image_len]).expect("write image"); + let mut content = vec![OLD_CONTENT_BYTE; image_len - part_offset]; + content[ZEROED_BLOCK * IMAGE_BLOCK_SIZE..(ZEROED_BLOCK + 1) * IMAGE_BLOCK_SIZE].fill(0); + fs::write(&partition, &content).expect("write partition"); + + write_partition( + image.to_str().expect("image path"), + partition.to_str().expect("partition path"), + &partition_info(), + ) + .expect("write partition"); + + let written = fs::read(&image).expect("read image"); + assert!(written[part_offset..] == content[..], "partition differs"); + } + + #[test] + fn write_partition_keeps_holes_of_unchanged_blocks() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + let partition = dir.path().join("partition.img"); + fs::File::create(&image) + .expect("create image") + .set_len((PART_START + PART_SECTORS) * SECTOR_SIZE) + .expect("set image size"); + fs::File::create(&partition) + .expect("create partition") + .set_len(PART_SECTORS * SECTOR_SIZE) + .expect("set partition size"); + let allocated = || fs::metadata(&image).expect("image metadata").blocks(); + let before = allocated(); + + write_partition( + image.to_str().expect("image path"), + partition.to_str().expect("partition path"), + &partition_info(), + ) + .expect("write partition"); + + assert_eq!(allocated(), before, "unchanged zero blocks were written"); + } + + #[test] + fn write_partition_maps_zero_block_of_an_unaligned_partition() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + let partition = dir.path().join("partition.img"); + let info = PartitionInfo { + num: PART_NUM, + start: UNALIGNED_PART_START, + count: PART_SECTORS, + }; + let image_file = fs::File::create(&image).expect("create image"); + image_file + .set_len((UNALIGNED_PART_START + PART_SECTORS) * SECTOR_SIZE) + .expect("set image size"); + let partition_file = fs::File::create(&partition).expect("create partition"); + partition_file + .set_len(PART_SECTORS * SECTOR_SIZE) + .expect("set partition size"); + let block = (ZEROED_BLOCK * IMAGE_BLOCK_SIZE) as u64; + partition_file + .write_all_at(&[0; IMAGE_BLOCK_SIZE], block) + .expect("write zero block"); + + write_partition( + image.to_str().expect("image path"), + partition.to_str().expect("partition path"), + &info, + ) + .expect("write partition"); + + let block = UNALIGNED_PART_START * SECTOR_SIZE + block; + let block = block..block + IMAGE_BLOCK_SIZE as u64; + let image_data = + sparse::data_ranges(&image_file, block.start, block.end).expect("data ranges"); + assert_eq!(image_data, vec![block], "zero block is still a hole"); + } + + #[test] + fn write_partition_maps_zero_block_written_into_a_hole() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + let partition = dir.path().join("partition.img"); + let image_file = fs::File::create(&image).expect("create image"); + image_file + .set_len((PART_START + PART_SECTORS) * SECTOR_SIZE) + .expect("set image size"); + let partition_file = fs::File::create(&partition).expect("create partition"); + partition_file + .set_len(PART_SECTORS * SECTOR_SIZE) + .expect("set partition size"); + let block = (ZEROED_BLOCK * IMAGE_BLOCK_SIZE) as u64; + partition_file + .write_all_at(&[0; IMAGE_BLOCK_SIZE], block) + .expect("write zero block"); + + write_partition( + image.to_str().expect("image path"), + partition.to_str().expect("partition path"), + &partition_info(), + ) + .expect("write partition"); + + let block = PART_START * SECTOR_SIZE + block; + assert_eq!( + sparse::data_ranges( + &image_file, + 0, + image_file.metadata().expect("metadata").len() + ) + .expect("data ranges"), + vec![block..block + IMAGE_BLOCK_SIZE as u64] + ); + } +} diff --git a/src/file/mod.rs b/src/file/mod.rs index 6f9db48..61a9069 100644 --- a/src/file/mod.rs +++ b/src/file/mod.rs @@ -1,6 +1,11 @@ +pub mod bmap; pub mod compression; pub mod functions; +pub(crate) mod output; mod partition; +pub(crate) mod signal; +pub(crate) mod sparse; + use super::validators::{ device_update, identity::{IdentityConfig, IdentityType, validate_identity}, @@ -15,6 +20,27 @@ use std::{ path::{Path, PathBuf}, }; +/// Granularity at which zero data in an image file is kept as a hole. +const IMAGE_BLOCK_SIZE: usize = 4096; +const COPY_CHUNK_SIZE: usize = 1024 * 1024; + +static ZERO_BLOCK: [u8; IMAGE_BLOCK_SIZE] = [0; IMAGE_BLOCK_SIZE]; + +/// The directory of `path`; `.` for a path without one. +fn dir_of(path: &Path) -> &Path { + match path.parent() { + Some(dir) if !dir.as_os_str().is_empty() => dir, + _ => Path::new("."), + } +} + +/// A slice compare compiles to `memcmp`, which is many times faster than a byte loop with an +/// early exit. +fn is_zero(data: &[u8]) -> bool { + data.chunks(IMAGE_BLOCK_SIZE) + .all(|chunk| chunk == &ZERO_BLOCK[..chunk.len()]) +} + pub fn set_iotedge_gateway_config( config_file: &Path, image_file: &Path, @@ -293,3 +319,27 @@ pub(crate) fn get_file_path(image_path: &Path, file_name: &str) -> Result, + target: Option<&Compression>, + explicit: Option<&Path>, +) -> Result { + if let Some(path) = explicit { + match target { + Some(c) => anyhow::ensure!( + c.matches_output_extension(path), + "{} does not end in an extension of the -p compression that bmaptool reads; name \ + it e.g. {}", + path.display(), + c.compressed_path(&without_extension_of(path, Compression::iter())) + .display() + ), + None => anyhow::ensure!( + !Compression::has_extension(path), + "{} has a compression extension, but the image is written uncompressed; pass -p", + path.display() + ), + } + return Ok(path.to_path_buf()); + } + if target.is_some_and(|c| c.matches_output_extension(image)) { + return Ok(image.to_path_buf()); + } + + let output = without_extension_of(image, source); + Ok(match target { + Some(c) => c.compressed_path(&output), + None => output, + }) +} + +/// `path` without its extension if it is an extension of one of `compressions`. +fn without_extension_of>( + path: &Path, + compressions: impl IntoIterator, +) -> PathBuf { + let mut path = path.to_path_buf(); + if compressions + .into_iter() + .any(|c| c.borrow().matches_extension(&path)) + { + path.set_extension(""); + } + path +} + +/// Fails if writing `output_image` and its bmap would overwrite a file the user did not name, +/// replace the bmap of an image that stays, or leave a bmap that `bmaptool` would use for the +/// output image and that does not match it. +pub fn check_conflicts( + input_image: &Path, + output_image: &Path, + explicit_output: bool, + output_bmap: Option<&Path>, + input_bmap: Option<&Path>, +) -> Result<()> { + let in_place = same_file(input_image, output_image)?; + ensure_replaceable(output_image)?; + + anyhow::ensure!( + // a symlink counts also if its target does not exist yet + explicit_output || in_place || output_image.symlink_metadata().is_err(), + "{} exists; choose the output image with --output-image", + output_image.display() + ); + if let Some(target_bmap) = output_bmap { + ensure_replaceable(target_bmap)?; + // e.g. an image named `x.bmap`, for which `bmaptool` finds the image itself as its bmap + anyhow::ensure!( + !same_file(target_bmap, output_image)?, + "{} would be both the output image and its bmap; choose another output image with \ + --output-image", + output_image.display() + ); + ensure_bmap_next_to(output_image, target_bmap)?; + if !in_place && let Some(bmap) = input_bmap { + anyhow::ensure!( + !same_file(bmap, target_bmap)?, + "{} is the bmap of {} and would be replaced; choose another output image with \ + --output-image", + bmap.display(), + input_image.display() + ); + } + for image in images_using(target_bmap)? { + anyhow::ensure!( + same_file(&image, output_image)?, + "bmaptool would use {} also for {}, which stays and would not match it; choose \ + another output image with --output-image", + target_bmap.display(), + image.display() + ); + } + } else if let Some(stale_bmap) = bmap::find_input_bmap(output_image, None)? { + anyhow::bail!( + "{} would not match {}; pass -b to write a matching bmap", + stale_bmap.display(), + output_image.display() + ); + } + + Ok(()) +} + +/// The existing images next to `bmap` for which `bmaptool` would use `bmap`: `` and +/// `.`, if no other bmap comes first in the lookup of `bmaptool`. The stem is +/// taken from `bmap` and from its resolved path, because a symlinked bmap, e.g. a Yocto deploy +/// link, is also the bmap of the images named after its target. +fn images_using(bmap: &Path) -> Result> { + let stems = BTreeSet::from([bmap.with_extension(""), resolve(bmap)?.with_extension("")]); + let candidates = stems.into_iter().flat_map(|stem| { + iter::once(stem.clone()).chain( + BMAPTOOL_COMPRESSIONS + .iter() + .map(move |extension| stem.with_added_extension(extension)), + ) + }); + let mut images = vec![]; + for image in candidates.filter(|image| image.is_file()) { + for lookup in bmap::bmaptool_lookup(&image) { + if same_file(&lookup, bmap)? { + images.push(image); + break; + } + if lookup.is_file() { + break; + } + } + } + Ok(images) +} + +/// Fails if `output_image` is a mount point, e.g. an image file mounted into a container, and +/// `bmap` is not: the bmap would then be written next to the mount point, not next to the image. +fn ensure_bmap_next_to(output_image: &Path, bmap: &Path) -> Result<()> { + anyhow::ensure!( + !is_mount_root(&resolve(output_image)?) || is_mount_root(&resolve(bmap)?), + "{} is a mount point, so its bmap {} would not be written next to the image; mount the \ + directory of the image instead", + output_image.display(), + bmap.display() + ); + Ok(()) +} + +/// `.bmap`, but for an image changed in place the bmap that `bmaptool` finds next +/// to it, e.g. `x.wic.bmap` for `x.wic.xz`, so no bmap of the old image stays. +pub fn output_bmap_path(input_image: &Path, output_image: &Path) -> Result { + if same_file(input_image, output_image)? + && let Some(bmap) = bmap::find_input_bmap(input_image, None)? + { + return Ok(bmap); + } + Ok(bmap::bmap_path(output_image)) +} + +/// Fails if `Staged::write` and `commit` could not replace `path`. +fn ensure_replaceable(path: &Path) -> Result<()> { + ensure_regular_file(path)?; + ensure_writable(path)?; + ensure_staging_dir_writable(path) +} + +/// Fails if `path` exists and is no regular file, e.g. a block device, which the rename of the +/// staged file would replace instead of writing to it. +fn ensure_regular_file(path: &Path) -> Result<()> { + match fs::metadata(path) { + Ok(metadata) => anyhow::ensure!( + metadata.is_file(), + "{} is not a regular file; choose another output image with --output-image", + path.display() + ), + Err(e) => anyhow::ensure!( + e.kind() == ErrorKind::NotFound, + "cannot read {}: {e}", + path.display() + ), + } + Ok(()) +} + +/// Fails if `path` exists and the user cannot write it, so a read-only file is not replaced. +fn ensure_writable(path: &Path) -> Result<()> { + match rustix::fs::access(path, Access::WRITE_OK) { + Err(Errno::NOENT) => Ok(()), + result => result.context(format!( + "cannot write {}; make it writable or choose another output image with \ + --output-image", + path.display() + )), + } +} + +/// Fails before the command runs if `Staged::write` could not create its temp dir for `dest`. +fn ensure_staging_dir_writable(dest: &Path) -> Result<()> { + let (dir, _) = staging_dir(&resolve(dest)?)?; + rustix::fs::access(&dir, Access::WRITE_OK).context(format!( + "cannot write to {}, where the new {} is staged; make it writable or choose another \ + output image with --output-image", + dir.display(), + dest.display() + )) +} + +/// The directory for the staged file of the resolved `dest`, and whether the staged file is +/// copied into `dest`. A rename cannot replace a mount point, and needs write access to the +/// directory of `dest`, so an existing `dest` without either is staged in TMPDIR. +fn staging_dir(dest: &Path) -> Result<(PathBuf, bool)> { + let dir = dest + .parent() + .context(format!("{} has no directory", dest.display()))?; + let dir_writable = rustix::fs::access(dir, Access::WRITE_OK).is_ok(); + if is_mount_root(dest) || (dest.exists() && !dir_writable) { + return Ok((env::temp_dir(), true)); + } + Ok((dir.to_path_buf(), false)) +} + +/// A new file in the directory of its destination, which `commit` renames to the destination. +/// Until then the destination stays unchanged. A rename cannot replace a mount point, e.g. an +/// image file mounted into a container, or a file in a directory the user cannot write, so for +/// such a destination the new file is staged in TMPDIR and copied into it. +pub struct Staged { + tmp_dir: RegisteredTempDir, + file: PathBuf, + dest: PathBuf, + in_tmpdir: bool, +} + +impl Staged { + /// Writes and syncs the new file for `dest`. A symlink is followed, and an existing `dest` + /// keeps its permissions. + pub fn write(dest: &Path, write: impl FnOnce(&mut File) -> Result<()>) -> Result { + let dest = resolve(dest)?; + let (tmp_parent, in_tmpdir) = staging_dir(&dest)?; + let tmp_dir = RegisteredTempDir::create_in(&tmp_parent, TMP_DIR_PREFIX).context( + format!("cannot create a temp dir in {}", tmp_parent.display()), + )?; + let file = tmp_dir.path().join( + dest.file_name() + .context(format!("{} has no file name", dest.display()))?, + ); + let staged = Self { + tmp_dir, + file, + dest, + in_tmpdir, + }; + + let mut handle = File::create(&staged.file) + .context(format!("cannot create {}", staged.file.display()))?; + write(&mut handle)?; + if let Ok(metadata) = fs::metadata(&staged.dest) { + handle + .set_permissions(metadata.permissions()) + .context(format!( + "cannot set permissions of {}", + staged.file.display() + ))?; + } + // a file in TMPDIR is only copied into `dest`, and the copy is synced + if !in_tmpdir { + // the handle that wrote the data also gets the write errors + handle + .sync_all() + .context(format!("cannot sync {}", staged.file.display()))?; + } + + Ok(staged) + } + + fn rename(&mut self) -> Result<()> { + if self.in_tmpdir { + return self.copy_into_dest(); + } + match fs::rename(&self.file, &self.dest) { + // a sticky dir, e.g. /tmp, refuses to replace a file of another user + Err(e) + if matches!( + e.kind(), + ErrorKind::ResourceBusy + | ErrorKind::CrossesDevices + | ErrorKind::PermissionDenied + ) => + { + info!( + "cannot rename to {}: {e}; copying into it instead", + self.dest.display() + ); + self.copy_into_dest() + } + result => result.context(format!( + "cannot rename {} to {}", + self.file.display(), + self.dest.display() + )), + } + } + + /// Keeps the staged file if the copy fails or a signal stops it, because `dest` is truncated + /// already. + fn copy_into_dest(&mut self) -> Result<()> { + self.tmp_dir.unregister(); + info!( + "copying {} into {}; it stays if the copy fails", + self.file.display(), + self.dest.display() + ); + let open = |create_new| { + OpenOptions::new() + .write(true) + .truncate(true) + .create_new(create_new) + // `dest` is resolved already; a symlink here was put in its place since then + .custom_flags(libc::O_NOFOLLOW) + .open(&self.dest) + }; + // O_CREAT only for a new `dest`: `fs.protected_regular` refuses it for an existing file + // of another user in a sticky dir + let dest = match open(false) { + Err(e) if e.kind() == ErrorKind::NotFound => open(true), + result => result, + } + .context(format!("cannot open {}", self.dest.display()))?; + + let result = sparse::copy_file(&self.file, &dest).and_then(|()| { + dest.sync_all() + .context(format!("cannot sync {}", self.dest.display())) + }); + match result { + Ok(()) => self.tmp_dir.register(), + Err(_) => self.tmp_dir.keep(), + } + result.with_context(|| { + format!( + "cannot copy into {}; the new file is kept in {}", + self.dest.display(), + self.file.display() + ) + }) + } +} + +/// `false` if the kernel cannot tell. +fn is_mount_root(path: &Path) -> bool { + rustix::fs::statx( + CWD, + path, + AtFlags::SYMLINK_NOFOLLOW | AtFlags::NO_AUTOMOUNT, + StatxFlags::empty(), + ) + .is_ok_and(|statx| { + statx + .stx_attributes_mask + .contains(StatxAttributes::MOUNT_ROOT) + && statx.stx_attributes.contains(StatxAttributes::MOUNT_ROOT) + }) +} + +/// Renames the staged `image` and then its `bmap`, then syncs their directories, so the renames +/// are not lost on a power loss. If the image fails, the old image keeps its bmap, unless the +/// image was truncated already: then the old bmap is removed and the new one is kept next to the +/// kept image. If the bmap fails, the old bmap is removed, because it does not match the new +/// image. An old bmap that cannot be removed is named in the error. A stop signal after the image +/// step removes the staged bmap and keeps the old one. +pub fn commit(mut image: Staged, mut bmap: Option) -> Result<()> { + if let Err(e) = image.rename() { + return match &bmap { + Some(bmap) if image.tmp_dir.is_kept() => Err(keep_next_to(bmap, &image, e)), + _ => Err(e), + }; + } + if let Some(bmap) = &mut bmap + && let Err(e) = bmap.rename() + { + return Err(remove_old_bmap(bmap, e)); + } + + let dirs: BTreeSet<_> = iter::once(&image) + .chain(&bmap) + .filter_map(|staged| staged.dest.parent()) + .collect(); + for dir in dirs { + // both files are complete already, only a power loss could undo the rename + if let Err(e) = File::open(dir).and_then(|dir| dir.sync_all()) { + warn!("cannot sync {}: {e}", dir.display()); + } + } + + Ok(()) +} + +/// Copies the staged `bmap` next to the kept staged `image` and removes the old bmap, which +/// matches no image anymore. +fn keep_next_to(bmap: &Staged, image: &Staged, error: anyhow::Error) -> anyhow::Error { + let error = remove_old_bmap(bmap, error); + let Some(name) = bmap.dest.file_name() else { + return error; + }; + let kept = image.tmp_dir.path().join(name); + match fs::copy(&bmap.file, &kept) { + Ok(_) => error.context(format!("the new bmap is kept in {}", kept.display())), + Err(e) => { + warn!("cannot keep the new bmap in {}: {e}", kept.display()); + error + } + } +} + +/// Removes the old bmap at the destination of `bmap`, and adds to `error` if it stays. A kept +/// `bmap` was truncated already by the copy into its destination. +fn remove_old_bmap(bmap: &Staged, error: anyhow::Error) -> anyhow::Error { + let Err(e) = remove_old(&bmap.dest) else { + return error; + }; + let state = if bmap.tmp_dir.is_kept() { + "is incomplete" + } else { + "does not match the new image" + }; + error.context(format!( + "the old {} stays and {state}: {e}", + bmap.dest.display() + )) +} + +fn remove_old(path: &Path) -> io::Result<()> { + match fs::remove_file(path) { + Err(e) if e.kind() == ErrorKind::NotFound => Ok(()), + result => result, + } +} + +fn same_file(a: &Path, b: &Path) -> Result { + Ok(resolve(a)? == resolve(b)?) +} + +/// The absolute path of `path` with symlinks resolved. A file that does not exist yet, also as +/// the target of a symlink, is resolved through its directory. +fn resolve(path: &Path) -> Result { + let mut path = path.to_path_buf(); + for _ in 0..MAX_SYMLINKS { + if !path.is_symlink() || path.exists() { + return resolve_existing_dir(&path); + } + let target = fs::read_link(&path).context(format!("cannot read {}", path.display()))?; + // a relative target is relative to the directory of the link + path = path.parent().unwrap_or(Path::new("")).join(target); + } + anyhow::bail!("too many symlinks in {}", path.display()) +} + +fn resolve_existing_dir(path: &Path) -> Result { + if path.exists() { + return fs::canonicalize(path).context(format!("cannot resolve {}", path.display())); + } + let dir = dir_of(path); + let name = path + .file_name() + .context(format!("{} has no file name", path.display()))?; + + Ok(fs::canonicalize(dir) + .context(format!("cannot resolve {}", dir.display()))? + .join(name)) +} + +#[cfg(test)] +mod tests { + use crate::file::{output::*, signal}; + use std::{ + io::Write, + os::unix::fs::{PermissionsExt, symlink}, + }; + + const IMAGE: &str = "image.wic"; + const IMAGE_XZ: &str = "image.wic.xz"; + const OTHER_IMAGE: &str = "device.wic"; + const XZ_LEVEL: u32 = 9; + const OLD_CONTENT: &[u8] = b"old image"; + const NEW_CONTENT: &[u8] = b"new image"; + const IMAGE_MODE: u32 = 0o644; + const READ_ONLY_MODE: u32 = 0o444; + const MODE_MASK: u32 = 0o777; + const DIR_MODE: u32 = 0o755; + const READ_ONLY_DIR_MODE: u32 = 0o555; + const UNRESIZABLE_FILE: &str = "/dev/full"; + const DEVICE_FILE: &str = "/dev/null"; + const MOUNTINFO: &str = "/proc/self/mountinfo"; + const MOUNTINFO_MOUNT_POINT: usize = 4; + + fn bmap_of(input_image: &Path, output_image: &Path) -> PathBuf { + output_bmap_path(input_image, output_image).expect("output bmap path") + } + + fn touch(path: &Path) { + fs::write(path, []).expect("create file"); + } + + #[test] + fn output_image_follows_compression() { + let image = Path::new(IMAGE); + let image_xz = Path::new(IMAGE_XZ); + let xz = Compression::xz { + compression_level: XZ_LEVEL, + }; + let output = |image, source, target| { + output_image(image, source, target, None).expect("output image") + }; + + assert_eq!(output(image, None, None), image); + assert_eq!(output(image_xz, Some(&xz), None), image); + assert_eq!(output(image, None, Some(&xz)), image_xz); + assert_eq!(output(image_xz, Some(&xz), Some(&xz)), image_xz); + } + + #[test] + fn output_image_uses_the_extension_bmaptool_reads() { + for (image, compression, output) in [ + ("image.wic.gz", Compression::gzip, "image.wic.gz"), + ("image.wic.gzip", Compression::gzip, "image.wic.gzip"), + ("image.wic.bz2", Compression::bzip2, "image.wic.bz2"), + ("image.wic.bzip2", Compression::bzip2, "image.wic.bz2"), + ] { + assert_eq!( + output_image( + Path::new(image), + Some(&compression), + Some(&compression), + None + ) + .expect("output image"), + Path::new(output) + ); + } + let error = output_image( + Path::new(IMAGE), + None, + Some(&Compression::bzip2), + Some(Path::new("device.wic.bzip2")), + ) + .expect_err("long name with -o"); + assert!(error.to_string().contains("name it e.g. device.wic.bz2")); + let gz = Path::new("device.wic.gz"); + assert_eq!( + output_image(Path::new(IMAGE), None, Some(&Compression::gzip), Some(gz)) + .expect("short gzip name with -o"), + gz + ); + } + + #[test] + fn output_image_strips_only_the_source_compression_extension() { + let image = Path::new("image.bin"); + let xz = Compression::xz { + compression_level: XZ_LEVEL, + }; + + assert_eq!( + output_image(image, Some(&xz), None, None).expect("output image"), + image + ); + } + + #[test] + fn output_image_takes_explicit_path_with_matching_extension() { + let xz = Compression::xz { + compression_level: XZ_LEVEL, + }; + let other_xz = Path::new("device.wic.xz"); + + assert_eq!( + output_image( + Path::new(IMAGE_XZ), + Some(&xz), + None, + Some(Path::new(OTHER_IMAGE)) + ) + .expect("uncompressed"), + Path::new(OTHER_IMAGE) + ); + assert_eq!( + output_image(Path::new(IMAGE), None, Some(&xz), Some(other_xz)).expect("compressed"), + other_xz + ); + } + + #[test] + fn output_image_rejects_explicit_path_with_wrong_extension() { + let xz = Compression::xz { + compression_level: XZ_LEVEL, + }; + + assert!( + output_image( + Path::new(IMAGE), + None, + Some(&xz), + Some(Path::new(OTHER_IMAGE)) + ) + .is_err() + ); + assert!( + output_image( + Path::new(IMAGE), + None, + Some(&xz), + Some(Path::new("device.wic.gz")) + ) + .is_err() + ); + assert!( + output_image( + Path::new(IMAGE_XZ), + Some(&xz), + None, + Some(Path::new("device.wic.xz")) + ) + .is_err() + ); + } + + #[test] + fn check_conflicts_allows_in_place_with_bmap() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + touch(&image); + touch(&image_bmap); + + check_conflicts( + &image, + &image, + false, + Some(&bmap_of(&image, &image)), + Some(&image_bmap), + ) + .expect("no conflict"); + } + + #[test] + fn check_conflicts_rejects_an_image_that_is_its_own_bmap() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.bmap"); + touch(&image); + + let error = check_conflicts(&image, &image, false, Some(&bmap_of(&image, &image)), None) + .expect_err("image is its own bmap"); + assert!( + error + .to_string() + .contains("both the output image and its bmap") + ); + } + + #[test] + fn output_bmap_path_replaces_the_bmap_bmaptool_finds_in_place() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_xz = dir.path().join(IMAGE_XZ); + let image_bmap = bmap::bmap_path(&dir.path().join(IMAGE)); + let other = dir.path().join(OTHER_IMAGE); + touch(&image_xz); + + let output = |output| bmap_of(&image_xz, output); + assert_eq!(output(&image_xz), bmap::bmap_path(&image_xz)); + touch(&image_bmap); + assert_eq!(output(&image_xz), image_bmap); + assert_eq!(output(&other), bmap::bmap_path(&other)); + } + + #[test] + fn check_conflicts_rejects_a_read_only_output() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + touch(&image); + touch(&image_bmap); + let read_only = fs::Permissions::from_mode(READ_ONLY_MODE); + fs::set_permissions(&image_bmap, read_only.clone()).expect("set permissions"); + // root can write a read-only file + if OpenOptions::new().write(true).open(&image_bmap).is_ok() { + return; + } + + assert!( + check_conflicts( + &image, + &image, + false, + Some(&bmap_of(&image, &image)), + Some(&image_bmap) + ) + .is_err() + ); + fs::remove_file(&image_bmap).expect("remove bmap"); + check_conflicts(&image, &image, false, None, None).expect("writable image"); + fs::set_permissions(&image, read_only).expect("set permissions"); + assert!(check_conflicts(&image, &image, false, None, None).is_err()); + } + + #[test] + fn check_conflicts_allows_only_in_place_in_a_read_only_dir() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + touch(&image); + fs::set_permissions(dir.path(), fs::Permissions::from_mode(READ_ONLY_DIR_MODE)) + .expect("set permissions"); + // root can write a read-only directory + let writable = rustix::fs::access(dir.path(), Access::WRITE_OK).is_ok(); + + let in_place = check_conflicts(&image, &image, false, None, None); + let new_file = check_conflicts(&image, &dir.path().join(OTHER_IMAGE), true, None, None); + fs::set_permissions(dir.path(), fs::Permissions::from_mode(DIR_MODE)) + .expect("set permissions"); + in_place.expect("in place"); + assert_eq!(new_file.is_ok(), writable); + } + + #[test] + fn staged_commit_copies_into_a_file_in_a_read_only_dir() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + fs::write(&image, OLD_CONTENT).expect("write image"); + fs::set_permissions(dir.path(), fs::Permissions::from_mode(READ_ONLY_DIR_MODE)) + .expect("set permissions"); + let writable = rustix::fs::access(dir.path(), Access::WRITE_OK).is_ok(); + + let staged = stage(&image, NEW_CONTENT); + let in_tmpdir = staged.in_tmpdir; + let result = commit(staged, None); + fs::set_permissions(dir.path(), fs::Permissions::from_mode(DIR_MODE)) + .expect("set permissions"); + result.expect("commit"); + assert_eq!(in_tmpdir, !writable); + assert_eq!(fs::read(&image).expect("read image"), NEW_CONTENT); + } + + #[test] + fn check_conflicts_rejects_an_output_that_is_no_regular_file() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let device = dir.path().join(OTHER_IMAGE); + let output_dir = dir.path().join("output.wic"); + touch(&image); + symlink(DEVICE_FILE, &device).expect("create symlink"); + fs::create_dir(&output_dir).expect("create dir"); + + for output in [&device, &output_dir] { + let error = check_conflicts(&image, output, true, None, None).expect_err("no file"); + assert!(error.to_string().contains("is not a regular file")); + } + } + + #[test] + fn check_conflicts_rejects_existing_derived_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_xz = dir.path().join(IMAGE_XZ); + let image = dir.path().join(IMAGE); + touch(&image_xz); + touch(&image); + + assert!(check_conflicts(&image_xz, &image, false, None, None).is_err()); + check_conflicts(&image_xz, &image, true, None, None).expect("explicit output"); + } + + #[test] + fn check_conflicts_rejects_replacing_the_input_bmap() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_xz = dir.path().join(IMAGE_XZ); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + touch(&image_xz); + touch(&image_bmap); + + assert!( + check_conflicts( + &image_xz, + &image, + false, + Some(&bmap_of(&image_xz, &image)), + Some(&image_bmap) + ) + .is_err() + ); + let other = dir.path().join(OTHER_IMAGE); + check_conflicts( + &image_xz, + &other, + true, + Some(&bmap_of(&image_xz, &other)), + Some(&image_bmap), + ) + .expect("other output"); + } + + #[test] + fn check_conflicts_rejects_leaving_a_stale_bmap() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + touch(&image); + touch(&bmap::bmap_path(&image)); + + assert!(check_conflicts(&image, &image, false, None, None).is_err()); + } + + #[test] + fn check_conflicts_rejects_a_stale_bmap_bmaptool_finds() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + touch(&image); + touch(&dir.path().join("image.bmap")); + + assert!(check_conflicts(&image, &image, false, None, None).is_err()); + } + + #[test] + fn check_conflicts_rejects_a_stale_bmap_of_the_uncompressed_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_xz = dir.path().join(IMAGE_XZ); + let image_bmap = bmap::bmap_path(&dir.path().join(IMAGE)); + touch(&image_xz); + touch(&image_bmap); + + // bmaptool uses image.wic.bmap for image.wic.xz + assert!(check_conflicts(&image_xz, &image_xz, false, None, None).is_err()); + check_conflicts( + &image_xz, + &image_xz, + false, + Some(&bmap_of(&image_xz, &image_xz)), + Some(&image_bmap), + ) + .expect("with -b"); + } + + #[test] + fn check_conflicts_rejects_replacing_the_bmap_next_to_the_input_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_xz = dir.path().join(IMAGE_XZ); + let image = dir.path().join(IMAGE); + let other_bmap = dir.path().join("other.bmap"); + touch(&image_xz); + touch(&bmap::bmap_path(&image)); + touch(&other_bmap); + + // --input-bmap and --no-input-bmap + assert!( + check_conflicts( + &image_xz, + &image, + false, + Some(&bmap_of(&image_xz, &image)), + Some(&other_bmap) + ) + .is_err() + ); + assert!( + check_conflicts( + &image_xz, + &image, + false, + Some(&bmap_of(&image_xz, &image)), + None + ) + .is_err() + ); + } + + fn stage(dest: &Path, content: &'static [u8]) -> Staged { + Staged::write(dest, |file| Ok(file.write_all(content)?)).expect("stage") + } + + #[test] + fn staged_write_keeps_dest_when_write_fails() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + fs::write(&image, OLD_CONTENT).expect("write image"); + + assert!( + Staged::write(&image, |file| { + file.write_all(NEW_CONTENT).expect("write file"); + anyhow::bail!("write failed") + }) + .is_err() + ); + assert_eq!(fs::read(&image).expect("read image"), OLD_CONTENT); + assert_eq!( + fs::read_dir(dir.path()).expect("read dir").count(), + 1, + "temp dir was left" + ); + } + + #[test] + fn staged_registers_its_temp_dir_until_drop() { + let dir = tempfile::tempdir().expect("create temp dir"); + let staged = stage(&dir.path().join(IMAGE), NEW_CONTENT); + let tmp_dir = staged.tmp_dir.path().to_path_buf(); + + assert!(signal::is_registered(&tmp_dir)); + drop(staged); + assert!(!signal::is_registered(&tmp_dir)); + } + + #[test] + fn staged_commit_keeps_permissions_and_symlink() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let link = dir.path().join(OTHER_IMAGE); + fs::write(&image, OLD_CONTENT).expect("write image"); + fs::set_permissions(&image, fs::Permissions::from_mode(IMAGE_MODE)) + .expect("set permissions"); + symlink(&image, &link).expect("create symlink"); + + commit(stage(&link, NEW_CONTENT), None).expect("commit"); + + assert!(link.is_symlink()); + assert_eq!(fs::read(&image).expect("read image"), NEW_CONTENT); + assert_eq!( + fs::metadata(&image).expect("metadata").permissions().mode() & MODE_MASK, + IMAGE_MODE + ); + } + + #[test] + fn commit_leaves_no_old_bmap_when_the_bmap_rename_fails() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + fs::write(&image, OLD_CONTENT).expect("write image"); + fs::write(&image_bmap, OLD_CONTENT).expect("write bmap"); + + let staged_bmap = stage(&image_bmap, NEW_CONTENT); + fs::remove_file(&staged_bmap.file).expect("remove staged bmap"); + + assert!(commit(stage(&image, NEW_CONTENT), Some(staged_bmap)).is_err()); + assert_eq!(fs::read(&image).expect("read image"), NEW_CONTENT); + assert!(!image_bmap.exists()); + } + + #[test] + fn commit_names_an_old_bmap_that_stays_when_the_bmap_rename_fails() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + fs::write(&image, OLD_CONTENT).expect("write image"); + + let staged_bmap = stage(&image_bmap, NEW_CONTENT); + fs::remove_file(&staged_bmap.file).expect("remove staged bmap"); + // a dir cannot be removed as a file + fs::create_dir(&image_bmap).expect("create dir in place of the bmap"); + + let error = commit(stage(&image, NEW_CONTENT), Some(staged_bmap)).expect_err("commit"); + assert!(format!("{error:#}").contains("stays and does not match")); + } + + #[test] + fn commit_names_an_old_bmap_that_stays_next_to_an_image_kept_after_a_failed_copy() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + let staged_bmap = stage(&image_bmap, NEW_CONTENT); + // a dir cannot be removed as a file + fs::create_dir(&image_bmap).expect("create dir in place of the bmap"); + let mut staged_image = stage(&image, NEW_CONTENT); + staged_image.dest = PathBuf::from(UNRESIZABLE_FILE); + staged_image.in_tmpdir = true; + + let error = commit(staged_image, Some(staged_bmap)).expect_err("commit fails"); + assert!(format!("{error:#}").contains("stays and does not match")); + } + + #[test] + fn remove_old_bmap_names_a_truncated_bmap_incomplete() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_bmap = bmap::bmap_path(&dir.path().join(IMAGE)); + let mut staged_bmap = stage(&image_bmap, NEW_CONTENT); + fs::create_dir(&image_bmap).expect("create dir in place of the bmap"); + staged_bmap.tmp_dir.keep(); + + let error = remove_old_bmap(&staged_bmap, anyhow::anyhow!("copy failed")); + assert!(format!("{error:#}").contains("stays and is incomplete")); + } + + #[test] + fn commit_keeps_the_old_bmap_when_the_image_rename_fails() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + fs::write(&image, OLD_CONTENT).expect("write image"); + fs::write(&image_bmap, OLD_CONTENT).expect("write bmap"); + + let staged_image = stage(&image, NEW_CONTENT); + fs::remove_file(&staged_image.file).expect("remove staged image"); + + assert!(commit(staged_image, Some(stage(&image_bmap, NEW_CONTENT))).is_err()); + assert_eq!(fs::read(&image).expect("read image"), OLD_CONTENT); + assert_eq!(fs::read(&image_bmap).expect("read bmap"), OLD_CONTENT); + } + + #[test] + fn copy_into_dest_replaces_the_content() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + fs::write(&image, b"old image with more bytes").expect("write image"); + + let mut staged = stage(&image, NEW_CONTENT); + staged.copy_into_dest().expect("copy into dest"); + + assert_eq!(fs::read(&image).expect("read image"), NEW_CONTENT); + assert!(signal::is_registered(staged.tmp_dir.path())); + } + + #[test] + fn copy_into_dest_does_not_follow_a_symlink_put_in_place_of_dest() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let other = dir.path().join(OTHER_IMAGE); + fs::write(&image, OLD_CONTENT).expect("write image"); + fs::write(&other, OLD_CONTENT).expect("write other image"); + let mut staged = stage(&image, NEW_CONTENT); + fs::remove_file(&image).expect("remove image"); + symlink(&other, &image).expect("create symlink"); + + assert!(staged.copy_into_dest().is_err()); + assert_eq!(fs::read(&other).expect("read other image"), OLD_CONTENT); + } + + #[test] + fn copy_into_dest_creates_a_missing_dest() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let mut staged = stage(&image, NEW_CONTENT); + + staged.copy_into_dest().expect("copy into dest"); + assert_eq!(fs::read(&image).expect("read image"), NEW_CONTENT); + } + + #[test] + fn copy_into_dest_keeps_the_staged_file_when_the_copy_fails() { + let dir = tempfile::tempdir().expect("create temp dir"); + let mut staged = stage(&dir.path().join(IMAGE), NEW_CONTENT); + // a character device cannot be resized, so the copy fails after the open + staged.dest = PathBuf::from(UNRESIZABLE_FILE); + + assert!(staged.copy_into_dest().is_err()); + assert!(!signal::is_registered(staged.tmp_dir.path())); + let file = staged.file.clone(); + drop(staged); + assert_eq!(fs::read(&file).expect("read staged file"), NEW_CONTENT); + } + + #[test] + fn commit_keeps_the_new_bmap_next_to_an_image_kept_after_a_failed_copy() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + fs::write(&image_bmap, OLD_CONTENT).expect("write bmap"); + let mut staged_image = stage(&image, NEW_CONTENT); + staged_image.dest = PathBuf::from(UNRESIZABLE_FILE); + staged_image.in_tmpdir = true; + let kept_bmap = staged_image + .file + .with_file_name(image_bmap.file_name().expect("bmap name")); + + let e = + commit(staged_image, Some(stage(&image_bmap, NEW_CONTENT))).expect_err("commit fails"); + + assert!(format!("{e:#}").contains(&kept_bmap.display().to_string())); + assert_eq!(fs::read(&kept_bmap).expect("read kept bmap"), NEW_CONTENT); + assert!(!image_bmap.exists()); + } + + #[test] + fn is_mount_root_detects_the_root_dir() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + touch(&image); + + assert!(!is_mount_root(&image)); + + // kernels before 5.8 do not report mount roots, and `/` of a chroot is no mount root + let root = Path::new("/"); + let reported = rustix::fs::statx(CWD, root, AtFlags::empty(), StatxFlags::empty()) + .is_ok_and(|statx| { + statx + .stx_attributes_mask + .contains(StatxAttributes::MOUNT_ROOT) + }); + let mounted = fs::read_to_string(MOUNTINFO).is_ok_and(|mountinfo| { + mountinfo + .lines() + .any(|line| line.split(' ').nth(MOUNTINFO_MOUNT_POINT) == Some("/")) + }); + if reported && mounted { + assert!(is_mount_root(root)); + } + } + + #[test] + fn ensure_bmap_next_to_rejects_a_bmap_next_to_a_mount_point() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let root = Path::new("/"); + touch(&image); + + ensure_bmap_next_to(&image, &bmap::bmap_path(&image)).expect("no mount point"); + // the kernel cannot always tell, see is_mount_root_detects_the_root_dir + if is_mount_root(root) { + assert!(ensure_bmap_next_to(root, &bmap::bmap_path(&image)).is_err()); + ensure_bmap_next_to(root, root).expect("bmap is a mount point too"); + } + } + + #[test] + fn check_conflicts_rejects_replacing_the_bmap_of_another_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_xz = dir.path().join(IMAGE_XZ); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + touch(&image_xz); + touch(&image); + touch(&image_bmap); + + // bmaptool uses image.wic.bmap for both images + for input in [&image_xz, &image] { + let error = check_conflicts(input, input, false, Some(&image_bmap), None) + .expect_err("shared bmap"); + assert!(error.to_string().contains("which stays")); + } + touch(&bmap::bmap_path(&image_xz)); + check_conflicts(&image, &image, false, Some(&image_bmap), None).expect("own bmaps"); + fs::remove_file(&image_bmap).expect("remove bmap"); + fs::remove_file(bmap::bmap_path(&image_xz)).expect("remove bmap"); + // the new bmap would be the first bmap that bmaptool finds for image.wic.xz + assert!(check_conflicts(&image, &image, false, Some(&image_bmap), None).is_err()); + } + + #[test] + fn images_using_finds_the_images_bmaptool_maps_to_a_bmap() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_xz = dir.path().join(IMAGE_XZ); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + for path in [ + &image_xz, + &image, + &image_bmap, + &dir.path().join("image.wicked"), + &dir.path().join("image.wic.xz.sha256"), + &dir.path().join("image.wic.bmap.bak"), + ] { + touch(path); + } + + let mut images = images_using(&image_bmap).expect("images"); + images.sort(); + assert_eq!(images, vec![image, image_xz]); + } + + #[test] + fn images_using_finds_the_images_named_after_the_target_of_a_symlinked_bmap() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + let link_bmap = dir.path().join("latest.wic.bmap"); + touch(&image); + touch(&image_bmap); + symlink(&image_bmap, &link_bmap).expect("create symlink"); + + assert_eq!(images_using(&link_bmap).expect("images"), vec![image]); + } + + #[test] + fn commit_replaces_the_target_of_a_symlinked_bmap() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + let target_bmap = dir.path().join("target.bmap"); + fs::write(&image, OLD_CONTENT).expect("write image"); + fs::write(&target_bmap, OLD_CONTENT).expect("write bmap"); + symlink(&target_bmap, &image_bmap).expect("create symlink"); + + commit( + stage(&image, NEW_CONTENT), + Some(stage(&image_bmap, NEW_CONTENT)), + ) + .expect("commit"); + + assert!(image_bmap.is_symlink()); + assert_eq!(fs::read(&target_bmap).expect("read bmap"), NEW_CONTENT); + } + + #[test] + fn staged_commit_creates_the_target_of_a_dangling_symlink() { + let dir = tempfile::tempdir().expect("create temp dir"); + let target = dir.path().join("target").join(IMAGE); + let link = dir.path().join(IMAGE); + fs::create_dir(target.parent().expect("target dir")).expect("create dir"); + symlink(Path::new("target").join(IMAGE), &link).expect("create symlink"); + + commit(stage(&link, NEW_CONTENT), None).expect("commit"); + + assert!(link.is_symlink()); + assert_eq!(fs::read(&target).expect("read target"), NEW_CONTENT); + } + + #[test] + fn check_conflicts_rejects_a_dangling_derived_output() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_xz = dir.path().join(IMAGE_XZ); + let image = dir.path().join(IMAGE); + touch(&image_xz); + symlink(dir.path().join("missing.wic"), &image).expect("create symlink"); + + assert!(check_conflicts(&image_xz, &image, false, None, None).is_err()); + } + + #[test] + fn resolve_stops_at_a_symlink_loop() { + let dir = tempfile::tempdir().expect("create temp dir"); + let link = dir.path().join(IMAGE); + symlink(&link, &link).expect("create symlink"); + + assert!(resolve(&link).is_err()); + } + + #[test] + fn same_file_resolves_relative_and_missing_paths() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + touch(&image); + let relative = dir.path().join(".").join(IMAGE); + let missing = dir.path().join(OTHER_IMAGE); + + assert!(same_file(&image, &relative).expect("same file")); + assert!(same_file(&missing, &missing).expect("same file")); + assert!(!same_file(&image, &missing).expect("same file")); + } +} diff --git a/src/file/partition.rs b/src/file/partition.rs index 04d2fc6..eee0a8d 100644 --- a/src/file/partition.rs +++ b/src/file/partition.rs @@ -4,20 +4,42 @@ use std::io::Seek; use std::io::SeekFrom; use std::path::Path; +pub const SECTOR_SIZE: u64 = 512; + +#[derive(Debug)] pub struct PartitionData { pub num: u32, pub start: u64, pub count: u64, } +/// Fails if the partition does not fit into the image, so a corrupt partition table cannot make +/// a read or write go to another offset. pub fn get_partition_data>(path: P, partition_num: u32) -> Result { let path = path.as_ref(); let mut file = File::open(path).with_context(|| format!("failed to open image: {}", path.display()))?; + let data = find_partition(&mut file, partition_num)?; + let image_len = file + .metadata() + .context("failed to get the image size")? + .len(); + let end = data + .start + .checked_add(data.count) + .and_then(|end| end.checked_mul(SECTOR_SIZE)); + anyhow::ensure!( + end.is_some_and(|end| end <= image_len), + "partition {partition_num} ends after the end of the image" + ); + + Ok(data) +} +fn find_partition(file: &mut File, partition_num: u32) -> Result { // Try GPT first (validates CRC32 — more robust than signature check). // Capture any error so we can attach it as context if MBR also fails. - let gpt_err = match gptman::GPT::find_from(&mut file) { + let gpt_err = match gptman::GPT::find_from(file) { Ok(gpt) => { let entry = gpt .iter() @@ -29,10 +51,20 @@ pub fn get_partition_data>(path: P, partition_num: u32) -> Result entry.ending_lba >= entry.starting_lba, "GPT partition {partition_num} has invalid LBA range (ending_lba < starting_lba)" ); + // the callers count in SECTOR_SIZE units + let scale = gpt.sector_size / SECTOR_SIZE; + let sectors = |lba: u64| { + lba.checked_mul(scale) + .with_context(|| format!("GPT partition {partition_num} is too large")) + }; return Ok(PartitionData { num: partition_num, - start: entry.starting_lba, - count: entry.ending_lba - entry.starting_lba + 1, + start: sectors(entry.starting_lba)?, + count: sectors( + (entry.ending_lba - entry.starting_lba) + .checked_add(1) + .with_context(|| format!("GPT partition {partition_num} is too large"))?, + )?, }); } Err(e) => e, @@ -41,7 +73,7 @@ pub fn get_partition_data>(path: P, partition_num: u32) -> Result // Try MBR — iter() includes logical partitions (5, 6, ...) file.seek(SeekFrom::Start(0)) .context("failed to seek to start of image")?; - let mbr = mbrman::MBR::read_from(&mut file, 512) + let mbr = mbrman::MBR::read_from(file, SECTOR_SIZE as u32) .with_context(|| format!("image is neither valid GPT nor MBR (GPT error: {gpt_err})"))?; for (i, p) in mbr.iter() { if u32::try_from(i).ok() == Some(partition_num) && p.is_used() { @@ -59,3 +91,91 @@ pub fn is_gpt>(path: P) -> Result { let mut file = File::open(path.as_ref()).context("is_gpt: failed to open image")?; Ok(gptman::GPT::find_from(&mut file).is_ok()) } + +#[cfg(test)] +mod tests { + use crate::file::partition::*; + use gptman::{GPT, GPTPartitionEntry}; + use std::path::PathBuf; + + const GPT_SECTOR_SIZE: u64 = 4096; + const DISK_SECTORS: u64 = 64; + const PART_NUM: u32 = 1; + const PART_SECTORS: u64 = 8; + const GUID: [u8; 16] = [0xff; 16]; + // times 8 for 4K sectors and times 512 bytes is more than u64::MAX + const OVERFLOWING_LBA: u64 = 1 << 55; + + /// A 4K-sector GPT image with partition `PART_NUM` at `first_usable_lba`. + fn gpt_image(dir: &Path) -> (PathBuf, u64) { + let path = dir.join("image.wic"); + let mut file = File::create(&path).expect("create image"); + file.set_len(DISK_SECTORS * GPT_SECTOR_SIZE) + .expect("set image size"); + let mut gpt = GPT::new_from(&mut file, GPT_SECTOR_SIZE, GUID).expect("create gpt"); + let start = gpt.header.first_usable_lba; + gpt[PART_NUM] = GPTPartitionEntry { + partition_type_guid: GUID, + unique_partition_guid: GUID, + starting_lba: start, + ending_lba: start + PART_SECTORS - 1, + attribute_bits: 0, + partition_name: "part".into(), + }; + gpt.write_into(&mut file).expect("write gpt"); + (path, start) + } + + #[test] + fn get_partition_data_converts_4k_gpt_sectors() { + let dir = tempfile::tempdir().expect("create temp dir"); + let (path, start) = gpt_image(dir.path()); + + let data = get_partition_data(&path, PART_NUM).expect("partition data"); + let scale = GPT_SECTOR_SIZE / SECTOR_SIZE; + assert_eq!(data.start, start * scale); + assert_eq!(data.count, PART_SECTORS * scale); + } + + #[test] + fn get_partition_data_rejects_a_partition_after_the_image_end() { + let dir = tempfile::tempdir().expect("create temp dir"); + let (path, _) = gpt_image(dir.path()); + let file = File::options().write(true).open(&path).expect("open image"); + file.set_len(PART_SECTORS * GPT_SECTOR_SIZE) + .expect("set image size"); + + let error = get_partition_data(&path, PART_NUM).expect_err("partition after the end"); + assert!( + error + .to_string() + .contains("ends after the end of the image") + ); + } + + #[test] + fn get_partition_data_rejects_an_overflowing_offset() { + let dir = tempfile::tempdir().expect("create temp dir"); + let (path, _) = gpt_image(dir.path()); + // `GPT::write_into` refuses such an entry, the primary header and entries do not check it + let mut file = File::options() + .read(true) + .write(true) + .open(&path) + .expect("open image"); + let mut gpt = GPT::find_from(&mut file).expect("read gpt"); + gpt[PART_NUM].starting_lba = OVERFLOWING_LBA; + gpt[PART_NUM].ending_lba = OVERFLOWING_LBA + PART_SECTORS - 1; + let partitions: Vec<_> = gpt.iter().map(|(_, entry)| entry.clone()).collect(); + gpt.header + .write_into(&mut file, GPT_SECTOR_SIZE, &partitions) + .expect("write gpt header"); + + let error = get_partition_data(&path, PART_NUM).expect_err("overflowing offset"); + assert!( + error + .to_string() + .contains("ends after the end of the image") + ); + } +} diff --git a/src/file/signal.rs b/src/file/signal.rs new file mode 100644 index 0000000..778d588 --- /dev/null +++ b/src/file/signal.rs @@ -0,0 +1,212 @@ +use log::warn; +use signal_hook::{ + consts::{SIGHUP, SIGINT, SIGTERM}, + iterator::Signals, + low_level, +}; +use std::{ + collections::BTreeSet, + env, + ffi::c_int, + fs::{self, Permissions}, + io, + mem::MaybeUninit, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + process, ptr, + sync::{Mutex, MutexGuard, Once, PoisonError}, + thread, +}; +use tempfile::{Builder, TempDir}; + +const STOP_SIGNALS: [c_int; 3] = [SIGHUP, SIGINT, SIGTERM]; +// exit code of a process that a signal stopped, plus the signal number +const SIGNAL_EXIT_CODE_BASE: c_int = 128; +const DEFAULT_PREFIX: &str = ".tmp"; +// the dirs hold images and keys, so other users must not read them +const DIR_MODE: u32 = 0o700; + +static DIRS: Mutex> = Mutex::new(BTreeSet::new()); +static HANDLER: Once = Once::new(); + +/// A temp dir that is also removed if a stop signal ends the process. +pub struct RegisteredTempDir { + dir: TempDir, + kept: bool, +} + +impl RegisteredTempDir { + /// Creates the dir in TMPDIR. + pub fn create() -> io::Result { + Self::create_in(&env::temp_dir(), DEFAULT_PREFIX) + } + + /// Creates the dir in `parent`, with a name that starts with `prefix`. + pub fn create_in(parent: &Path, prefix: &str) -> io::Result { + let mut dir = Builder::new() + .prefix(prefix) + .permissions(Permissions::from_mode(DIR_MODE)) + .tempdir_in(parent)?; + // removed by Drop + dir.disable_cleanup(true); + register(dir.path()); + Ok(Self { dir, kept: false }) + } + + pub fn path(&self) -> &Path { + self.dir.path() + } + + /// Keeps the dir if a stop signal ends the process. Drop still removes it. + pub fn unregister(&self) { + unregister(self.path()); + } + + /// Removes the dir again if a stop signal ends the process. + pub fn register(&self) { + register(self.path()); + } + + /// Keeps the dir, also on drop. + pub fn keep(&mut self) { + self.unregister(); + self.kept = true; + } + + pub fn is_kept(&self) -> bool { + self.kept + } +} + +impl Drop for RegisteredTempDir { + /// Removes the dir before it unregisters it, so a stop signal during the removal of a large + /// file still removes the rest. + fn drop(&mut self) { + if !self.kept + && let Err(e) = fs::remove_dir_all(self.path()) + && e.kind() != io::ErrorKind::NotFound + { + warn!("cannot remove {}: {e}", self.path().display()); + } + self.unregister(); + } +} + +fn register(dir: &Path) { + HANDLER.call_once(start_handler); + dirs().insert(dir.to_path_buf()); +} + +fn unregister(dir: &Path) { + dirs().remove(dir); +} + +#[cfg(test)] +pub(crate) fn is_registered(dir: &Path) -> bool { + dirs().contains(dir) +} + +fn dirs() -> MutexGuard<'static, BTreeSet> { + DIRS.lock().unwrap_or_else(PoisonError::into_inner) +} + +fn start_handler() { + // e.g. `nohup` ignores SIGHUP, and a shell ignores SIGINT for a background job + let signals: Vec<_> = STOP_SIGNALS + .into_iter() + .filter(|signal| !is_ignored(*signal)) + .collect(); + let mut signals = match Signals::new(signals) { + Ok(signals) => signals, + Err(e) => { + warn!("cannot handle stop signals, so a stop leaves temp dirs: {e}"); + return; + } + }; + thread::spawn(move || { + if let Some(signal) = signals.forever().next() { + // the lock is held until the process ends, so no new dir is registered + let dirs = dirs(); + for dir in dirs.iter() { + if let Err(e) = fs::remove_dir_all(dir) { + warn!("cannot remove {}: {e}", dir.display()); + } + } + // the caller sees the signal as the cause of the stop + if let Err(e) = low_level::emulate_default_handler(signal) { + warn!("cannot stop with signal {signal}: {e}"); + } + process::exit(SIGNAL_EXIT_CODE_BASE + signal); + } + }); +} + +fn is_ignored(signal: c_int) -> bool { + let mut action = MaybeUninit::::zeroed(); + // SAFETY: without a new action, sigaction only writes the current one into `action` + let result = unsafe { libc::sigaction(signal, ptr::null(), action.as_mut_ptr()) }; + // SAFETY: the action was zeroed, and sigaction fills it on success + result == 0 && unsafe { action.assume_init() }.sa_sigaction == libc::SIG_IGN +} + +#[cfg(test)] +mod tests { + use crate::file::signal::*; + + const MODE_MASK: u32 = 0o777; + + #[test] + fn registered_temp_dir_is_registered_until_drop() { + let dir = RegisteredTempDir::create().expect("create temp dir"); + let path = dir.path().to_path_buf(); + + assert!(is_registered(&path)); + drop(dir); + assert!(!is_registered(&path)); + assert!(!path.exists()); + } + + #[test] + fn registered_temp_dir_is_private() { + let dir = RegisteredTempDir::create().expect("create temp dir"); + + let mode = fs::metadata(dir.path()) + .expect("metadata") + .permissions() + .mode(); + assert_eq!(mode & MODE_MASK, DIR_MODE); + } + + #[test] + fn registered_temp_dir_register_again_after_unregister() { + let dir = RegisteredTempDir::create().expect("create temp dir"); + + dir.unregister(); + assert!(!is_registered(dir.path())); + dir.register(); + assert!(is_registered(dir.path())); + } + + #[test] + fn registered_temp_dir_keep_unregisters_and_keeps_the_dir() { + let mut dir = RegisteredTempDir::create().expect("create temp dir"); + let path = dir.path().to_path_buf(); + + assert!(!dir.is_kept()); + dir.keep(); + assert!(dir.is_kept()); + assert!(!is_registered(&path)); + drop(dir); + assert!(path.exists()); + fs::remove_dir(&path).expect("remove dir"); + } + + #[test] + fn is_ignored_reads_the_disposition() { + // SAFETY: SIGUSR2 is not used by the tests + unsafe { libc::signal(libc::SIGUSR2, libc::SIG_IGN) }; + + assert!(is_ignored(libc::SIGUSR2)); + assert!(!is_ignored(libc::SIGUSR1)); + } +} diff --git a/src/file/sparse.rs b/src/file/sparse.rs new file mode 100644 index 0000000..76a80a1 --- /dev/null +++ b/src/file/sparse.rs @@ -0,0 +1,216 @@ +use anyhow::{Context, Result}; +use rustix::{ + fs::{SeekFrom, seek}, + io::Errno, +}; +use std::{ + fs::File, + io::{self, Read, Seek}, + ops::Range, + path::Path, +}; + +/// Returns the byte ranges of `file` between `start` and `end` that hold data. +pub fn data_ranges(file: &File, start: u64, end: u64) -> Result>> { + let mut ranges = vec![]; + let mut pos = start; + while pos < end { + let data = match seek(file, SeekFrom::Data(pos)) { + Ok(data) => data, + // no data after pos + Err(Errno::NXIO) => break, + Err(e) => return Err(e).context("data_ranges: cannot find data"), + }; + if data >= end { + break; + } + let hole = seek(file, SeekFrom::Hole(data)).context("data_ranges: cannot find hole")?; + ranges.push(data..hole.min(end)); + pos = hole; + } + + Ok(ranges) +} + +/// Sorts `ranges` and joins the ones that overlap or touch. +pub fn merge(mut ranges: Vec>) -> Vec> { + ranges.sort_by_key(|r| r.start); + let mut merged: Vec> = Vec::with_capacity(ranges.len()); + for range in ranges { + match merged.last_mut() { + Some(last) if range.start <= last.end => last.end = last.end.max(range.end), + _ => merged.push(range), + } + } + merged +} + +/// Copies `len` bytes of `src` at `src_offset` to `dst` at `dst_offset`. Holes of `src` are +/// skipped, so they stay holes in a new `dst`. +fn copy_range(src: &File, src_offset: u64, dst: &File, dst_offset: u64, len: u64) -> Result<()> { + let src_len = src + .metadata() + .context("copy_range: cannot get source size")? + .len(); + anyhow::ensure!( + src_offset + len <= src_len, + "copy_range: source has {src_len} bytes, cannot copy {len} bytes at {src_offset}" + ); + + for data in data_ranges(src, src_offset, src_offset + len)? { + let mut src = src; + let mut dst = dst; + src.seek(io::SeekFrom::Start(data.start)) + .context("copy_range: cannot seek source")?; + dst.seek(io::SeekFrom::Start(data.start - src_offset + dst_offset)) + .context("copy_range: cannot seek destination")?; + // one copy_file_range call copies at most 2 GiB; io::copy repeats it until a range is done + let copied = io::copy(&mut src.take(data.end - data.start), &mut dst) + .context("copy_range: cannot copy data")?; + anyhow::ensure!( + copied == data.end - data.start, + "copy_range: source ended early" + ); + } + + Ok(()) +} + +/// Creates `dst` with `len` bytes and copies `len` bytes of `src` at `src_offset` into it. +/// Holes of `src` stay holes in `dst`. +pub fn copy_to_new_file(src: &File, src_offset: u64, dst: &Path, len: u64) -> Result<()> { + let dst = + File::create(dst).context(format!("copy_to_new_file: cannot create {}", dst.display()))?; + + copy_to_empty_file(src, src_offset, &dst, len) +} + +/// Sets the size of the empty `dst` to `len` and copies `len` bytes of `src` at `src_offset` +/// into it. Holes of `src` stay holes in `dst`. +fn copy_to_empty_file(src: &File, src_offset: u64, dst: &File, len: u64) -> Result<()> { + dst.set_len(len) + .context("copy_to_empty_file: cannot set destination size")?; + + copy_range(src, src_offset, dst, 0, len) +} + +/// Copies the file `src` into the empty `dst`. Holes of `src` stay holes in `dst`. +pub fn copy_file(src: &Path, dst: &File) -> Result<()> { + let src_file = File::open(src).context(format!("copy_file: cannot open {}", src.display()))?; + let len = src_file + .metadata() + .context(format!( + "copy_file: cannot get the size of {}", + src.display() + ))? + .len(); + + copy_to_empty_file(&src_file, 0, dst, len) +} + +#[cfg(test)] +mod tests { + use crate::file::sparse::*; + use std::os::unix::fs::FileExt; + + const BLOCK_SIZE: u64 = 4096; + const FILE_BLOCKS: u64 = 8; + const DATA_BLOCK: u64 = 2; + const DATA_BYTE: u8 = 0x5a; + const DST_OFFSET_BLOCKS: u64 = 1; + + /// A sparse file with data in `DATA_BLOCK` and holes everywhere else. + fn sparse_file(dir: &std::path::Path) -> File { + let file = File::options() + .read(true) + .write(true) + .create_new(true) + .open(dir.join("src")) + .expect("create file"); + file.set_len(FILE_BLOCKS * BLOCK_SIZE).expect("set size"); + file.write_all_at(&[DATA_BYTE; BLOCK_SIZE as usize], DATA_BLOCK * BLOCK_SIZE) + .expect("write data block"); + file + } + + #[test] + fn data_ranges_finds_data_between_holes() { + let dir = tempfile::tempdir().expect("create temp dir"); + let file = sparse_file(dir.path()); + let data = DATA_BLOCK * BLOCK_SIZE..(DATA_BLOCK + 1) * BLOCK_SIZE; + + assert_eq!( + data_ranges(&file, 0, FILE_BLOCKS * BLOCK_SIZE).expect("data ranges"), + vec![data.clone()] + ); + assert_eq!( + data_ranges(&file, 0, data.start).expect("data ranges"), + vec![] + ); + assert_eq!( + data_ranges(&file, data.start + 1, data.end - 1).expect("data ranges"), + vec![data.start + 1..data.end - 1] + ); + } + + #[test] + fn merge_joins_overlapping_and_touching_ranges() { + assert_eq!( + merge(vec![8..10, 0..2, 1..4, 4..5, 12..12]), + vec![0..5, 8..10, 12..12] + ); + } + + #[test] + fn copy_to_new_file_keeps_holes() { + let dir = tempfile::tempdir().expect("create temp dir"); + let src = sparse_file(dir.path()); + let dst_path = dir.path().join("dst"); + let len = (FILE_BLOCKS - DST_OFFSET_BLOCKS) * BLOCK_SIZE; + + copy_to_new_file(&src, DST_OFFSET_BLOCKS * BLOCK_SIZE, &dst_path, len) + .expect("copy to new file"); + + let dst = File::open(&dst_path).expect("open file"); + assert_eq!(dst.metadata().expect("metadata").len(), len); + let data_start = (DATA_BLOCK - DST_OFFSET_BLOCKS) * BLOCK_SIZE; + assert_eq!( + data_ranges(&dst, 0, len).expect("data ranges"), + vec![data_start..data_start + BLOCK_SIZE] + ); + let mut block = [0u8; BLOCK_SIZE as usize]; + dst.read_exact_at(&mut block, data_start) + .expect("read block"); + assert!(block.iter().all(|b| *b == DATA_BYTE)); + } + + #[test] + fn copy_file_keeps_holes() { + let dir = tempfile::tempdir().expect("create temp dir"); + sparse_file(dir.path()); + let dst = File::options() + .read(true) + .write(true) + .create_new(true) + .open(dir.path().join("dst")) + .expect("create file"); + + copy_file(&dir.path().join("src"), &dst).expect("copy file"); + + let len = FILE_BLOCKS * BLOCK_SIZE; + assert_eq!(dst.metadata().expect("metadata").len(), len); + assert_eq!( + data_ranges(&dst, 0, len).expect("data ranges"), + vec![DATA_BLOCK * BLOCK_SIZE..(DATA_BLOCK + 1) * BLOCK_SIZE] + ); + } + + #[test] + fn copy_range_rejects_range_after_source_end() { + let dir = tempfile::tempdir().expect("create temp dir"); + let src = sparse_file(dir.path()); + let dst = File::create(dir.path().join("dst")).expect("create file"); + + assert!(copy_range(&src, BLOCK_SIZE, &dst, 0, FILE_BLOCKS * BLOCK_SIZE).is_err()); + } +} diff --git a/src/lib.rs b/src/lib.rs index 343f912..88e9280 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,140 +17,169 @@ use cli::{ SetConfig, SetDeviceCertificate, SetDeviceCertificateNoEst, SetEdgeCaCertificate, SetIotLeafSasConfig, SetIotedgeGatewayConfig, }, + ImageOutput, IotHubDeviceUpdate::{self, SetDeviceConfig as IotHubDeviceUpdateSet}, SshConfig::{SetCertificate, SetConnection}, }; -use file::{compression::Compression, functions::FileCopyToParams}; -use log::error; -use std::{fs, path::Path, path::PathBuf}; -use tokio::fs::remove_dir_all; -use uuid::Uuid; - -use crate::file::compression; - -struct TempDirGuard(PathBuf); - -impl Drop for TempDirGuard { - fn drop(&mut self) { - let Ok(rt) = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - else { - error!("cannot create tokio runtime"); - return; +use file::{ + bmap::{self, Bmap}, + compression::{self, Compression}, + functions::FileCopyToParams, + output::{self, Staged, check_conflicts, output_image}, + signal::RegisteredTempDir, + sparse, +}; +use log::info; +use std::{ + fs, + path::{Path, PathBuf}, +}; + +fn run_image_command(image_file: PathBuf, output: ImageOutput, command: F) -> Result<()> +where + F: FnOnce(&PathBuf) -> Result<()>, +{ + let ImageOutput { + output_bmap, + input_bmap, + no_input_bmap, + output_image: explicit_output, + compress_image: target_compression, + } = output; + + ensure_image_exists(&image_file)?; + + let input_bmap = if output_bmap && !no_input_bmap { + bmap::find_input_bmap(&image_file, input_bmap.as_deref())? + .map(|path| { + Bmap::read(&path) + .with_context(|| input_bmap_error(&path)) + .map(|bmap| (path, bmap)) + }) + .transpose()? + } else { + None + }; + let source_compression = Compression::from_file(&image_file)?; + let dest_image_file = output_image( + &image_file, + source_compression.as_ref(), + target_compression.as_ref(), + explicit_output.as_deref(), + )?; + let dest_bmap_file = output_bmap + .then(|| output::output_bmap_path(&image_file, &dest_image_file)) + .transpose()?; + check_conflicts( + &image_file, + &dest_image_file, + explicit_output.is_some(), + dest_bmap_file.as_deref(), + input_bmap.as_ref().map(|(path, _)| path.as_path()), + )?; + + let verify_input_bmap = |image: &Path| -> Result<()> { + let Some((path, bmap)) = &input_bmap else { + return Ok(()); }; + bmap::verify_checksums(image, bmap).with_context(|| input_bmap_error(path)) + }; - rt.block_on(async { - if let Err(e) = remove_dir_all(self.0.clone()).await { - error!("cannot remove tmp dir: {e}") + // a wrong bmap of an uncompressed image fails before the copy + if source_compression.is_none() { + verify_input_bmap(&image_file)?; + } + let tmp_dir = RegisteredTempDir::create().context("cannot create a temp dir")?; + let tmp_image_file = temp_image(&image_file, source_compression.as_ref(), tmp_dir.path())?; + if source_compression.is_some() { + verify_input_bmap(&tmp_image_file)?; + } + + // run command + command(&tmp_image_file)?; + + let image = Staged::write(&dest_image_file, |dest| match &target_compression { + Some(c) => compression::compress(&tmp_image_file, c, dest), + None => sparse::copy_file(&tmp_image_file, dest), + })?; + let bmap = dest_bmap_file + .map(|dest_bmap_file| { + match &input_bmap { + Some((path, _)) => info!("the bmap maps the ranges of {}", path.display()), + None => info!("no input bmap, so the bmap maps the whole image"), } + Staged::write(&dest_bmap_file, |dest| { + bmap::write_bmap( + &tmp_image_file, + input_bmap.as_ref().map(|(_, bmap)| bmap), + dest, + ) + }) }) - } + .transpose()?; + + output::commit(image, bmap) +} + +fn input_bmap_error(path: &Path) -> String { + format!( + "cannot use input bmap {}; pass the right one with --input-bmap, or map the whole \ + image with --no-input-bmap", + path.display() + ) } -fn run_image_command( - image_file: PathBuf, - generate_bmap: bool, - target_compression: Option, - command: F, -) -> Result<()> +/// Runs `command` on the image, or on a decompressed temp copy of a compressed image. +fn run_read_only_image_command(image_file: PathBuf, command: F) -> Result<()> where F: FnOnce(&PathBuf) -> Result<()>, { - if let Ok("true") | Ok("1") = std::env::var("CONTAINERIZED").as_deref() { - anyhow::ensure!( - !generate_bmap, - "run_image_command: generating bmap file is not supported in containerized environments." - ); + ensure_image_exists(&image_file)?; + + match Compression::from_file(&image_file)? { + Some(c) => { + let tmp_dir = RegisteredTempDir::create().context("cannot create a temp dir")?; + command(&compression::decompress(&image_file, &c, tmp_dir.path())?) + } + None => command(&image_file), } +} +fn ensure_image_exists(image_file: &Path) -> Result<()> { anyhow::ensure!( image_file.try_exists().is_ok_and(|exists| exists), - "run_image_command: image doesn't exist {}", - image_file.to_str().context("cannot get image file path")? - ); - - let mut dest_image_file = image_file.clone(); - - // create /tmp/{uuid}/ and copy image into - let tmp_dir = PathBuf::from(format!("/tmp/{}", Uuid::new_v4())); - fs::create_dir_all(tmp_dir.clone()).context(format!( - "run_image_command: couldn't create destination path {}", - tmp_dir.to_str().context("cannot get tmp dir name")? - ))?; - - let _guard = TempDirGuard(tmp_dir.clone()); - - let mut tmp_image_file = tmp_dir.join( - image_file - .file_name() - .context("cannot get image file name")?, + "image doesn't exist {}", + image_file.display() ); + Ok(()) +} - // if applicable decompress image to *.wic - if let Some(source_compression) = Compression::from_file(&image_file)? { - std::fs::copy(&image_file, &tmp_image_file)?; - tmp_image_file = compression::decompress(&tmp_image_file, &source_compression)?; - dest_image_file.set_extension(""); - } else { - // copy sparse file (std::fs::copy isn't able) - libfs::copy_file(&image_file, &tmp_image_file).context(format!( - "error: libfs::copy_file({:?}, {:?})", - image_file, tmp_image_file - ))?; - } - - // run command - command(&tmp_image_file)?; - - // create and copy back bmap file if one was created - if generate_bmap { - let mut target_bmap = image_file - .parent() - .context("cannot get parent dir of image path")? - .to_path_buf(); - let tmp_bmap = PathBuf::from(format!( - "{}.bmap", - tmp_image_file - .to_str() - .context("cannot get image file path")? - )); - file::functions::generate_bmap_file( - tmp_image_file - .to_str() - .context("cannot get image file path")?, - )?; - target_bmap.push(tmp_bmap.file_name().context("cannot get bmap file name")?); - std::fs::copy(&tmp_bmap, &target_bmap).context(format!( - "error: std::fs::copy({:?}, {:?})", - tmp_bmap, target_bmap - ))?; - } - - // if applicable compress image - if let Some(c) = target_compression { - tmp_image_file = compression::compress(&tmp_image_file, &c)?; - dest_image_file.set_file_name( - tmp_image_file - .file_name() - .context("cannot get image file name")?, - ); - std::fs::copy(&tmp_image_file, &dest_image_file).context(format!( - "error: std::fs::copy({:?}, {:?})", - tmp_image_file, dest_image_file - ))?; - } else { - // copy sparse file (std::fs::copy isn't able) - libfs::copy_file(&tmp_image_file, &dest_image_file).context(format!( - "error: libfs::copy_file({:?}, {:?})", - tmp_image_file, dest_image_file - ))?; +/// Copies `image_file` into `tmp_dir`, decompressed if it is compressed, and returns the copy. +fn temp_image( + image_file: &Path, + source_compression: Option<&Compression>, + tmp_dir: &Path, +) -> Result { + match source_compression { + Some(c) => compression::decompress(image_file, c, tmp_dir), + None => { + let tmp_image_file = tmp_dir.join( + image_file + .file_name() + .context("cannot get image file name")?, + ); + let tmp_image = fs::File::create(&tmp_image_file) + .context(format!("cannot create {}", tmp_image_file.display()))?; + sparse::copy_file(image_file, &tmp_image)?; + Ok(tmp_image_file) + } } - - Ok(()) } +/// A new device certificate and key in a temp dir, so parallel runs do not share them and no +/// private key is left behind, also not after a stop signal. struct CertInfo { + _dir: RegisteredTempDir, cert_path: PathBuf, key_path: PathBuf, } @@ -164,7 +193,7 @@ struct CertificateOptions<'a> { validity_days: u32, } -fn create_image_cert(image: &Path, cert_opts: CertificateOptions) -> Result { +fn create_image_cert(cert_opts: CertificateOptions) -> Result { let intermediate_full_chain_cert_str = std::fs::read_to_string(cert_opts.intermediate_full_chain_cert) .context("create_and_set_image_cert: couldn't read intermediate fullchain cert")?; @@ -178,16 +207,20 @@ fn create_image_cert(image: &Path, cert_opts: CertificateOptions) -> Result Result<()> { @@ -197,9 +230,8 @@ pub fn run() -> Result<()> { image, partition, dest, - generate_bmap, - compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img| { + output, + }) => run_image_command(image, output, |img| { anyhow::ensure!( dest.to_string_lossy().ends_with(".tar.gz"), format!( @@ -237,9 +269,8 @@ pub fn run() -> Result<()> { config, image, payload, - generate_bmap, - compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img| { + output, + }) => run_image_command(image, output, |img| { file::set_identity_config(&config, img, payload.as_deref()) })?, Command::Identity(SetDeviceCertificate { @@ -248,23 +279,19 @@ pub fn run() -> Result<()> { image, device_id, days, - generate_bmap, - compress_image, + output, }) => { - let cert_info = create_image_cert( - &image, - CertificateOptions { - intermediate_full_chain_cert: &intermediate_full_chain_cert, - intermediate_key: &intermediate_key, - target_cert: "device_cert_path.pem", - target_key: "device_key_path.key.pem", - subject: &device_id, - validity_days: days, - }, - ) + let cert_info = create_image_cert(CertificateOptions { + intermediate_full_chain_cert: &intermediate_full_chain_cert, + intermediate_key: &intermediate_key, + target_cert: "device_cert_path.pem", + target_key: "device_key_path.key.pem", + subject: &device_id, + validity_days: days, + }) .context("set_device_certificate: could not create certificate")?; - run_image_command(image, generate_bmap, compress_image, |img| { + run_image_command(image, output, |img| { file::set_device_cert( Some(&intermediate_full_chain_cert), &cert_info.cert_path, @@ -279,23 +306,19 @@ pub fn run() -> Result<()> { image, device_id, days, - generate_bmap, - compress_image, + output, }) => { - let cert_info = create_image_cert( - &image, - CertificateOptions { - intermediate_full_chain_cert: &intermediate_full_chain_cert, - intermediate_key: &intermediate_key, - target_cert: "edge_ca_cert_path.pem", - target_key: "edge_ca_key_path.key.pem", - subject: &device_id, - validity_days: days, - }, - ) + let cert_info = create_image_cert(CertificateOptions { + intermediate_full_chain_cert: &intermediate_full_chain_cert, + intermediate_key: &intermediate_key, + target_cert: "edge_ca_cert_path.pem", + target_key: "edge_ca_key_path.key.pem", + subject: &device_id, + validity_days: days, + }) .context("set_edge_ca_certificate: could not create certificate")?; - run_image_command(image, generate_bmap, compress_image, |img| { + run_image_command(image, output, |img| { file::set_edge_ca_cert( Some(&intermediate_full_chain_cert), &cert_info.cert_path, @@ -308,9 +331,8 @@ pub fn run() -> Result<()> { device_cert: device_cert_pem, device_key: device_key_pem, image, - generate_bmap, - compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img| { + output, + }) => run_image_command(image, output, |img| { file::set_device_cert(None, &device_cert_pem, &device_key_pem, img) })?, Command::Identity(SetIotedgeGatewayConfig { @@ -319,9 +341,8 @@ pub fn run() -> Result<()> { root_ca, device_identity, device_identity_key, - generate_bmap, - compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img: &PathBuf| { + output, + }) => run_image_command(image, output, |img: &PathBuf| { file::set_iotedge_gateway_config( &config, img, @@ -334,25 +355,22 @@ pub fn run() -> Result<()> { config, image, root_ca, - generate_bmap, - compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img: &PathBuf| { + output, + }) => run_image_command(image, output, |img: &PathBuf| { file::set_iot_leaf_sas_config(&config, img, &root_ca) })?, Command::Ssh(SetCertificate { image, root_ca, - generate_bmap, - compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img: &PathBuf| { + output, + }) => run_image_command(image, output, |img: &PathBuf| { file::set_ssh_tunnel_certificate(img, &root_ca) })?, Command::IotHubDeviceUpdate(IotHubDeviceUpdateSet { iot_hub_device_update_config, image, - generate_bmap, - compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img: &PathBuf| { + output, + }) => run_image_command(image, output, |img: &PathBuf| { file::set_iot_hub_device_update_config(&iot_hub_device_update_config, img) })?, Command::IotHubDeviceUpdate(IotHubDeviceUpdate::ImportUpdate { @@ -467,15 +485,14 @@ pub fn run() -> Result<()> { Command::File(CopyToImage { file_copy_params, image, - generate_bmap, - compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img: &PathBuf| { + output, + }) => run_image_command(image, output, |img: &PathBuf| { file::copy_to_image(&file_copy_params, img) })?, Command::File(CopyFromImage { file_copy_params, image, - }) => run_image_command(image, false, None, |img: &PathBuf| { + }) => run_read_only_image_command(image, |img: &PathBuf| { file::copy_from_image(&file_copy_params, img) })?, } diff --git a/testfiles/image.wic.bmap b/testfiles/image.wic.bmap new file mode 100644 index 0000000..7b84de9 --- /dev/null +++ b/testfiles/image.wic.bmap @@ -0,0 +1,69 @@ + + + + + + 30408704 + + + 4096 + + + 7424 + + + 77 + + + sha256 + + + 43b997b9550b07ad422eddddba268d0f0607660f664592a7d749034a4f8b9eea + + + + 0-1 + 1024-1025 + 2048-2055 + 2058-2059 + 2062-2063 + 2190-2191 + 3072-3079 + 3082-3083 + 4094-4103 + 4106-4107 + 5104-5105 + 5120-5127 + 5130-5131 + 6128-6129 + 6144-6151 + 6154-6155 + 7152-7153 + 7168-7175 + 7178-7179 + 7423 + + diff --git a/testfiles/image.wic.xz b/testfiles/image.wic.xz index b86e932..6053d9a 100644 Binary files a/testfiles/image.wic.xz and b/testfiles/image.wic.xz differ diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index ac3e716..97773c3 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -3,9 +3,17 @@ use assert_cmd::Command; use assert_json_diff::assert_json_eq; use common::Testrunner; use httpmock::prelude::*; -use omnect_cli::ssh; +use omnect_cli::{ + file::bmap::{Bmap, bmap_path}, + ssh, +}; use predicates::prelude::*; -use std::{fs::create_dir_all, path::PathBuf}; +use std::{ + ffi::OsStr, + fs::{self, create_dir_all}, + os::unix::fs::MetadataExt, + path::{Path, PathBuf}, +}; use stdext::function_name; #[test] @@ -476,6 +484,14 @@ fn check_set_identity_config_tpm_template() { ); } +/// Fails if a file with one of `names` is in the directory of `image`. +fn assert_no_files_next_to(image: &Path, names: &[&str]) { + for name in names { + let path = image.with_file_name(name); + assert!(!path.exists(), "{} was left behind", path.display()); + } +} + #[test] fn check_set_device_cert_est() { let tr = Testrunner::new(function_name!().split("::").last().unwrap()); @@ -499,6 +515,10 @@ fn check_set_device_cert_est() { .arg("1") .assert(); assert.success(); + assert_no_files_next_to( + &image_path, + &["device_cert_path.pem", "device_key_path.key.pem"], + ); let mut device_id_cert_out_path = tr.pathbuf(); device_id_cert_out_path.push("dir1"); @@ -630,6 +650,10 @@ fn check_set_edge_ca_cert() { .arg("1") .assert(); assert.success(); + assert_no_files_next_to( + &image_path, + &["edge_ca_cert_path.pem", "edge_ca_key_path.key.pem"], + ); let mut edge_ca_cert_out_path = tr.pathbuf(); edge_ca_cert_out_path.push("dir1"); @@ -895,12 +919,480 @@ fn check_file_copy(tr: Testrunner, partition: &str) { assert!(file_diff::diff(in_file4, out_file4)); } +// unit of `st_blocks` +const STAT_BLOCK_SIZE: u64 = 512; +// content an earlier installation left on the device +const OLD_CONTENT_BYTE: u8 = 0xa5; +// a file with a zero range that spans whole image blocks +const ZERO_BLOCKS_FILE_EDGE: usize = 4096; +const ZERO_BLOCKS_FILE_ZEROS: usize = 64 * 1024; +// a valid bmap of a one-block image, so only the image size check rejects it +const OTHER_IMAGE_BMAP: &str = "4096\ + 4096sha256\ + 0\ + "; +const OTHER_IMAGE_SIZE_ERROR: &str = "bmap is for an image of 4096 bytes"; + +fn read_bmap(path: &Path) -> Bmap { + Bmap::read(path).expect("read bmap") +} + +fn mapped_blocks(bmap: &Bmap) -> u64 { + bmap.ranges.iter().map(|r| r.end() - r.start() + 1).sum() +} + +fn maps_whole_image(bmap: &Bmap) -> bool { + mapped_blocks(bmap) == bmap.image_size.div_ceil(bmap.block_size) +} + +fn maps_all_ranges_of(bmap: &Bmap, input: &Bmap) -> bool { + input + .ranges + .iter() + .flat_map(|r| r.clone()) + .all(|b| bmap.ranges.iter().any(|r| r.contains(&b))) +} + +/// Writes the blocks `bmap` maps onto a device that still holds old content, as a bmap flash +/// does. +fn bmap_flash(image: &[u8], bmap: &Bmap) -> Vec { + let mut device = vec![OLD_CONTENT_BYTE; image.len()]; + for range in &bmap.ranges { + let start = usize::try_from(range.start() * bmap.block_size).expect("start"); + let end = usize::try_from((range.end() + 1) * bmap.block_size).expect("end"); + let end = end.min(image.len()); + device[start..end].copy_from_slice(&image[start..end]); + } + device +} + +fn sparse_copy(from: &Path, to: &Path) { + let status = std::process::Command::new("cp") + .arg("--sparse=always") + .arg(from) + .arg(to) + .status() + .expect("run cp"); + assert!(status.success(), "sparse copy of the test image failed"); +} + +#[test] +fn check_file_copy_without_input_bmap_maps_whole_image() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let bmap_path = bmap_path(&image_path); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},factory:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .arg("-b") + .assert() + .success(); + + assert!(maps_whole_image(&read_bmap(&bmap_path))); +} + +#[test] +fn check_file_copy_keeps_holes() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let input_bmap_path = tr.to_pathbuf("testfiles/image.wic.bmap"); + let sparse_path = image_path.with_file_name("sparse.wic"); + let bmap_path = bmap_path(&sparse_path); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + + sparse_copy(&image_path, &sparse_path); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},factory:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&sparse_path) + .arg("--input-bmap") + .arg(&input_bmap_path) + .arg("-b") + .assert() + .success(); + + let meta = sparse_path.metadata().expect("image metadata"); + assert!(meta.blocks() * STAT_BLOCK_SIZE < meta.len()); + let bmap = read_bmap(&bmap_path); + let input_bmap = read_bmap(&input_bmap_path); + assert!(!maps_whole_image(&bmap)); + assert!(maps_all_ranges_of(&bmap, &input_bmap)); + + // every data block and every block of the input bmap must replace the old content + let image = fs::read(&sparse_path).expect("read image"); + let device = bmap_flash(&image, &bmap); + let block_size = usize::try_from(input_bmap.block_size).expect("block size"); + let blocks = image.chunks(block_size).zip(device.chunks(block_size)); + for (block, (image_block, device_block)) in (0u64..).zip(blocks) { + let needed = image_block.iter().any(|b| *b != 0) + || input_bmap.ranges.iter().any(|r| r.contains(&block)); + assert!( + !needed || image_block == device_block, + "block {block} keeps old content" + ); + } +} + +#[test] +fn check_file_copy_maps_zero_blocks_of_copied_files() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let input_bmap_path = tr.to_pathbuf("testfiles/image.wic.bmap"); + let sparse_path = image_path.with_file_name("sparse.wic"); + let bmap_path = bmap_path(&sparse_path); + let device_path = tr.pathbuf().join("device.wic"); + let in_file = tr.pathbuf().join("zero-blocks.bin"); + let in_content = [ + vec![0x11; ZERO_BLOCKS_FILE_EDGE], + vec![0; ZERO_BLOCKS_FILE_ZEROS], + vec![0x22; ZERO_BLOCKS_FILE_EDGE], + ] + .concat(); + fs::write(&in_file, &in_content).expect("write input file"); + sparse_copy(&image_path, &sparse_path); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},factory:/my-file", in_file.to_string_lossy())) + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&sparse_path) + .arg("--input-bmap") + .arg(&input_bmap_path) + .arg("-b") + .assert() + .success(); + + let image = fs::read(&sparse_path).expect("read image"); + fs::write(&device_path, bmap_flash(&image, &read_bmap(&bmap_path))).expect("write device"); + + for partition in ["factory", "boot"] { + let out_file = tr.pathbuf().join(format!("{partition}-out.bin")); + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-from-image") + .arg("-f") + .arg(format!( + "{partition}:/my-file,{}", + out_file.to_string_lossy() + )) + .arg("-i") + .arg(&device_path) + .assert() + .success(); + assert!( + fs::read(&out_file).expect("read output file") == in_content, + "{partition}: file differs after a bmap flash" + ); + } +} + +#[test] +fn check_input_bmap_of_another_image() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let bmap_path = bmap_path(&image_path); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + let out_file = tr.pathbuf().join("hosts"); + fs::write(&bmap_path, OTHER_IMAGE_BMAP).expect("write bmap"); + let image_hash = Testrunner::file_hash(&image_path); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .arg("-b") + .assert() + .failure() + .stderr(predicate::str::contains("cannot use input bmap")) + .stderr(predicate::str::contains(OTHER_IMAGE_SIZE_ERROR)); + assert_eq!(Testrunner::file_hash(&image_path), image_hash); + + // reading from the image does not use the bmap + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-from-image") + .arg("-f") + .arg(format!("rootA:/etc/hosts,{}", out_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .assert() + .success(); +} + +#[test] +fn check_file_copy_without_output_bmap_keeps_image_sparse() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let sparse_path = image_path.with_file_name("sparse.wic"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + sparse_copy(&image_path, &sparse_path); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},factory:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&sparse_path) + .assert() + .success(); + + let meta = sparse_path.metadata().expect("image metadata"); + assert!( + meta.blocks() * STAT_BLOCK_SIZE < meta.len(), + "image is not sparse" + ); +} + +#[test] +fn check_output_image_that_is_no_regular_file() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic.xz"); + let written_image_path = image_path.with_extension(""); + let bmap_path = bmap_path(&written_image_path); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + create_dir_all(&written_image_path).expect("create directory"); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .arg("-b") + .arg("-o") + .arg(&written_image_path) + .assert() + .failure() + .stderr(predicate::str::contains("is not a regular file")); + assert!(written_image_path.is_dir(), "the directory was replaced"); + assert!( + !bmap_path.exists(), + "bmap written for an image that was not written back" + ); +} + +#[test] +fn check_input_bmap_requires_output_bmap() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let bmap_path = tr.to_pathbuf("testfiles/image.wic.bmap"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + let image_hash = Testrunner::file_hash(&image_path); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .arg("--input-bmap") + .arg(&bmap_path) + .assert() + .failure(); + assert_eq!(Testrunner::file_hash(&image_path), image_hash); +} + +#[test] +fn check_input_bmap_of_another_image_with_the_same_size() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let bmap_path = tr.to_pathbuf("testfiles/image.wic.bmap"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + let bmap = read_bmap(&bmap_path); + // another build: one byte differs in a block the bmap maps + let mapped_byte = usize::try_from(bmap.ranges[0].start() * bmap.block_size).expect("offset"); + let mut image = fs::read(&image_path).expect("read image"); + image[mapped_byte] = !image[mapped_byte]; + fs::write(&image_path, image).expect("write image"); + let image_hash = Testrunner::file_hash(&image_path); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .arg("-b") + .assert() + .failure() + .stderr(predicate::str::contains("cannot use input bmap")) + .stderr(predicate::str::contains("--input-bmap")); + assert_eq!(Testrunner::file_hash(&image_path), image_hash); +} + +#[test] +fn check_chained_commands_with_input_bmap() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_xz_path = tr.to_pathbuf("testfiles/image.wic.xz"); + let image_xz_hash = Testrunner::file_hash(&image_xz_path); + let input_bmap_path = tr.to_pathbuf("testfiles/image.wic.bmap"); + let input_bmap_hash = Testrunner::file_hash(&input_bmap_path); + let device_path = tr.pathbuf().join("device.wic"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + let copy_to_image = |image: &Path, args: &[&OsStr]| { + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(image) + .args(args) + .assert() + }; + + // image.wic would be written next to image.wic.bmap without updating it + copy_to_image(&image_xz_path, &[]) + .failure() + .stderr(predicate::str::contains("pass -b to write a matching bmap")); + // bmaptool would use image.wic.bmap for the changed image.wic.xz + copy_to_image(&image_xz_path, &["-p".as_ref(), "xz".as_ref()]) + .failure() + .stderr(predicate::str::contains("pass -b to write a matching bmap")); + // the bmap of image.wic would replace the bmap of image.wic.xz + copy_to_image(&image_xz_path, &["-b".as_ref()]) + .failure() + .stderr(predicate::str::contains("--output-image")); + copy_to_image(&image_xz_path, &["-b".as_ref(), "--no-input-bmap".as_ref()]) + .failure() + .stderr(predicate::str::contains("--output-image")); + + copy_to_image( + &image_xz_path, + &["-b".as_ref(), "-o".as_ref(), device_path.as_os_str()], + ) + .success(); + copy_to_image(&device_path, &["-b".as_ref()]).success(); + copy_to_image(&device_path, &[]) + .failure() + .stderr(predicate::str::contains("pass -b to write a matching bmap")); + + assert_eq!(Testrunner::file_hash(&image_xz_path), image_xz_hash); + assert_eq!(Testrunner::file_hash(&input_bmap_path), input_bmap_hash); + assert!(maps_all_ranges_of( + &read_bmap(&bmap_path(&device_path)), + &read_bmap(&input_bmap_path) + )); +} + +#[test] +fn check_derived_output_image_exists() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_xz_path = tr.to_pathbuf("testfiles/image.wic.xz"); + let image_path = image_xz_path.with_extension(""); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + fs::write(&image_path, "an older image").expect("write image"); + let copy_to_image = |args: &[&OsStr]| { + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_xz_path) + .args(args) + .assert() + }; + + copy_to_image(&[]) + .failure() + .stderr(predicate::str::contains("--output-image")); + copy_to_image(&[ + "-o".as_ref(), + image_path.as_os_str(), + "-p".as_ref(), + "xz".as_ref(), + ]) + .failure() + .stderr(predicate::str::contains( + "does not end in an extension of the -p compression that bmaptool reads; name it e.g. ", + )); + // a path given with -o is overwritten + copy_to_image(&["-o".as_ref(), image_path.as_os_str()]).success(); + assert_ne!( + fs::read(&image_path).expect("read image"), + b"an older image" + ); +} + +#[test] +fn check_no_input_bmap() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let bmap_path = bmap_path(&image_path); + let other_bmap_path = tr.pathbuf().join("other.bmap"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + fs::write(&bmap_path, OTHER_IMAGE_BMAP).expect("write bmap"); + fs::write(&other_bmap_path, OTHER_IMAGE_BMAP).expect("write bmap"); + let copy_to_image = |args: &[&OsStr]| { + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .args(args) + .assert() + }; + + copy_to_image(&["--no-input-bmap".as_ref()]).failure(); + copy_to_image(&[ + "-b".as_ref(), + "--no-input-bmap".as_ref(), + "--input-bmap".as_ref(), + other_bmap_path.as_os_str(), + ]) + .failure(); + copy_to_image(&["-b".as_ref()]) + .failure() + .stderr(predicate::str::contains("--no-input-bmap")) + .stderr(predicate::str::contains(OTHER_IMAGE_SIZE_ERROR)); + + copy_to_image(&["-b".as_ref(), "--no-input-bmap".as_ref()]).success(); + assert!(maps_whole_image(&read_bmap(&bmap_path))); +} + #[test] fn check_bmap_generation_wic() { let tr = Testrunner::new(function_name!().split("::").last().unwrap()); let image_path = tr.to_pathbuf("testfiles/image.wic"); let image_path_copy = PathBuf::from(format!("{}.copy", image_path.to_str().unwrap())); - let bmap_path = PathBuf::from(format!("{}.bmap", image_path.to_str().unwrap())); + let bmap_path = bmap_path(&image_path); let in_file = tr.to_pathbuf("testfiles/boot.scr"); let in_file = in_file.to_str().unwrap(); @@ -947,9 +1439,10 @@ fn check_bmap_generation_wic() { fn check_bmap_generation_wic_xz() { let tr = Testrunner::new(function_name!().split("::").last().unwrap()); let image_path_wic_xz = tr.to_pathbuf("testfiles/image.wic.xz"); - let image_path_wic = image_path_wic_xz.with_extension(""); + let image_path_wic = tr.pathbuf().join("device.wic"); let image_path_wic_copy = image_path_wic_xz.with_extension("copy"); - let image_path_bmap = image_path_wic_xz.with_extension("bmap"); + let image_path_bmap = bmap_path(&image_path_wic); + let input_bmap = read_bmap(&tr.to_pathbuf("testfiles/image.wic.bmap")); let in_file = tr.to_pathbuf("testfiles/boot.scr"); let in_file = in_file.to_str().unwrap(); @@ -962,9 +1455,15 @@ fn check_bmap_generation_wic_xz() { .arg("-i") .arg(&image_path_wic_xz) .arg("-b") + .arg("-o") + .arg(&image_path_wic) .assert(); assert.success(); + let bmap = read_bmap(&image_path_bmap); + assert!(!maps_whole_image(&bmap)); + assert!(maps_all_ranges_of(&bmap, &input_bmap)); + // use bmaptool to verify that the checksum of the bmap file and the image // still match after the copy operations let assert = Command::new("bmaptool") @@ -976,6 +1475,147 @@ fn check_bmap_generation_wic_xz() { assert.success(); } +#[test] +fn check_bmap_generation_wic_xz_without_input_bmap() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path_wic_xz = tr.to_pathbuf("testfiles/image.wic.xz"); + let image_path_wic = tr.pathbuf().join("device.wic"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + let copy_to_image = |output: Option<&Path>| { + let mut command = Command::cargo_bin("omnect-cli").expect("omnect-cli binary"); + command + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path_wic_xz) + .arg("-b"); + if let Some(output) = output { + command.arg("-o").arg(output); + } + command.assert() + }; + + // bmaptool would use the new image.wic.bmap also for image.wic.xz, which stays + copy_to_image(None) + .failure() + .stderr(predicate::str::contains("which stays")); + copy_to_image(Some(&image_path_wic)).success(); + + assert!(maps_whole_image(&read_bmap(&bmap_path(&image_path_wic)))); +} + +#[test] +fn check_bmap_of_compressed_output_keeps_input_bmap() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let input_bmap_path = tr.to_pathbuf("testfiles/image.wic.bmap"); + let input_bmap_hash = Testrunner::file_hash(&input_bmap_path); + let image_path_xz = image_path.with_extension("wic.xz"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .arg("-b") + .arg("-p") + .arg("xz") + .assert() + .success(); + + assert_eq!( + Testrunner::file_hash(&input_bmap_path), + input_bmap_hash, + "bmap of the unchanged input image was replaced" + ); + // bmaptool finds .bmap without --bmap and checks the image against it + Command::new("bmaptool") + .arg("copy") + .arg(&image_path_xz) + .arg(image_path.with_extension("copy")) + .assert() + .success() + .stderr(predicate::str::contains("image.wic.xz.bmap")); +} + +#[test] +fn check_bmap_of_compressed_image_changed_in_place_replaces_the_input_bmap() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path_xz = tr.to_pathbuf("testfiles/image.wic.xz"); + let input_bmap_path = tr.to_pathbuf("testfiles/image.wic.bmap"); + let input_bmap_hash = Testrunner::file_hash(&input_bmap_path); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path_xz) + .arg("-b") + .arg("-p") + .arg("xz") + .assert() + .success(); + + assert_ne!( + Testrunner::file_hash(&input_bmap_path), + input_bmap_hash, + "bmap of the replaced image stayed" + ); + assert!(!bmap_path(&image_path_xz).exists()); + Command::new("bmaptool") + .arg("copy") + .args(["--bmap", &input_bmap_path.to_string_lossy()]) + .arg(&image_path_xz) + .arg(image_path_xz.with_extension("copy")) + .assert() + .success(); +} + +#[test] +fn check_copy_from_image_to_a_file_in_the_current_dir() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + let out_file = "boot.out"; + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .assert() + .success(); + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .current_dir(tr.pathbuf()) + .arg("file") + .arg("copy-from-image") + .arg("-f") + .arg(format!("boot:/my-file,{out_file}")) + .arg("-i") + .arg(&image_path) + .assert() + .success(); + + assert_eq!( + Testrunner::file_hash(&tr.pathbuf().join(out_file)), + Testrunner::file_hash(&in_file) + ); +} + #[test] fn check_image_compression() { let tr = Testrunner::new(function_name!().split("::").last().unwrap()); @@ -1057,6 +1697,78 @@ fn check_image_decompression() { assert_eq!(image_path_wic_xz_hash1, image_path_wic_xz_hash2); } +#[test] +fn check_copy_from_image_writes_no_image() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path_wic_xz = tr.to_pathbuf("testfiles/image.wic.xz"); + let image_hash = Testrunner::file_hash(&image_path_wic_xz); + let out_file = tr.pathbuf().join("hosts"); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-from-image") + .arg("-f") + .arg(format!("rootA:/etc/hosts,{}", out_file.to_string_lossy())) + .arg("-i") + .arg(&image_path_wic_xz) + .assert() + .success(); + + assert!(out_file.exists()); + assert!(!image_path_wic_xz.with_extension("").exists()); + assert_eq!(Testrunner::file_hash(&image_path_wic_xz), image_hash); +} + +#[test] +fn check_partition_files_stay_out_of_the_image_dir() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + // the name of the file that holds partition 1 + let image_path = tr.pathbuf().join("1.img"); + fs::rename(tr.to_pathbuf("testfiles/image.wic"), &image_path).expect("rename image"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + let out_file = tr.pathbuf().join("my-file"); + let file_image = |command: &str, files: String| { + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg(command) + .arg("-f") + .arg(files) + .arg("-i") + .arg(&image_path) + .assert() + .success(); + }; + let dir_entries = || { + let mut entries = fs::read_dir(tr.pathbuf()) + .expect("read test dir") + .map(|entry| entry.expect("dir entry").file_name()) + .collect::>(); + entries.sort(); + entries + }; + + file_image( + "copy-to-image", + format!("{},boot:/my-file", in_file.to_string_lossy()), + ); + let entries = dir_entries(); + file_image( + "copy-from-image", + format!("boot:/my-file,{}", out_file.to_string_lossy()), + ); + + assert_eq!( + fs::read(&out_file).expect("read out file"), + fs::read(&in_file).expect("read in file") + ); + let mut expected = entries; + expected.push(out_file.file_name().expect("file name").to_owned()); + expected.sort(); + assert_eq!(dir_entries(), expected); +} + #[tokio::test] async fn check_ssh_tunnel_setup() { let tr = Testrunner::new("check_ssh_tunnel_setup");