From 06cf3df3bd5c161f69d1070489f74d00b015d011 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:04:10 +0100 Subject: [PATCH 1/6] fix(io): create og's temporary output files new and owner-only AtomicFile created its temporary file with File::create on a name made of the target and the pid. The mode followed the umask, so under umask 022 a new output was 0644 and a replaced 0600 output became 0644. File::create also opened whatever was already at that name, and two AtomicFiles on one target in one process shared one temporary file. The temporary file is now opened with create_new (O_CREAT | O_EXCL) and mode 0600, or the replaced target's mode when that is stricter. Its name carries a process-wide counter, and a taken name moves on to the next. Bead: rv-io-owner-only-writes --- rust/crates/og-io/src/lib.rs | 76 +++++++++-- rust/crates/og-io/tests/atomic_modes.rs | 118 ++++++++++++++++++ .../crates/og-io/tests/atomic_planted_temp.rs | 55 ++++++++ 3 files changed, 238 insertions(+), 11 deletions(-) create mode 100644 rust/crates/og-io/tests/atomic_modes.rs create mode 100644 rust/crates/og-io/tests/atomic_planted_temp.rs diff --git a/rust/crates/og-io/src/lib.rs b/rust/crates/og-io/src/lib.rs index 18d0652b4..4901ed3a3 100644 --- a/rust/crates/og-io/src/lib.rs +++ b/rust/crates/og-io/src/lib.rs @@ -6,7 +6,8 @@ //! - [`Lines`]: Python's text mode with `newline=None`: `\n`, `\r\n` and a //! lone `\r` each end a line and read as `\n`; UTF-8 strict or `replace`. //! - [`AtomicFile`]: written beside the target and renamed over it on -//! [`AtomicFile::commit`], so a reader never sees half a file. +//! [`AtomicFile::commit`], so a reader never sees half a file. What it +//! writes is owner-only (0600) whatever the umask. //! - [`bgzf`]: the same reading with BGZF inflated on many cores, and a //! BGZF writer that compresses on many cores. //! - [`tabix`]: the lines of a region of a bgzipped file, through its @@ -22,6 +23,7 @@ use flate2::bufread::GzDecoder; use std::fs::{self, File}; use std::io::{self, BufRead, BufReader, BufWriter, Read, Write}; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; const GZIP_MAGIC: [u8; 2] = [0x1f, 0x8b]; @@ -198,28 +200,52 @@ impl Iterator for Lines { /// A file written under a temporary name and renamed into place by /// [`commit`](AtomicFile::commit). Dropped without a commit, the temporary /// file is removed and the target is untouched. +/// +/// The temporary file is always a new file: a file or link already at its +/// name is left alone and the next name is tried. On Unix it is created +/// mode 0600, or with the mode of the target it replaces when that is +/// stricter, so the umask never makes an output readable by others. pub struct AtomicFile { tmp: PathBuf, target: PathBuf, out: Option>, } +/// The number in the next temporary name, so two files open in one process +/// never share one. +static NEXT_TMP: AtomicU64 = AtomicU64::new(0); + +/// How many temporary names are tried before giving up. +const TMP_TRIES: usize = 64; + impl AtomicFile { pub fn create(target: impl AsRef) -> io::Result { let target = target.as_ref().to_path_buf(); let name = target .file_name() .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "no file name"))?; - let mut tmp_name = std::ffi::OsString::from("."); - tmp_name.push(name); - tmp_name.push(format!(".tmp-{}", std::process::id())); - let tmp = target.with_file_name(tmp_name); - let out = BufWriter::new(File::create(&tmp)?); - Ok(AtomicFile { - tmp, - target, - out: Some(out), - }) + let mode = temp_mode(&target); + let pid = std::process::id(); + let mut taken = None; + for _ in 0..TMP_TRIES { + let n = NEXT_TMP.fetch_add(1, Ordering::Relaxed); + let mut tmp_name = std::ffi::OsString::from("."); + tmp_name.push(name); + tmp_name.push(format!(".tmp-{pid}-{n}")); + let tmp = target.with_file_name(tmp_name); + match new_file(&tmp, mode) { + Ok(f) => { + return Ok(AtomicFile { + tmp, + target, + out: Some(BufWriter::new(f)), + }); + } + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => taken = Some(e), + Err(e) => return Err(e), + } + } + Err(taken.expect("at least one name was tried")) } /// Flushes and renames over the target. On an error the temporary file @@ -240,6 +266,34 @@ impl AtomicFile { } } +/// The mode of a new temporary file: 0600, less any owner bit the target +/// it replaces lacks (a 0400 target stays 0400). +#[cfg(unix)] +fn temp_mode(target: &Path) -> u32 { + use std::os::unix::fs::PermissionsExt; + match fs::metadata(target) { + Ok(m) if m.is_file() => 0o600 & m.permissions().mode(), + _ => 0o600, + } +} + +#[cfg(not(unix))] +fn temp_mode(_target: &Path) -> u32 { + 0 +} + +/// Creates `path`, failing with `AlreadyExists` when anything is there, +/// a link included (`O_CREAT | O_EXCL`). +fn new_file(path: &Path, mode: u32) -> io::Result { + let mut options = fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + std::os::unix::fs::OpenOptionsExt::mode(&mut options, mode); + #[cfg(not(unix))] + let _ = mode; + options.open(path) +} + impl Write for AtomicFile { fn write(&mut self, buf: &[u8]) -> io::Result { self.out.as_mut().expect("not committed").write(buf) diff --git a/rust/crates/og-io/tests/atomic_modes.rs b/rust/crates/og-io/tests/atomic_modes.rs new file mode 100644 index 000000000..1947c518c --- /dev/null +++ b/rust/crates/og-io/tests/atomic_modes.rs @@ -0,0 +1,118 @@ +//! The modes and temporary names of [`og_io::AtomicFile`]: what it writes is +//! owner-only whatever the umask, and two writers in one process never share +//! a temporary file. + +use og_io::AtomicFile; +use std::fs; +use std::io::Write; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::Command; + +const CHILD: &str = "OG_IO_UMASK_022_CHILD"; + +fn scratch(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("og-io-{name}-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + dir +} + +/// The permission bits in octal, so a failure reads `644`, not `420`. +fn mode(path: &Path) -> String { + format!( + "{:o}", + fs::metadata(path).unwrap().permissions().mode() & 0o7777 + ) +} + +fn write(target: &Path, text: &str) { + let mut f = AtomicFile::create(target).unwrap(); + f.write_all(text.as_bytes()).unwrap(); + f.commit().unwrap(); +} + +/// Runs itself again in a child under `umask 022`, the umask most shells +/// start with, and checks that the child ran this test and passed it. +#[test] +fn owner_only_under_umask_022() { + if std::env::var_os(CHILD).is_none() { + let exe = std::env::current_exe().unwrap(); + let out = Command::new("/bin/sh") + .arg("-c") + .arg("umask 022 && exec \"$0\" --exact owner_only_under_umask_022 --nocapture --test-threads=1") + .arg(&exe) + .env(CHILD, "1") + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&out.stdout); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + out.status.success() && stdout.contains("1 passed"), + "the umask 022 child failed or ran no test:\n{stdout}\n{stderr}" + ); + return; + } + let dir = scratch("umask"); + // The control: under this umask a plain new file is 0644. + let control = dir.join("control"); + fs::File::create(&control).unwrap(); + assert_eq!( + mode(&control), + "644", + "the child does not run under umask 022" + ); + + // A new target. + let new = dir.join("new.tsv"); + write(&new, "new\n"); + assert_eq!(mode(&new), "600", "a new target"); + + // A 0600 target that is replaced stays 0600. + let private = dir.join("private.tsv"); + fs::write(&private, "old\n").unwrap(); + fs::set_permissions(&private, fs::Permissions::from_mode(0o600)).unwrap(); + write(&private, "new\n"); + assert_eq!(mode(&private), "600", "a replaced 0600 target"); + assert_eq!(fs::read_to_string(&private).unwrap(), "new\n"); + + // A 0644 target becomes owner-only. + let open = dir.join("open.tsv"); + fs::write(&open, "old\n").unwrap(); + fs::set_permissions(&open, fs::Permissions::from_mode(0o644)).unwrap(); + write(&open, "new\n"); + assert_eq!(mode(&open), "600", "a replaced 0644 target"); + + // A stricter target keeps its mode. + let read_only = dir.join("read-only.tsv"); + fs::write(&read_only, "old\n").unwrap(); + fs::set_permissions(&read_only, fs::Permissions::from_mode(0o400)).unwrap(); + write(&read_only, "new\n"); + assert_eq!(mode(&read_only), "400", "a replaced 0400 target"); + assert_eq!(fs::read_to_string(&read_only).unwrap(), "new\n"); + + fs::remove_dir_all(&dir).unwrap(); +} + +/// Two files open on one target in one process each write their own +/// temporary file: both commits succeed and the last one wins whole. +#[test] +fn two_writers_on_one_target() { + let dir = scratch("two"); + let target = dir.join("out.tsv"); + let mut a = AtomicFile::create(&target).unwrap(); + let mut b = AtomicFile::create(&target).unwrap(); + a.write_all(b"from a\n").unwrap(); + b.write_all(b"from b\n").unwrap(); + a.commit().unwrap(); + assert_eq!(fs::read_to_string(&target).unwrap(), "from a\n"); + b.commit().unwrap(); + assert_eq!(fs::read_to_string(&target).unwrap(), "from b\n"); + // Nothing but the target is left. + let names: Vec<_> = fs::read_dir(&dir) + .unwrap() + .map(|e| e.unwrap().file_name()) + .collect(); + assert_eq!(names, ["out.tsv"]); + fs::remove_dir_all(&dir).unwrap(); +} diff --git a/rust/crates/og-io/tests/atomic_planted_temp.rs b/rust/crates/og-io/tests/atomic_planted_temp.rs new file mode 100644 index 000000000..e6d3bd7c5 --- /dev/null +++ b/rust/crates/og-io/tests/atomic_planted_temp.rs @@ -0,0 +1,55 @@ +//! [`og_io::AtomicFile`] creates its temporary file and never opens one that +//! is already there. This is the only test in its binary, so the first +//! temporary name this process tries is known: `..tmp--0`. + +use og_io::AtomicFile; +use std::fs; +use std::io::Write; +use std::os::unix::fs::symlink; + +#[test] +fn a_link_at_the_temporary_name_is_left_alone() { + let pid = std::process::id(); + let dir = std::env::temp_dir().join(format!("og-io-planted-{pid}")); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + let target = dir.join("out.tsv"); + let other = dir.join("other.tsv"); + fs::write(&other, "keep\n").unwrap(); + let missing = dir.join("missing.tsv"); + // Links at the first two names this process tries, and at the name + // releases before 1.0.0 used (`..tmp-`). + let planted = [ + (format!(".out.tsv.tmp-{pid}"), &other), + (format!(".out.tsv.tmp-{pid}-0"), &other), + (format!(".out.tsv.tmp-{pid}-1"), &missing), + ]; + for (name, to) in &planted { + symlink(to, dir.join(name)).unwrap(); + } + + let mut f = AtomicFile::create(&target).unwrap(); + f.write_all(b"new\n").unwrap(); + f.commit().unwrap(); + + assert_eq!(fs::read_to_string(&other).unwrap(), "keep\n"); + assert!( + !missing.exists(), + "a file was created through a dangling link" + ); + assert!( + !fs::symlink_metadata(&target) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!(fs::read_to_string(&target).unwrap(), "new\n"); + for (name, to) in &planted { + assert_eq!( + &fs::read_link(dir.join(name)).unwrap(), + *to, + "{name} changed" + ); + } + fs::remove_dir_all(&dir).unwrap(); +} From 80fd49444eea3553d768469a74e660a571d19ba6 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:04:10 +0100 Subject: [PATCH 2/6] docs: say og writes owner-only files and keep --outdir private Bead: rv-io-owner-only-writes --- CHANGELOG.md | 1 + docs/nextflow.md | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05c3388e1..46c499382 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- The files the `og` binary writes are owner-only (mode 0600) whatever the umask. Before, `og` wrote each output under a temporary name with the umask's mode, so under the common umask 022 an output was readable by every user of the machine, and replacing a 0600 file made it 0644. The step scripts set umask 077, so a run through them was already owner-only; `og` run by hand or by the Nextflow pipeline was not. Now the temporary file is created mode 0600, or with the mode of the file it replaces when that is stricter (a 0400 file stays 0400). It is always a new file: a file or link already at its name is left as it is and the next name is tried, and two outputs written at once to one name in one process get two temporary files. The temporary name gains a number, `..tmp--`. No output name or content changes. [Nextflow Execution](docs/nextflow.md#run-the-pipeline) now says to keep `--outdir` private. - The self-test of `scripts/ci/check-images.sh` copies the whole tree again once the list of tracked paths passes 128 KiB. It piped that list through `xargs` to `tar`, and GNU `xargs` split a list that long over two `tar` runs; the reading `tar` stopped after the first archive, so Guard's `images` job failed on a broken pipe (exit 123). One `tar` now reads the list itself (`tar --null -T -`). Nothing the check reports changes. - The phased family VCF is indexed by the Rust `og` binary. WhatsHap's step (step 38 and the Nextflow WHATSHAP_PED process) wrote `family.phased.vcf.gz.tbi` with a Python one-liner, `pysam.tabix_index`. It now runs `og vcf index -f`. The `.tbi` keeps its name and format, and its bytes, once inflated, are the ones htslib 1.24 writes (the htslib of the pysam and bcftools the pipeline pins): the same bins in the same order, the same chunks and the same linear index. Only the deflate stream can differ, as two deflate libraries differ. On a 5-million-record three-sample VCF it took 0.75 to 0.91 s and 31 MB at peak, where htslib's own `tabix` took 0.81 to 0.93 s and 19 MB. - As in htslib 1.24, a record ends at the largest of INFO/END, the REF length, the SVLEN of a ``, ``, `` or `` ALT, and a sample's LEN on a gVCF block. A line with no position to read, unsorted records, a sequence that starts again, a record past 2^29 and an empty file stop the index with a message, and nothing is written. Without `-f` it never writes over an index, even one that appears while it runs. diff --git a/docs/nextflow.md b/docs/nextflow.md index 6332f369e..8e6d79fca 100644 --- a/docs/nextflow.md +++ b/docs/nextflow.md @@ -91,6 +91,8 @@ nextflow run main.nf \ # it names (two or more, each with a gVCF; docs/multi-sample.md) ``` +**Keep `--outdir` private.** The results hold genome data. Create the folder before the run with `mkdir -m 700 results`, or put it in a folder only you can open. The `og` tools write their files owner-only (mode 0600), but Nextflow creates the folders, and the other tools write their files, with the modes your umask gives. + **Family workflow (opt-in).** `--pedigree` (tab separated: sample, sex, father, mother, partner; [`assets/pedigree.example.tsv`](https://github.com/GeiserX/OwnGenome/blob/main/assets/pedigree.example.tsv)) runs `workflows/family.nf` over the samplesheet rows it names. `--pedigree` may be a families file (a leading family column, [`assets/families.example.tsv`](https://github.com/GeiserX/OwnGenome/blob/main/assets/families.example.tsv)). `--family_name` names the family to run, which a file with more than one family needs, and the folder its results go to (`//` instead of `/family/`). The workflow does the same work as step 38: `FAMILY_PREPARE` checks the pedigree and that each member's gVCF names its own sample, `GLNEXUS` and `GLNEXUS_VCF` joint-call the gVCFs (`GLNEXUS` asks for 16 GB and gives GLnexus half of its memory as `--mem-gbytes`), `FAMILY_RELATE` and `FAMILY_CHECK` stop the run when somalier's relatedness of a pair disagrees with the pedigree (a sample swap), `WHATSHAP_PED` phases the children of complete trios by inheritance, and `FAMILY_REPORT` writes each child's allele origin, de novo candidates when both parents are sequenced, and compound heterozygotes among the clinical filter's candidates. With `--ancestry_fine_panel` (the panel folder of step 26b, `setup.sh --ancestry-fine-panel`), four more processes find the DNA each pair shares by descent, once `FAMILY_CHECK` has passed: `FAMILY_IBD_SITES` keeps the joint call's genotypes at the panel's SNVs on each autosome (a chromosome with fewer than 100 is left out), `FAMILY_IBD_PHASE` phases them with Beagle against the panel, `FAMILY_IBD_SEGMENTS` runs hap-ibd (its default 2 cM minimum) with the panel's genetic maps, and `FAMILY_IBD_REPORT` writes `ibd_pairs.tsv`, `ibd_segments.tsv` and `ibd_report.md`. They feed only the report, so a failure of one is logged and the run goes on. Without the panel none of them runs. Each member needs a gVCF: DEEPVARIANT's for a FASTQ or BAM row, the `gvcf` column for a VCF row (the run stops on a VCF row without one). Fewer than two named samples, or no `--somalier_sites`, stops the run at the start. When the pedigree marks two of those samples as each other's partner, `COUPLE_CARRIER` runs the couple carrier screen of step 39 on them ([Couple Carrier Screen](39-couple.md)): it reads each partner's ClinVar hits, so it needs `clinvar` in `--tools` (without it the run logs a warning and skips the screen), uses deletions over an exon only with `--gencode_exons` (the GENCODE exons BED `setup.sh` installs as `reference/gencode.v50.basic.exons.bed`), reads Delly's calls, ExpansionHunter's, and Parascopy's SMN1, interval and CYP21A2 tables when those steps ran, and, when `--annotsv_annotations` holds AnnotSV's benign loss set (`benign_Loss_SV_GRCh38.sorted.bed`, with gnomAD-SV's common deletions merged in by `setup.sh --annotsv-data`), sets aside a Delly deletion that matches a common one there. Without `--pedigree` none of these processes exists and a run is unchanged. What it checks and what the files hold: [Multi-sample comparison](multi-sample.md#family-analysis-step-38-opt-in). `--reference` is the FASTA the reads are aligned to and the BAMs were aligned to, the GRCh38 no-ALT analysis set that `setup.sh` installs as `reference/GRCh38_no_alt_analysis_set.fasta`. The Nextflow pipeline does not compare a given BAM's header with it; `./scripts/validate-setup.sh ` does, and [Realigning after a reference change](realignment.md) covers a BAM aligned to another reference. From 2817f5980c8ff112b49b357a7d27fec122e5644d Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 22:12:02 +0100 Subject: [PATCH 3/6] docs: say og's atomic writer, not every og write, is owner-only --- CHANGELOG.md | 2 +- docs/nextflow.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46c499382..1e09bc81a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- The files the `og` binary writes are owner-only (mode 0600) whatever the umask. Before, `og` wrote each output under a temporary name with the umask's mode, so under the common umask 022 an output was readable by every user of the machine, and replacing a 0600 file made it 0644. The step scripts set umask 077, so a run through them was already owner-only; `og` run by hand or by the Nextflow pipeline was not. Now the temporary file is created mode 0600, or with the mode of the file it replaces when that is stricter (a 0400 file stays 0400). It is always a new file: a file or link already at its name is left as it is and the next name is tried, and two outputs written at once to one name in one process get two temporary files. The temporary name gains a number, `..tmp--`. No output name or content changes. [Nextflow Execution](docs/nextflow.md#run-the-pipeline) now says to keep `--outdir` private. +- The files `og` writes through its atomic writer (every analysis output) are owner-only (mode 0600) whatever the umask. Before, `og` wrote each output under a temporary name with the umask's mode, so under the common umask 022 an output was readable by every user of the machine, and replacing a 0600 file made it 0644. The step scripts set umask 077, so a run through them was already owner-only; `og` run by hand or by the Nextflow pipeline was not. Now the temporary file is created mode 0600, or with the mode of the file it replaces when that is stricter (a 0400 file stays 0400). It is always a new file: a file or link already at its name is left as it is and the next name is tried, and two outputs written at once to one name in one process get two temporary files. The temporary name gains a number, `..tmp--`. No output name or content changes. [Nextflow Execution](docs/nextflow.md#run-the-pipeline) now says to keep `--outdir` private. - The self-test of `scripts/ci/check-images.sh` copies the whole tree again once the list of tracked paths passes 128 KiB. It piped that list through `xargs` to `tar`, and GNU `xargs` split a list that long over two `tar` runs; the reading `tar` stopped after the first archive, so Guard's `images` job failed on a broken pipe (exit 123). One `tar` now reads the list itself (`tar --null -T -`). Nothing the check reports changes. - The phased family VCF is indexed by the Rust `og` binary. WhatsHap's step (step 38 and the Nextflow WHATSHAP_PED process) wrote `family.phased.vcf.gz.tbi` with a Python one-liner, `pysam.tabix_index`. It now runs `og vcf index -f`. The `.tbi` keeps its name and format, and its bytes, once inflated, are the ones htslib 1.24 writes (the htslib of the pysam and bcftools the pipeline pins): the same bins in the same order, the same chunks and the same linear index. Only the deflate stream can differ, as two deflate libraries differ. On a 5-million-record three-sample VCF it took 0.75 to 0.91 s and 31 MB at peak, where htslib's own `tabix` took 0.81 to 0.93 s and 19 MB. - As in htslib 1.24, a record ends at the largest of INFO/END, the REF length, the SVLEN of a ``, ``, `` or `` ALT, and a sample's LEN on a gVCF block. A line with no position to read, unsorted records, a sequence that starts again, a record past 2^29 and an empty file stop the index with a message, and nothing is written. Without `-f` it never writes over an index, even one that appears while it runs. diff --git a/docs/nextflow.md b/docs/nextflow.md index 8e6d79fca..ef5ddefad 100644 --- a/docs/nextflow.md +++ b/docs/nextflow.md @@ -91,7 +91,7 @@ nextflow run main.nf \ # it names (two or more, each with a gVCF; docs/multi-sample.md) ``` -**Keep `--outdir` private.** The results hold genome data. Create the folder before the run with `mkdir -m 700 results`, or put it in a folder only you can open. The `og` tools write their files owner-only (mode 0600), but Nextflow creates the folders, and the other tools write their files, with the modes your umask gives. +**Keep `--outdir` private.** The results hold genome data. Create the folder before the run with `mkdir -m 700 results`, or put it in a folder only you can open. The `og` tools write their analysis outputs owner-only (mode 0600), but Nextflow creates the folders, and the other tools write their files, with the modes your umask gives. **Family workflow (opt-in).** `--pedigree` (tab separated: sample, sex, father, mother, partner; [`assets/pedigree.example.tsv`](https://github.com/GeiserX/OwnGenome/blob/main/assets/pedigree.example.tsv)) runs `workflows/family.nf` over the samplesheet rows it names. `--pedigree` may be a families file (a leading family column, [`assets/families.example.tsv`](https://github.com/GeiserX/OwnGenome/blob/main/assets/families.example.tsv)). `--family_name` names the family to run, which a file with more than one family needs, and the folder its results go to (`//` instead of `/family/`). The workflow does the same work as step 38: `FAMILY_PREPARE` checks the pedigree and that each member's gVCF names its own sample, `GLNEXUS` and `GLNEXUS_VCF` joint-call the gVCFs (`GLNEXUS` asks for 16 GB and gives GLnexus half of its memory as `--mem-gbytes`), `FAMILY_RELATE` and `FAMILY_CHECK` stop the run when somalier's relatedness of a pair disagrees with the pedigree (a sample swap), `WHATSHAP_PED` phases the children of complete trios by inheritance, and `FAMILY_REPORT` writes each child's allele origin, de novo candidates when both parents are sequenced, and compound heterozygotes among the clinical filter's candidates. With `--ancestry_fine_panel` (the panel folder of step 26b, `setup.sh --ancestry-fine-panel`), four more processes find the DNA each pair shares by descent, once `FAMILY_CHECK` has passed: `FAMILY_IBD_SITES` keeps the joint call's genotypes at the panel's SNVs on each autosome (a chromosome with fewer than 100 is left out), `FAMILY_IBD_PHASE` phases them with Beagle against the panel, `FAMILY_IBD_SEGMENTS` runs hap-ibd (its default 2 cM minimum) with the panel's genetic maps, and `FAMILY_IBD_REPORT` writes `ibd_pairs.tsv`, `ibd_segments.tsv` and `ibd_report.md`. They feed only the report, so a failure of one is logged and the run goes on. Without the panel none of them runs. Each member needs a gVCF: DEEPVARIANT's for a FASTQ or BAM row, the `gvcf` column for a VCF row (the run stops on a VCF row without one). Fewer than two named samples, or no `--somalier_sites`, stops the run at the start. When the pedigree marks two of those samples as each other's partner, `COUPLE_CARRIER` runs the couple carrier screen of step 39 on them ([Couple Carrier Screen](39-couple.md)): it reads each partner's ClinVar hits, so it needs `clinvar` in `--tools` (without it the run logs a warning and skips the screen), uses deletions over an exon only with `--gencode_exons` (the GENCODE exons BED `setup.sh` installs as `reference/gencode.v50.basic.exons.bed`), reads Delly's calls, ExpansionHunter's, and Parascopy's SMN1, interval and CYP21A2 tables when those steps ran, and, when `--annotsv_annotations` holds AnnotSV's benign loss set (`benign_Loss_SV_GRCh38.sorted.bed`, with gnomAD-SV's common deletions merged in by `setup.sh --annotsv-data`), sets aside a Delly deletion that matches a common one there. Without `--pedigree` none of these processes exists and a run is unchanged. What it checks and what the files hold: [Multi-sample comparison](multi-sample.md#family-analysis-step-38-opt-in). From 23dd4d278305a5b9f00470709d16e66c2b6c03d4 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 18 Nov 2026 16:09:19 +0100 Subject: [PATCH 4/6] fix(io): give og's owner-only files to the work folder's owner when og runs as root The docker profile runs containers as root over a work folder the host user owns. Nextflow's fixOwnership never hands the files back there: the biocontainers images ship a busybox chown without --from. A 0600 file og wrote as root was then unreadable to the user, and Nextflow could not publish it. The atomic writer now gives its temporary file to the owner and group of the nearest folder above it that root does not own. --- CHANGELOG.md | 2 +- docs/nextflow.md | 2 +- rust/crates/og-io/src/lib.rs | 82 ++++++++++++++++++++ rust/crates/og-io/tests/atomic_root_owner.rs | 70 +++++++++++++++++ 4 files changed, 154 insertions(+), 2 deletions(-) create mode 100644 rust/crates/og-io/tests/atomic_root_owner.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index ef7c25418..6255249b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- The files `og` writes through its atomic writer (every analysis output) are owner-only (mode 0600) whatever the umask. Before, `og` wrote each output under a temporary name with the umask's mode, so under the common umask 022 an output was readable by every user of the machine, and replacing a 0600 file made it 0644. The step scripts set umask 077, so a run through them was already owner-only; `og` run by hand or by the Nextflow pipeline was not. Now the temporary file is created mode 0600, or with the mode of the file it replaces when that is stricter (a 0400 file stays 0400). It is always a new file: a file or link already at its name is left as it is and the next name is tried, and two outputs written at once to one name in one process get two temporary files. The temporary name gains a number, `..tmp--`. No output name or content changes. [Nextflow Execution](docs/nextflow.md#run-the-pipeline) now says to keep `--outdir` private. +- The files `og` writes through its atomic writer (every analysis output) are owner-only (mode 0600) whatever the umask. Before, `og` wrote each output under a temporary name with the umask's mode, so under the common umask 022 an output was readable by every user of the machine, and replacing a 0600 file made it 0644. The step scripts set umask 077, so a run through them was already owner-only; `og` run by hand or by the Nextflow pipeline was not. Now the temporary file is created mode 0600, or with the mode of the file it replaces when that is stricter (a 0400 file stays 0400). It is always a new file: a file or link already at its name is left as it is and the next name is tried, and two outputs written at once to one name in one process get two temporary files. The temporary name gains a number, `..tmp--`. When `og` runs as root, each of these files goes to the owner and group of the nearest folder above it that root does not own; when every folder up to `/` is root's, nothing changes. The docker profile of the Nextflow pipeline runs its containers as root over a work folder the host user owns; Nextflow's `docker.fixOwnership` does not reach these files, because the biocontainers images ship a `chown` without `--from`. Without the hand-over the host user could not read an output `og` wrote there, and Nextflow could not publish it. No output name or content changes. [Nextflow Execution](docs/nextflow.md#run-the-pipeline) now says to keep `--outdir` private. - A new page, [User journeys](docs/user-journeys.md), records where a new user gets stuck on each surface: the docs, the demo, the shell and the local site. It gives the outcome of each journey in the first test-as-user run (2026-11-17, commit `65095e97`), the fix for each finding, and what the run could not reach. A fix that lands has a journey to walk again. - VCF intake now refuses a record with GT after another FORMAT key, such as `DP:GT:GQ`, before a reader can mistake depth for the genotype. The check exits 2 and names the sample, file, first offending position and FORMAT field. It tells users to reorder the keys and sample values together or get a corrected VCF. Records without GT remain allowed by this ordering check. Valid input and output formats are unchanged. Part of #411 (clin-37). - The `archaic-same-person` job of `public-evidence.yml` puts step 43's Neanderthal totals beside Chen et al.'s released calls with `og archaic compare` and `og archaic genomes`, in Rust. They replace `bin/archaic_compare.py`, which is gone with its test, `tests/test_archaic_compare.py`; those checks are now cargo tests in og-ancestry. The tables are the same, byte for byte, on 20 parity cases and on the inputs of the last green run of the job (NA12878, chromosomes 19 to 22). No step of the pipeline runs them, and no option or output changes. What changes on purpose: a missing file, an unreadable number or an empty spans file stops with an `ERROR:` line and exit 1 instead of a Python traceback; an interval with a negative start or an end before its start stops the command, where the Python counted a negative length; and a `--mine` value without `=` is a usage error (exit 2). diff --git a/docs/nextflow.md b/docs/nextflow.md index ef5ddefad..56e6fa044 100644 --- a/docs/nextflow.md +++ b/docs/nextflow.md @@ -91,7 +91,7 @@ nextflow run main.nf \ # it names (two or more, each with a gVCF; docs/multi-sample.md) ``` -**Keep `--outdir` private.** The results hold genome data. Create the folder before the run with `mkdir -m 700 results`, or put it in a folder only you can open. The `og` tools write their analysis outputs owner-only (mode 0600), but Nextflow creates the folders, and the other tools write their files, with the modes your umask gives. +**Keep `--outdir` private.** The results hold genome data. Create the folder before the run with `mkdir -m 700 results`, or put it in a folder only you can open. The `og` tools write their analysis outputs owner-only (mode 0600; run as root in a container, they give each file to the owner of the work folder it sits in, so you can read it), but Nextflow creates the folders, and the other tools write their files, with the modes your umask gives. **Family workflow (opt-in).** `--pedigree` (tab separated: sample, sex, father, mother, partner; [`assets/pedigree.example.tsv`](https://github.com/GeiserX/OwnGenome/blob/main/assets/pedigree.example.tsv)) runs `workflows/family.nf` over the samplesheet rows it names. `--pedigree` may be a families file (a leading family column, [`assets/families.example.tsv`](https://github.com/GeiserX/OwnGenome/blob/main/assets/families.example.tsv)). `--family_name` names the family to run, which a file with more than one family needs, and the folder its results go to (`//` instead of `/family/`). The workflow does the same work as step 38: `FAMILY_PREPARE` checks the pedigree and that each member's gVCF names its own sample, `GLNEXUS` and `GLNEXUS_VCF` joint-call the gVCFs (`GLNEXUS` asks for 16 GB and gives GLnexus half of its memory as `--mem-gbytes`), `FAMILY_RELATE` and `FAMILY_CHECK` stop the run when somalier's relatedness of a pair disagrees with the pedigree (a sample swap), `WHATSHAP_PED` phases the children of complete trios by inheritance, and `FAMILY_REPORT` writes each child's allele origin, de novo candidates when both parents are sequenced, and compound heterozygotes among the clinical filter's candidates. With `--ancestry_fine_panel` (the panel folder of step 26b, `setup.sh --ancestry-fine-panel`), four more processes find the DNA each pair shares by descent, once `FAMILY_CHECK` has passed: `FAMILY_IBD_SITES` keeps the joint call's genotypes at the panel's SNVs on each autosome (a chromosome with fewer than 100 is left out), `FAMILY_IBD_PHASE` phases them with Beagle against the panel, `FAMILY_IBD_SEGMENTS` runs hap-ibd (its default 2 cM minimum) with the panel's genetic maps, and `FAMILY_IBD_REPORT` writes `ibd_pairs.tsv`, `ibd_segments.tsv` and `ibd_report.md`. They feed only the report, so a failure of one is logged and the run goes on. Without the panel none of them runs. Each member needs a gVCF: DEEPVARIANT's for a FASTQ or BAM row, the `gvcf` column for a VCF row (the run stops on a VCF row without one). Fewer than two named samples, or no `--somalier_sites`, stops the run at the start. When the pedigree marks two of those samples as each other's partner, `COUPLE_CARRIER` runs the couple carrier screen of step 39 on them ([Couple Carrier Screen](39-couple.md)): it reads each partner's ClinVar hits, so it needs `clinvar` in `--tools` (without it the run logs a warning and skips the screen), uses deletions over an exon only with `--gencode_exons` (the GENCODE exons BED `setup.sh` installs as `reference/gencode.v50.basic.exons.bed`), reads Delly's calls, ExpansionHunter's, and Parascopy's SMN1, interval and CYP21A2 tables when those steps ran, and, when `--annotsv_annotations` holds AnnotSV's benign loss set (`benign_Loss_SV_GRCh38.sorted.bed`, with gnomAD-SV's common deletions merged in by `setup.sh --annotsv-data`), sets aside a Delly deletion that matches a common one there. Without `--pedigree` none of these processes exists and a run is unchanged. What it checks and what the files hold: [Multi-sample comparison](multi-sample.md#family-analysis-step-38-opt-in). diff --git a/rust/crates/og-io/src/lib.rs b/rust/crates/og-io/src/lib.rs index 4901ed3a3..8c8d73b49 100644 --- a/rust/crates/og-io/src/lib.rs +++ b/rust/crates/og-io/src/lib.rs @@ -205,6 +205,12 @@ impl Iterator for Lines { /// name is left alone and the next name is tried. On Unix it is created /// mode 0600, or with the mode of the target it replaces when that is /// stricter, so the umask never makes an output readable by others. +/// +/// When og runs as root, the temporary file is handed to the owner and +/// group of the nearest folder above it that root does not own. That is a +/// container run as root over a work folder bind-mounted from the host, as +/// the pipeline's docker profile does: without it the host user could not +/// read an owner-only output og wrote there. pub struct AtomicFile { tmp: PathBuf, target: PathBuf, @@ -235,6 +241,7 @@ impl AtomicFile { let tmp = target.with_file_name(tmp_name); match new_file(&tmp, mode) { Ok(f) => { + give_to_folder_owner(&f, &tmp); return Ok(AtomicFile { tmp, target, @@ -282,6 +289,54 @@ fn temp_mode(_target: &Path) -> u32 { 0 } +/// Hands a file og created as root to the owner and group of the nearest +/// folder above it that root does not own: the work folder a container run +/// as root got from the host, even through a subfolder made as root inside +/// it. Nothing changes when og is not root or every folder up to `/` is +/// root's. Best effort: where root may not change owners, the file stays +/// root's. +#[cfg(unix)] +fn give_to_folder_owner(file: &File, path: &Path) { + use std::os::unix::fs::MetadataExt; + let Ok(f) = file.metadata() else { + return; + }; + if f.uid() != 0 { + return; + } + let dir = match path.parent() { + Some(d) if !d.as_os_str().is_empty() => d, + _ => Path::new("."), + }; + let Ok(dir) = fs::canonicalize(dir) else { + return; + }; + let owners = dir + .ancestors() + .map_while(|d| fs::metadata(d).ok()) + .map(|m| (m.uid(), m.gid())); + if let Some((uid, gid)) = hand_over_to(f.uid(), owners) { + let _ = std::os::unix::fs::fchown(file, Some(uid), Some(gid)); + } +} + +#[cfg(not(unix))] +fn give_to_folder_owner(_file: &File, _path: &Path) {} + +/// The owner and group a new file goes to, given the owners of its folders +/// from the nearest up: the first folder not owned by root, when the file +/// is root's (og runs as root); otherwise none. +#[cfg(unix)] +fn hand_over_to( + file_uid: u32, + mut folders: impl Iterator, +) -> Option<(u32, u32)> { + if file_uid != 0 { + return None; + } + folders.find(|&(uid, _)| uid != 0) +} + /// Creates `path`, failing with `AlreadyExists` when anything is there, /// a link included (`O_CREAT | O_EXCL`). fn new_file(path: &Path, mode: u32) -> io::Result { @@ -418,6 +473,33 @@ mod tests { } } +#[cfg(all(test, unix))] +mod owner_tests { + use super::hand_over_to; + + #[test] + fn root_in_a_user_folder_hands_the_file_over() { + assert_eq!( + hand_over_to(0, [(1001, 118), (0, 0)].into_iter()), + Some((1001, 118)) + ); + // Through a subfolder made as root inside the user's folder. + assert_eq!( + hand_over_to(0, [(0, 0), (1001, 118), (0, 0)].into_iter()), + Some((1001, 118)) + ); + } + + #[test] + fn otherwise_the_owner_stays() { + // og as root where every folder is root's, and og as a user. + assert_eq!(hand_over_to(0, [(0, 0), (0, 0)].into_iter()), None); + assert_eq!(hand_over_to(0, std::iter::empty()), None); + assert_eq!(hand_over_to(1001, [(1001, 118)].into_iter()), None); + assert_eq!(hand_over_to(1001, [(0, 0), (4321, 1)].into_iter()), None); + } +} + #[cfg(test)] mod bounded_line_tests { use super::*; diff --git a/rust/crates/og-io/tests/atomic_root_owner.rs b/rust/crates/og-io/tests/atomic_root_owner.rs new file mode 100644 index 000000000..7853bb477 --- /dev/null +++ b/rust/crates/og-io/tests/atomic_root_owner.rs @@ -0,0 +1,70 @@ +//! Run as root, [`og_io::AtomicFile`] hands what it writes in another user's +//! folder, or in a subfolder root made inside it, to that user, as in a +//! container run as root over a work folder bind-mounted from the host. The output stays owner-only, and the user who +//! owns the folder can read it. +//! +//! The test needs root, so it runs in a container (`docker run --user root`). +//! Run as any other user it prints that it was skipped, unless +//! `OG_IO_ROOT_TEST=1` is set: then it fails, so a run meant to check this +//! cannot pass without checking it. + +use og_io::AtomicFile; +use std::fs; +use std::io::Write; +use std::os::unix::fs::{MetadataExt, PermissionsExt, chown}; +use std::path::Path; + +fn write(target: &Path) { + let mut f = AtomicFile::create(target).unwrap(); + f.write_all(b"new\n").unwrap(); + f.commit().unwrap(); +} + +#[test] +fn root_hands_the_file_to_the_folder_owner() { + let dir = std::env::temp_dir().join(format!("og-io-root-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + let probe = dir.join("probe"); + fs::write(&probe, "").unwrap(); + if fs::metadata(&probe).unwrap().uid() != 0 { + fs::remove_dir_all(&dir).unwrap(); + assert!( + std::env::var_os("OG_IO_ROOT_TEST").is_none(), + "OG_IO_ROOT_TEST is set but the test does not run as root" + ); + eprintln!("skipped: needs root"); + return; + } + + // A folder another user owns: the file goes to that user, still 0600. + let user = dir.join("user"); + fs::create_dir(&user).unwrap(); + chown(&user, Some(4321), Some(4322)).unwrap(); + let out = user.join("out.tsv"); + write(&out); + let m = fs::metadata(&out).unwrap(); + assert_eq!((m.uid(), m.gid()), (4321, 4322), "the folder owner's file"); + assert_eq!(m.permissions().mode() & 0o7777, 0o600); + assert_eq!(fs::read_to_string(&out).unwrap(), "new\n"); + + // A subfolder made as root inside the user's folder: still the user's. + let sub = user.join("sub"); + fs::create_dir(&sub).unwrap(); + let deep = sub.join("deep.tsv"); + write(&deep); + let m = fs::metadata(&deep).unwrap(); + assert_eq!( + (m.uid(), m.gid()), + (4321, 4322), + "a file in a root subfolder" + ); + + // Root's own folder (the temp folder, all root's up to `/` in a + // container): the file stays root's. + let own = dir.join("own.tsv"); + write(&own); + assert_eq!(fs::metadata(&own).unwrap().uid(), 0, "root's file"); + + fs::remove_dir_all(&dir).unwrap(); +} From 8b8b1f7adf26a49da674d31352caae6d308a0d5d Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:49:56 +0100 Subject: [PATCH 5/6] fix(io): hand og's root-written files only to the owner NXF_OWNER names Guessing the owner from the folders above a file handed root's owner-only outputs to whoever owned any ancestor folder, such as a shared-service owner of a top-level mount. Nextflow already names the user who started the run in NXF_OWNER in each docker task when docker.fixOwnership is on. og run as root now hands its temporary file to that uid:gid and to no one else; without NXF_OWNER the file stays root's. A failed hand-over prints one warning. The root integration test now runs in CI under sudo, and the same step plants a writer that ignores NXF_OWNER and checks that the test fails. --- .github/workflows/guard.yml | 28 +++++ CHANGELOG.md | 2 +- docs/nextflow.md | 2 +- rust/crates/og-io/src/lib.rs | 115 ++++++++++--------- rust/crates/og-io/tests/atomic_root_owner.rs | 96 ++++++++++------ 5 files changed, 151 insertions(+), 92 deletions(-) diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index d203d4668..fc2a49d49 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -480,6 +480,34 @@ jobs: - name: cargo test run: cargo test --workspace --locked --no-fail-fast + # og-io's root test (tests/atomic_root_owner.rs) skips unless it runs as + # root: run it with sudo, then plant a writer that never reads NXF_OWNER + # and check that the same test fails on it. + - name: og-io as root hands its files to NXF_OWNER, and the test fails without the hand-over + run: | + root_test() { + cargo test --locked -p og-io --test atomic_root_owner --no-run --message-format=json \ + | jq -r 'select(.executable != null) | .executable' + } + bin=$(root_test) + test -x "$bin" + sudo env OG_IO_ROOT_TEST=1 "$bin" --test-threads=1 + src=crates/og-io/src/lib.rs + cp "$src" "${RUNNER_TEMP}/og-io-lib.rs" + trap 'cp "${RUNNER_TEMP}/og-io-lib.rs" "$src"' EXIT + sed -i 's/std::env::var("NXF_OWNER")/std::env::var("NXF_OWNER_PLANTED")/' "$src" + grep -q 'NXF_OWNER_PLANTED' "$src" || { echo "ERROR: the break was not planted"; exit 1; } + bin=$(root_test) + log="${RUNNER_TEMP}/og-io-root-red.log" + if sudo env OG_IO_ROOT_TEST=1 "$bin" --test-threads=1 > "$log" 2>&1; then + cat "$log" + echo "ERROR: the root test passed with a writer that ignores NXF_OWNER" + exit 1 + fi + cat "$log" + grep -F 'the file NXF_OWNER names an owner for' "$log" + echo "Control: a writer that ignores NXF_OWNER failed the root test." + # The planned-verb test (og/tests/port_scaffold.rs) must fail when a # later merge drops a verb's arm. - name: The planned-verb test fails when a verb is dropped diff --git a/CHANGELOG.md b/CHANGELOG.md index 6255249b7..14d42e07f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- The files `og` writes through its atomic writer (every analysis output) are owner-only (mode 0600) whatever the umask. Before, `og` wrote each output under a temporary name with the umask's mode, so under the common umask 022 an output was readable by every user of the machine, and replacing a 0600 file made it 0644. The step scripts set umask 077, so a run through them was already owner-only; `og` run by hand or by the Nextflow pipeline was not. Now the temporary file is created mode 0600, or with the mode of the file it replaces when that is stricter (a 0400 file stays 0400). It is always a new file: a file or link already at its name is left as it is and the next name is tried, and two outputs written at once to one name in one process get two temporary files. The temporary name gains a number, `..tmp--`. When `og` runs as root, each of these files goes to the owner and group of the nearest folder above it that root does not own; when every folder up to `/` is root's, nothing changes. The docker profile of the Nextflow pipeline runs its containers as root over a work folder the host user owns; Nextflow's `docker.fixOwnership` does not reach these files, because the biocontainers images ship a `chown` without `--from`. Without the hand-over the host user could not read an output `og` wrote there, and Nextflow could not publish it. No output name or content changes. [Nextflow Execution](docs/nextflow.md#run-the-pipeline) now says to keep `--outdir` private. +- The files `og` writes through its atomic writer (every analysis output) are owner-only (mode 0600) whatever the umask. Before, `og` wrote each output under a temporary name with the umask's mode, so under the common umask 022 an output was readable by every user of the machine, and replacing a 0600 file made it 0644. The step scripts set umask 077, so a run through them was already owner-only; `og` run by hand or by the Nextflow pipeline was not. Now the temporary file is created mode 0600, or with the mode of the file it replaces when that is stricter (a 0400 file stays 0400). It is always a new file: a file or link already at its name is left as it is and the next name is tried, and two outputs written at once to one name in one process get two temporary files. The temporary name gains a number, `..tmp--`. When `og` runs as root and the environment variable `NXF_OWNER` holds `uid:gid` (two numbers), each of these files goes to that owner and group; without it, a file `og` writes as root stays root's, as before. The docker profile of the Nextflow pipeline runs its containers as root, and with `docker.fixOwnership` on, Nextflow sets `NXF_OWNER` in each task to the user who started the run. Nextflow's own hand-back after the task, a `chown --from root`, fails silently in images whose `chown` has no `--from` option (the busybox in biocontainers images) and leaves these files root's. Without the hand-over that user could not read an output `og` wrote there, and Nextflow could not publish it. When the hand-over fails, `og` prints one warning and the file stays root's. No output name or content changes. [Nextflow Execution](docs/nextflow.md#run-the-pipeline) now says to keep `--outdir` private. - A new page, [User journeys](docs/user-journeys.md), records where a new user gets stuck on each surface: the docs, the demo, the shell and the local site. It gives the outcome of each journey in the first test-as-user run (2026-11-17, commit `65095e97`), the fix for each finding, and what the run could not reach. A fix that lands has a journey to walk again. - VCF intake now refuses a record with GT after another FORMAT key, such as `DP:GT:GQ`, before a reader can mistake depth for the genotype. The check exits 2 and names the sample, file, first offending position and FORMAT field. It tells users to reorder the keys and sample values together or get a corrected VCF. Records without GT remain allowed by this ordering check. Valid input and output formats are unchanged. Part of #411 (clin-37). - The `archaic-same-person` job of `public-evidence.yml` puts step 43's Neanderthal totals beside Chen et al.'s released calls with `og archaic compare` and `og archaic genomes`, in Rust. They replace `bin/archaic_compare.py`, which is gone with its test, `tests/test_archaic_compare.py`; those checks are now cargo tests in og-ancestry. The tables are the same, byte for byte, on 20 parity cases and on the inputs of the last green run of the job (NA12878, chromosomes 19 to 22). No step of the pipeline runs them, and no option or output changes. What changes on purpose: a missing file, an unreadable number or an empty spans file stops with an `ERROR:` line and exit 1 instead of a Python traceback; an interval with a negative start or an end before its start stops the command, where the Python counted a negative length; and a `--mine` value without `=` is a usage error (exit 2). diff --git a/docs/nextflow.md b/docs/nextflow.md index 56e6fa044..544cc1d86 100644 --- a/docs/nextflow.md +++ b/docs/nextflow.md @@ -91,7 +91,7 @@ nextflow run main.nf \ # it names (two or more, each with a gVCF; docs/multi-sample.md) ``` -**Keep `--outdir` private.** The results hold genome data. Create the folder before the run with `mkdir -m 700 results`, or put it in a folder only you can open. The `og` tools write their analysis outputs owner-only (mode 0600; run as root in a container, they give each file to the owner of the work folder it sits in, so you can read it), but Nextflow creates the folders, and the other tools write their files, with the modes your umask gives. +**Keep `--outdir` private.** The results hold genome data. Create the folder before the run with `mkdir -m 700 results`, or put it in a folder only you can open. The `og` tools write their analysis outputs owner-only (mode 0600; run as root in a container of the docker profile, they give each file to the user who started the run, named in Nextflow's `NXF_OWNER`, so you can read it), but Nextflow creates the folders, and the other tools write their files, with the modes your umask gives. **Family workflow (opt-in).** `--pedigree` (tab separated: sample, sex, father, mother, partner; [`assets/pedigree.example.tsv`](https://github.com/GeiserX/OwnGenome/blob/main/assets/pedigree.example.tsv)) runs `workflows/family.nf` over the samplesheet rows it names. `--pedigree` may be a families file (a leading family column, [`assets/families.example.tsv`](https://github.com/GeiserX/OwnGenome/blob/main/assets/families.example.tsv)). `--family_name` names the family to run, which a file with more than one family needs, and the folder its results go to (`//` instead of `/family/`). The workflow does the same work as step 38: `FAMILY_PREPARE` checks the pedigree and that each member's gVCF names its own sample, `GLNEXUS` and `GLNEXUS_VCF` joint-call the gVCFs (`GLNEXUS` asks for 16 GB and gives GLnexus half of its memory as `--mem-gbytes`), `FAMILY_RELATE` and `FAMILY_CHECK` stop the run when somalier's relatedness of a pair disagrees with the pedigree (a sample swap), `WHATSHAP_PED` phases the children of complete trios by inheritance, and `FAMILY_REPORT` writes each child's allele origin, de novo candidates when both parents are sequenced, and compound heterozygotes among the clinical filter's candidates. With `--ancestry_fine_panel` (the panel folder of step 26b, `setup.sh --ancestry-fine-panel`), four more processes find the DNA each pair shares by descent, once `FAMILY_CHECK` has passed: `FAMILY_IBD_SITES` keeps the joint call's genotypes at the panel's SNVs on each autosome (a chromosome with fewer than 100 is left out), `FAMILY_IBD_PHASE` phases them with Beagle against the panel, `FAMILY_IBD_SEGMENTS` runs hap-ibd (its default 2 cM minimum) with the panel's genetic maps, and `FAMILY_IBD_REPORT` writes `ibd_pairs.tsv`, `ibd_segments.tsv` and `ibd_report.md`. They feed only the report, so a failure of one is logged and the run goes on. Without the panel none of them runs. Each member needs a gVCF: DEEPVARIANT's for a FASTQ or BAM row, the `gvcf` column for a VCF row (the run stops on a VCF row without one). Fewer than two named samples, or no `--somalier_sites`, stops the run at the start. When the pedigree marks two of those samples as each other's partner, `COUPLE_CARRIER` runs the couple carrier screen of step 39 on them ([Couple Carrier Screen](39-couple.md)): it reads each partner's ClinVar hits, so it needs `clinvar` in `--tools` (without it the run logs a warning and skips the screen), uses deletions over an exon only with `--gencode_exons` (the GENCODE exons BED `setup.sh` installs as `reference/gencode.v50.basic.exons.bed`), reads Delly's calls, ExpansionHunter's, and Parascopy's SMN1, interval and CYP21A2 tables when those steps ran, and, when `--annotsv_annotations` holds AnnotSV's benign loss set (`benign_Loss_SV_GRCh38.sorted.bed`, with gnomAD-SV's common deletions merged in by `setup.sh --annotsv-data`), sets aside a Delly deletion that matches a common one there. Without `--pedigree` none of these processes exists and a run is unchanged. What it checks and what the files hold: [Multi-sample comparison](multi-sample.md#family-analysis-step-38-opt-in). diff --git a/rust/crates/og-io/src/lib.rs b/rust/crates/og-io/src/lib.rs index 8c8d73b49..82f62a19d 100644 --- a/rust/crates/og-io/src/lib.rs +++ b/rust/crates/og-io/src/lib.rs @@ -206,11 +206,12 @@ impl Iterator for Lines { /// mode 0600, or with the mode of the target it replaces when that is /// stricter, so the umask never makes an output readable by others. /// -/// When og runs as root, the temporary file is handed to the owner and -/// group of the nearest folder above it that root does not own. That is a -/// container run as root over a work folder bind-mounted from the host, as -/// the pipeline's docker profile does: without it the host user could not -/// read an owner-only output og wrote there. +/// When og runs as root and `NXF_OWNER` names an owner as `uid:gid`, the +/// temporary file is handed to that owner. Nextflow sets `NXF_OWNER` to the +/// user who started the run in each docker task when `docker.fixOwnership` +/// is on, as the pipeline's docker profile does: without the hand-over that +/// user could not read an owner-only output og wrote as root. Without +/// `NXF_OWNER`, a file og writes as root stays root's. pub struct AtomicFile { tmp: PathBuf, target: PathBuf, @@ -241,7 +242,7 @@ impl AtomicFile { let tmp = target.with_file_name(tmp_name); match new_file(&tmp, mode) { Ok(f) => { - give_to_folder_owner(&f, &tmp); + give_to_nxf_owner(&f); return Ok(AtomicFile { tmp, target, @@ -289,52 +290,46 @@ fn temp_mode(_target: &Path) -> u32 { 0 } -/// Hands a file og created as root to the owner and group of the nearest -/// folder above it that root does not own: the work folder a container run -/// as root got from the host, even through a subfolder made as root inside -/// it. Nothing changes when og is not root or every folder up to `/` is -/// root's. Best effort: where root may not change owners, the file stays -/// root's. +/// Hands a file og created as root to the owner `NXF_OWNER` names +/// (`uid:gid`, both numbers), the user who started the Nextflow run. Nothing +/// changes when og is not root or `NXF_OWNER` is unset or not `uid:gid`. +/// Best effort: when the change fails, the file stays root's and og prints +/// one warning. #[cfg(unix)] -fn give_to_folder_owner(file: &File, path: &Path) { +fn give_to_nxf_owner(file: &File) { use std::os::unix::fs::MetadataExt; - let Ok(f) = file.metadata() else { - return; - }; - if f.uid() != 0 { + if !file.metadata().is_ok_and(|m| m.uid() == 0) { return; } - let dir = match path.parent() { - Some(d) if !d.as_os_str().is_empty() => d, - _ => Path::new("."), - }; - let Ok(dir) = fs::canonicalize(dir) else { + let Some((uid, gid)) = std::env::var("NXF_OWNER") + .ok() + .and_then(|v| parse_owner(&v)) + else { return; }; - let owners = dir - .ancestors() - .map_while(|d| fs::metadata(d).ok()) - .map(|m| (m.uid(), m.gid())); - if let Some((uid, gid)) = hand_over_to(f.uid(), owners) { - let _ = std::os::unix::fs::fchown(file, Some(uid), Some(gid)); + if let Err(e) = std::os::unix::fs::fchown(file, Some(uid), Some(gid)) { + static WARNED: std::sync::Once = std::sync::Once::new(); + WARNED.call_once(|| { + eprintln!("WARNING: could not give og's output to NXF_OWNER {uid}:{gid}: {e}"); + }); } } #[cfg(not(unix))] -fn give_to_folder_owner(_file: &File, _path: &Path) {} +fn give_to_nxf_owner(_file: &File) {} -/// The owner and group a new file goes to, given the owners of its folders -/// from the nearest up: the first folder not owned by root, when the file -/// is root's (og runs as root); otherwise none. +/// `uid:gid` as two numbers, the form Nextflow writes `NXF_OWNER` in +/// (`$(id -u):$(id -g)`); anything else is none. #[cfg(unix)] -fn hand_over_to( - file_uid: u32, - mut folders: impl Iterator, -) -> Option<(u32, u32)> { - if file_uid != 0 { - return None; - } - folders.find(|&(uid, _)| uid != 0) +fn parse_owner(value: &str) -> Option<(u32, u32)> { + let (uid, gid) = value.split_once(':')?; + let number = |s: &str| { + if s.is_empty() || !s.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + s.parse::().ok() + }; + Some((number(uid)?, number(gid)?)) } /// Creates `path`, failing with `AlreadyExists` when anything is there, @@ -475,28 +470,34 @@ mod tests { #[cfg(all(test, unix))] mod owner_tests { - use super::hand_over_to; + use super::parse_owner; #[test] - fn root_in_a_user_folder_hands_the_file_over() { - assert_eq!( - hand_over_to(0, [(1001, 118), (0, 0)].into_iter()), - Some((1001, 118)) - ); - // Through a subfolder made as root inside the user's folder. - assert_eq!( - hand_over_to(0, [(0, 0), (1001, 118), (0, 0)].into_iter()), - Some((1001, 118)) - ); + fn nxf_owner_is_two_numbers() { + assert_eq!(parse_owner("1001:118"), Some((1001, 118))); + assert_eq!(parse_owner("0:0"), Some((0, 0))); + assert_eq!(parse_owner("4294967295:1"), Some((u32::MAX, 1))); } #[test] - fn otherwise_the_owner_stays() { - // og as root where every folder is root's, and og as a user. - assert_eq!(hand_over_to(0, [(0, 0), (0, 0)].into_iter()), None); - assert_eq!(hand_over_to(0, std::iter::empty()), None); - assert_eq!(hand_over_to(1001, [(1001, 118)].into_iter()), None); - assert_eq!(hand_over_to(1001, [(0, 0), (4321, 1)].into_iter()), None); + fn anything_else_names_no_owner() { + for value in [ + "", + ":", + "1001", + "1001:", + ":118", + "1001:118:1", + "alice:users", + " 1001:118", + "1001:118 ", + "+1001:118", + "-1:118", + "4294967296:1", + "1001;118", + ] { + assert_eq!(parse_owner(value), None, "{value:?}"); + } } } diff --git a/rust/crates/og-io/tests/atomic_root_owner.rs b/rust/crates/og-io/tests/atomic_root_owner.rs index 7853bb477..3b20eae4b 100644 --- a/rust/crates/og-io/tests/atomic_root_owner.rs +++ b/rust/crates/og-io/tests/atomic_root_owner.rs @@ -1,27 +1,62 @@ -//! Run as root, [`og_io::AtomicFile`] hands what it writes in another user's -//! folder, or in a subfolder root made inside it, to that user, as in a -//! container run as root over a work folder bind-mounted from the host. The output stays owner-only, and the user who -//! owns the folder can read it. +//! Run as root, [`og_io::AtomicFile`] hands what it writes to the owner +//! `NXF_OWNER` names, as Nextflow sets it in a docker task run as root with +//! `docker.fixOwnership` on. The output stays owner-only, and that owner can +//! read it. Without `NXF_OWNER`, or with a value that is not `uid:gid`, the +//! file stays root's, even in a folder another user owns. //! -//! The test needs root, so it runs in a container (`docker run --user root`). -//! Run as any other user it prints that it was skipped, unless -//! `OG_IO_ROOT_TEST=1` is set: then it fails, so a run meant to check this -//! cannot pass without checking it. +//! The test needs root: CI runs it with `sudo`. Run as any other user it +//! prints that it was skipped, unless `OG_IO_ROOT_TEST=1` is set: then it +//! fails, so a run meant to check this cannot pass without checking it. +//! +//! Each write runs in a child process (this test binary again, running only +//! `child_writes`), because the environment of a running test cannot be +//! changed without `unsafe`. use og_io::AtomicFile; use std::fs; use std::io::Write; use std::os::unix::fs::{MetadataExt, PermissionsExt, chown}; use std::path::Path; +use std::process::Command; + +const CHILD: &str = "OG_IO_ROOT_CHILD"; -fn write(target: &Path) { - let mut f = AtomicFile::create(target).unwrap(); +/// Writes the file `OG_IO_ROOT_CHILD` names; does nothing without it. +#[test] +fn child_writes() { + let Some(target) = std::env::var_os(CHILD) else { + return; + }; + let mut f = AtomicFile::create(&target).unwrap(); f.write_all(b"new\n").unwrap(); f.commit().unwrap(); } +/// Writes `target` in a child process with `NXF_OWNER` set to `owner`, or +/// unset, and returns the new file's owner and group. +fn write_as_child(target: &Path, owner: Option<&str>) -> (u32, u32) { + let mut child = Command::new(std::env::current_exe().unwrap()); + child + .args(["--exact", "child_writes", "--test-threads=1"]) + .env(CHILD, target); + match owner { + Some(o) => child.env("NXF_OWNER", o), + None => child.env_remove("NXF_OWNER"), + }; + let out = child.output().unwrap(); + assert!( + out.status.success(), + "the child failed: {}", + String::from_utf8_lossy(&out.stdout) + ); + let m = fs::metadata(target).unwrap(); + assert_eq!(m.permissions().mode() & 0o7777, 0o600, "{target:?}"); + assert_eq!(fs::read_to_string(target).unwrap(), "new\n"); + (m.uid(), m.gid()) +} + #[test] -fn root_hands_the_file_to_the_folder_owner() { +fn root_hands_the_file_to_nxf_owner() { let dir = std::env::temp_dir().join(format!("og-io-root-{}", std::process::id())); let _ = fs::remove_dir_all(&dir); fs::create_dir_all(&dir).unwrap(); @@ -37,34 +72,29 @@ fn root_hands_the_file_to_the_folder_owner() { return; } - // A folder another user owns: the file goes to that user, still 0600. + // NXF_OWNER names the owner: the file is that owner's, still 0600. + assert_eq!( + write_as_child(&dir.join("named.tsv"), Some("4321:4322")), + (4321, 4322), + "the file NXF_OWNER names an owner for" + ); + + // No NXF_OWNER: root's, even in a folder another user owns. let user = dir.join("user"); fs::create_dir(&user).unwrap(); chown(&user, Some(4321), Some(4322)).unwrap(); - let out = user.join("out.tsv"); - write(&out); - let m = fs::metadata(&out).unwrap(); - assert_eq!((m.uid(), m.gid()), (4321, 4322), "the folder owner's file"); - assert_eq!(m.permissions().mode() & 0o7777, 0o600); - assert_eq!(fs::read_to_string(&out).unwrap(), "new\n"); - - // A subfolder made as root inside the user's folder: still the user's. - let sub = user.join("sub"); - fs::create_dir(&sub).unwrap(); - let deep = sub.join("deep.tsv"); - write(&deep); - let m = fs::metadata(&deep).unwrap(); assert_eq!( - (m.uid(), m.gid()), - (4321, 4322), - "a file in a root subfolder" + write_as_child(&user.join("unnamed.tsv"), None), + (0, 0), + "a file in another user's folder without NXF_OWNER" ); - // Root's own folder (the temp folder, all root's up to `/` in a - // container): the file stays root's. - let own = dir.join("own.tsv"); - write(&own); - assert_eq!(fs::metadata(&own).unwrap().uid(), 0, "root's file"); + // An NXF_OWNER that is not uid:gid names no one. + assert_eq!( + write_as_child(&user.join("bad.tsv"), Some("alice:users")), + (0, 0), + "a file with an NXF_OWNER that is not uid:gid" + ); fs::remove_dir_all(&dir).unwrap(); } From 7054cbbdf3ea01e064baf5e80c05d0135ebb9cec Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 18 Nov 2026 18:41:03 +0100 Subject: [PATCH 6/6] ci(guard): keep the root test's failure log as the runner's file --- .github/workflows/guard.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index fc2a49d49..1c0aa0c0c 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -499,6 +499,8 @@ jobs: grep -q 'NXF_OWNER_PLANTED' "$src" || { echo "ERROR: the break was not planted"; exit 1; } bin=$(root_test) log="${RUNNER_TEMP}/og-io-root-red.log" + # The log is meant to be the runner's file, not root's. + # shellcheck disable=SC2024 if sudo env OG_IO_ROOT_TEST=1 "$bin" --test-threads=1 > "$log" 2>&1; then cat "$log" echo "ERROR: the root test passed with a writer that ignores NXF_OWNER"