diff --git a/CHANGELOG.md b/CHANGELOG.md index 4236a45d2..9b94c67a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- `og summary collect` refuses a sample folder it cannot list. A mistyped `--sample-dir`, a file, or a folder without read permission read as a folder where no step ran, so the collect wrote a summary.json with every section missing and exited 0, over the good summary when `--out` named it. It now exits 1 with `ERROR: : not a readable folder: ` and writes nothing. An empty folder that exists is still collected, and a step folder inside it that cannot be read still counts as empty. No output file or format changes. - A new page, [User journeys](docs/user-journeys.md), records where a new user gets stuck on each surface: the docs, the demo, the shell and the local site. It gives the outcome of each journey in the first test-as-user run (2026-11-17, commit `65095e97`), the fix for each finding, and what the run could not reach. A fix that lands has a journey to walk again. - VCF intake now refuses a record with GT after another FORMAT key, such as `DP:GT:GQ`, before a reader can mistake depth for the genotype. The check exits 2 and names the sample, file, first offending position and FORMAT field. It tells users to reorder the keys and sample values together or get a corrected VCF. Records without GT remain allowed by this ordering check. Valid input and output formats are unchanged. Part of #411 (clin-37). - The `archaic-same-person` job of `public-evidence.yml` puts step 43's Neanderthal totals beside Chen et al.'s released calls with `og archaic compare` and `og archaic genomes`, in Rust. They replace `bin/archaic_compare.py`, which is gone with its test, `tests/test_archaic_compare.py`; those checks are now cargo tests in og-ancestry. The tables are the same, byte for byte, on 20 parity cases and on the inputs of the last green run of the job (NA12878, chromosomes 19 to 22). No step of the pipeline runs them, and no option or output changes. What changes on purpose: a missing file, an unreadable number or an empty spans file stops with an `ERROR:` line and exit 1 instead of a Python traceback; an interval with a negative start or an end before its start stops the command, where the Python counted a negative length; and a `--mine` value without `=` is a usage error (exit 2). diff --git a/rust/crates/og-summary/src/collect.rs b/rust/crates/og-summary/src/collect.rs index 2c87bff83..23ba181f5 100644 --- a/rust/crates/og-summary/src/collect.rs +++ b/rust/crates/og-summary/src/collect.rs @@ -92,8 +92,8 @@ pub enum Group { } /// `og summary collect`: exit 0 when summary.json is written (whatever -/// its sections say), 1 when it cannot be written, 2 for a command line it -/// cannot read. +/// its sections say), 1 when the sample folder cannot be read or the +/// summary cannot be written, 2 for a command line it cannot read. pub fn cli(argv: &[String], out: &mut impl Write, err: &mut impl Write) -> u8 { let a = match args::parse(argv, OPTS) { Ok(a) => a, @@ -130,6 +130,15 @@ pub fn cli(argv: &[String], out: &mut impl Write, err: &mut impl Write) -> u8 { return 2; } }; + // A mistyped or unreadable sample folder would read as one where no + // step ran, and its summary would replace a good one with every section + // missing. Only the folders below it count as empty when unreadable. + if let Err(e) = readable_folder(&input.sample_dir) { + return crate::finish( + Err(format!("{}: not a readable folder: {e}", input.sample_dir)), + err, + ); + } // The sections' own data is freed before the text is made, and the // tree before the file is written: a large ClinVar list is in memory in // one form at a time. @@ -160,6 +169,14 @@ pub fn cli(argv: &[String], out: &mut impl Write, err: &mut impl Write) -> u8 { crate::finish(r, err) } +/// Ok when `dir` is a folder whose entries can be listed. +fn readable_folder(dir: &str) -> std::io::Result<()> { + if !std::fs::metadata(dir)?.is_dir() { + return Err(std::io::Error::other("not a folder")); + } + std::fs::read_dir(dir).map(drop) +} + /// The `assets/` folder of the checkout the binary sits in (`bin/.og/og-*` /// and `rust/target/*/og` both do), else the one beside the binary's /// folder, as the Python found it beside `bin/`. diff --git a/rust/crates/og-summary/src/collect/scan.rs b/rust/crates/og-summary/src/collect/scan.rs index b81721f1f..26da7d11d 100644 --- a/rust/crates/og-summary/src/collect/scan.rs +++ b/rust/crates/og-summary/src/collect/scan.rs @@ -36,8 +36,10 @@ pub struct FolderIndex { impl FolderIndex { /// Walks `root`. A folder that cannot be read counts as empty, as it - /// does for `glob`; a missing root gives an empty index. A name that is - /// not UTF-8 is left out: no pattern of a section can name it. + /// does for `glob`; a missing root gives an empty index, so + /// `og summary collect` refuses a root it cannot list before it walks. + /// A name that is not UTF-8 is left out: no pattern of a section can + /// name it. pub fn walk(root: &Path) -> FolderIndex { let mut files = HashSet::new(); let mut stack: Vec<(PathBuf, String, usize)> = vec![(root.to_path_buf(), String::new(), 0)]; diff --git a/rust/crates/og-summary/src/collect/tests.rs b/rust/crates/og-summary/src/collect/tests.rs index 4f622c77f..fbad45cce 100644 --- a/rust/crates/og-summary/src/collect/tests.rs +++ b/rust/crates/og-summary/src/collect/tests.rs @@ -471,7 +471,7 @@ fn the_command_line() { "--sample", "S", "--sample-dir", - "x", + &d.path("S"), "--out", &d.path("no/such/dir/x.json"), "--only", diff --git a/rust/crates/og-summary/tests/rv_summary_collect_missing_dir.rs b/rust/crates/og-summary/tests/rv_summary_collect_missing_dir.rs new file mode 100644 index 000000000..c644b225c --- /dev/null +++ b/rust/crates/og-summary/tests/rv_summary_collect_missing_dir.rs @@ -0,0 +1,139 @@ +//! `og summary collect` refuses a sample folder it cannot list: a missing +//! one, a file, or one without read permission exits 1 and leaves `--out` +//! as it was. An empty folder that exists is still collected, with every +//! section missing. + +use og_summary::collect::cli; +use std::fs; +use std::path::PathBuf; + +const GOOD: &str = "{\"a good summary\": true}\n"; + +struct Dir(PathBuf); + +impl Dir { + fn new(tag: &str) -> Dir { + let p = std::env::temp_dir().join(format!( + "og-summary-missing-dir-{tag}-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&p); + fs::create_dir_all(&p).unwrap(); + Dir(p) + } + + fn path(&self, name: &str) -> String { + self.0.join(name).display().to_string() + } +} + +impl Drop for Dir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +fn assets() -> String { + concat!(env!("CARGO_MANIFEST_DIR"), "/../../../assets").into() +} + +/// Runs the verb; gives the exit code and stderr. +fn run(sample_dir: &str, extra: &[&str]) -> (u8, String) { + let mut argv: Vec = [ + "--sample", + "S", + "--sample-dir", + sample_dir, + "--assets", + &assets(), + ] + .iter() + .map(|s| (*s).to_owned()) + .collect(); + argv.extend(extra.iter().map(|s| (*s).to_owned())); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = cli(&argv, &mut out, &mut err); + (code, String::from_utf8(err).unwrap()) +} + +/// The folder is refused: exit 1, the error names it, and `out` keeps the +/// good summary, byte for byte. +fn assert_refused(sample_dir: &str, out: &str, extra: &[&str]) { + let mut args = vec!["--out", out]; + args.extend_from_slice(extra); + let (code, err) = run(sample_dir, &args); + assert_eq!(code, 1, "stderr: {err}"); + assert!( + err.contains(&format!("ERROR: {sample_dir}: not a readable folder")), + "stderr: {err}" + ); + assert_eq!(fs::read_to_string(out).unwrap(), GOOD); +} + +#[test] +fn a_missing_sample_folder_leaves_the_summary_as_it_was() { + let d = Dir::new("missing"); + let out = d.path("summary.json"); + fs::write(&out, GOOD).unwrap(); + let typo = d.path("SAMPEL"); + assert_refused(&typo, &out, &[]); + // Each group reads the same folder, so each refuses it. + for g in ["a", "b", "c"] { + assert_refused(&typo, &out, &["--only", g]); + } + assert!(!PathBuf::from(&typo).exists()); +} + +#[test] +fn without_out_a_missing_folder_writes_nothing() { + let d = Dir::new("no-out"); + let typo = d.path("SAMPEL"); + let (code, err) = run(&typo, &[]); + assert_eq!(code, 1, "stderr: {err}"); + // It stops on the folder, not on a write into a folder that is not there. + assert!(err.contains("not a readable folder"), "stderr: {err}"); + assert!(!PathBuf::from(&typo).exists()); +} + +#[test] +fn a_file_is_not_a_sample_folder() { + let d = Dir::new("file"); + let out = d.path("summary.json"); + fs::write(&out, GOOD).unwrap(); + let file = d.path("S.vcf.gz"); + fs::write(&file, "x").unwrap(); + assert_refused(&file, &out, &[]); +} + +#[cfg(unix)] +#[test] +fn an_unreadable_sample_folder_leaves_the_summary_as_it_was() { + use std::os::unix::fs::PermissionsExt; + let d = Dir::new("unreadable"); + let out = d.path("summary.json"); + fs::write(&out, GOOD).unwrap(); + let locked = d.path("S"); + fs::create_dir_all(format!("{locked}/qc")).unwrap(); + fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).unwrap(); + // Root lists any folder: there is nothing to refuse. + if fs::read_dir(&locked).is_err() { + assert_refused(&locked, &out, &[]); + } + fs::set_permissions(&locked, fs::Permissions::from_mode(0o755)).unwrap(); +} + +#[test] +fn an_empty_sample_folder_is_still_collected() { + let d = Dir::new("empty"); + let sample = d.path("S"); + fs::create_dir_all(&sample).unwrap(); + let out = d.path("summary.json"); + fs::write(&out, GOOD).unwrap(); + let (code, err) = run(&sample, &["--out", &out, "--only", "a"]); + assert_eq!(code, 0, "stderr: {err}"); + let body = fs::read_to_string(&out).unwrap(); + assert_ne!(body, GOOD); + assert!(body.contains("\"coverage\""), "{body}"); + assert!(body.contains("\"state\": \"missing\""), "{body}"); + assert!(!body.contains("\"state\": \"ok\""), "{body}"); +} diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index fac4eb549..c624a7fd8 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -1209,7 +1209,6 @@ report-render notgz gen:report-corpus.sh --summary notgz.json -o notgz.txt -o no report-render empty gen:report-corpus.sh --summary empty.json -o empty.txt -o empty.html empty.txt,empty.html bytes PYTHON_IMAGE report-render declared-arg gen:report-corpus.sh --summary declared-arg.json -o declared-arg.txt -o declared-arg.html declared-arg.txt,declared-arg.html bytes PYTHON_IMAGE report-render no-assets gen:report-corpus.sh --summary no-assets.json -o no-assets.txt -o no-assets.html no-assets.txt,no-assets.html bytes PYTHON_IMAGE -report-render no-folder gen:report-corpus.sh --summary no-folder.json -o no-folder.txt -o no-folder.html no-folder.txt,no-folder.html bytes PYTHON_IMAGE report-render c-full gen:report-corpus.sh --summary c-full.json -o c-full.txt c-full.txt bytes PYTHON_IMAGE report-render c-float-nan gen:report-corpus.sh --summary c-float-nan.json -o c-float-nan.txt c-float-nan.txt bytes PYTHON_IMAGE report-render c-float-inf gen:report-corpus.sh --summary c-float-inf.json -o c-float-inf.txt c-float-inf.txt bytes PYTHON_IMAGE diff --git a/tests/parity/golden/summary-collect/no-folder/exit b/tests/parity/golden/summary-collect/no-folder/exit new file mode 100644 index 000000000..d00491fd7 --- /dev/null +++ b/tests/parity/golden/summary-collect/no-folder/exit @@ -0,0 +1 @@ +1 diff --git a/tests/parity/golden/summary-collect/no-folder/no-folder.json.absent b/tests/parity/golden/summary-collect/no-folder/no-folder.json.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/summary-collect/no-folder/stderr b/tests/parity/golden/summary-collect/no-folder/stderr new file mode 100644 index 000000000..5dcf7c827 --- /dev/null +++ b/tests/parity/golden/summary-collect/no-folder/stderr @@ -0,0 +1 @@ +ERROR: no-such-folder: not a readable folder: No such file or directory (os error 2) diff --git a/tests/parity/golden/summary-collect/no-folder/stdout b/tests/parity/golden/summary-collect/no-folder/stdout new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_render_report.py b/tests/test_render_report.py index 3035fb7a8..98686bab5 100644 --- a/tests/test_render_report.py +++ b/tests/test_render_report.py @@ -859,6 +859,7 @@ def mosaic_table(rows): and "PMS22" in card_p, card_p) # 12. raw-data export card d = f"{work}/consumer/S" + os.makedirs(d) summ = collect_summary.collect("S", d) txt, html = render_report.text_report(summ), render_report.html_report(summ) check("raw-data export: no card and no text section without the step's table",