diff --git a/crates/rustmail-storage/src/error.rs b/crates/rustmail-storage/src/error.rs index 3193f60..2747481 100644 --- a/crates/rustmail-storage/src/error.rs +++ b/crates/rustmail-storage/src/error.rs @@ -345,6 +345,25 @@ pub enum RestoreRefusal { /// The name that is taken. kept: PathBuf, }, + /// The database is missing beside its backup, but the files beside them do + /// not show a restore that stopped between its two renames. + #[error( + "{} is missing and its backup {} exists, but {reason}, so restore-backup cannot tell \ + that it stopped between its two renames; nothing was changed. If {} holds the mail you \ + want, move it to {} by hand.", + .database.display(), + .backup.display(), + .backup.display(), + .database.display() + )] + NotAnInterruptedRestore { + /// The missing database. + database: PathBuf, + /// The backup, `.schema0.bak`. + backup: PathBuf, + /// What the files beside them show instead. + reason: String, + }, } /// SQLite primary result code for `SQLITE_BUSY`. diff --git a/crates/rustmail-storage/src/migrate.rs b/crates/rustmail-storage/src/migrate.rs index 3cc65ef..fa1efba 100644 --- a/crates/rustmail-storage/src/migrate.rs +++ b/crates/rustmail-storage/src/migrate.rs @@ -416,10 +416,23 @@ fn existing_file(path: &Path) -> SqliteConnectOptions { } async fn inspect_db(path: &Path) -> Result { + inspect_db_with(path, existing_file(path)).await +} + +/// Like [`inspect_db`], with the file opened as immutable, so SQLite neither +/// writes to it nor creates a `-wal` or `-shm` beside it. +async fn inspect_db_immutable(path: &Path) -> Result { + inspect_db_with(path, existing_file(path).immutable(true)).await +} + +async fn inspect_db_with( + path: &Path, + options: SqliteConnectOptions, +) -> Result { if !exists(path).await? { return Ok(DbState::Absent); } - let mut conn = SqliteConnection::connect_with(&existing_file(path)).await?; + let mut conn = SqliteConnection::connect_with(&options).await?; let schema = probe(&mut conn).await; conn.close().await?; Ok(match schema? { diff --git a/crates/rustmail-storage/src/migrate/restore.rs b/crates/rustmail-storage/src/migrate/restore.rs index 88ad1ee..9e5c6ad 100644 --- a/crates/rustmail-storage/src/migrate/restore.rs +++ b/crates/rustmail-storage/src/migrate/restore.rs @@ -7,8 +7,8 @@ use time::OffsetDateTime; use tracing::info; use super::{ - DbState, JOURNAL_SUFFIX, Paths, SHM_SUFFIX, WAL_SUFFIX, acquire_lock, existing_file, exists, - inspect_db, rename, sidecar, sync_parent, + DbState, JOURNAL_SUFFIX, Paths, SHM_SUFFIX, WAL_SUFFIX, acquire_lock, blocking, existing_file, + exists, inspect_db, inspect_db_immutable, io_error, rename, sidecar, sync_parent, }; use crate::error::{RestoreRefusal, StorageError}; use crate::schema::{FileSchema, probe}; @@ -21,6 +21,10 @@ const KEPT_SUFFIX: &str = ".schema1-"; const SIDE_SUFFIXES: [&str; 3] = [WAL_SUFFIX, SHM_SUFFIX, JOURNAL_SUFFIX]; /// What `PRAGMA quick_check` answers for an intact file. const QUICK_CHECK_OK: &str = "ok"; +/// Digits in the date part of a kept file's timestamp, `YYYYMMDD`. +const DATE_DIGITS: usize = 8; +/// Digits in the time part of a kept file's timestamp, `HHMMSS`. +const TIME_DIGITS: usize = 6; /// What [`restore_backup`] did. #[derive(Debug, Clone, PartialEq, Eq)] @@ -46,11 +50,18 @@ pub struct RestoreReport { /// `path` and the directory is synced. Neither file keeps a `-wal`, `-shm` or /// `-journal`, so nothing of one can be applied to the other. /// +/// A restore that stopped between those two renames is finished instead: +/// when `path` is missing, the backup passes the same checks, nothing has +/// `path` or the backup open, and the newest `.schema1-` +/// beside them has no `-wal`, `-shm` or `-journal` and reads, without being +/// written, as a schema-1 database, the backup is moved to `path` and the +/// directory is synced. Every kept file is left as it is. +/// /// # Errors /// /// Returns [`StorageError::MigrationLocked`] if another process holds the /// lock, [`StorageError::RestoreRefused`] for files a restore cannot start -/// from, including a database another process still has open, +/// from or finish, including a database another process still has open, /// [`StorageError::NewerSchema`] or [`StorageError::UnrecognizedSchema`] for a /// database rustmail cannot read, and [`StorageError::Database`] or /// [`StorageError::MigrationIo`] if SQLite or the filesystem fails. Nothing is @@ -58,17 +69,16 @@ pub struct RestoreReport { pub async fn restore_backup(path: &Path) -> Result { let paths = Paths::new(path); let _lock = acquire_lock(&paths).await?; - match inspect_db(&paths.db).await? { - DbState::Current => {} - state => { - return Err( - RestoreRefusal::NotMigrated { - database: paths.db.clone(), - database_state: state.describe(), - } - .into(), - ); - } + let state = inspect_db(&paths.db).await?; + let interrupted = state == DbState::Absent && exists(&paths.backup).await?; + if !interrupted && state != DbState::Current { + return Err( + RestoreRefusal::NotMigrated { + database: paths.db.clone(), + database_state: state.describe(), + } + .into(), + ); } if exists(&paths.migrating).await? { return Err( @@ -79,6 +89,9 @@ pub async fn restore_backup(path: &Path) -> Result .into(), ); } + if interrupted { + return finish_interrupted_restore(paths).await; + } if !exists(&paths.backup).await? { return Err( RestoreRefusal::NoBackup { @@ -99,6 +112,32 @@ pub async fn restore_backup(path: &Path) -> Result return Err(RestoreRefusal::KeptNameTaken { kept: kept_as }.into()); } rename(&paths.db, &kept_as).await?; + put_backup_in_place(paths, kept_as).await +} + +/// Finishes a restore that moved the schema-1 database away but not the +/// backup into its place, after the same checks a restore starts with. +async fn finish_interrupted_restore(paths: Paths) -> Result { + let kept_as = interrupted_kept_file(&paths).await?; + refuse_side_files(&paths.db, &paths.db).await?; + refuse_side_files(&paths.db, &paths.backup).await?; + check_backup(&paths.backup).await?; + refuse_side_files(&paths.db, &paths.backup).await?; + refuse_side_files(&paths.db, &paths.db).await?; + info!( + event = "storage_restore_resume", + database = %paths.db.display(), + kept_as = %kept_as.display(), + "Finishing a restore that stopped between its two renames" + ); + put_backup_in_place(paths, kept_as).await +} + +/// Moves the backup to the database path, now free, and syncs the directory. +async fn put_backup_in_place( + paths: Paths, + kept_as: PathBuf, +) -> Result { rename(&paths.backup, &paths.db).await?; sync_parent(&paths.db).await?; info!( @@ -115,6 +154,79 @@ pub async fn restore_backup(path: &Path) -> Result }) } +/// The schema-1 database an interrupted restore moved away: the newest +/// `.schema1-` beside the missing database. It must have no +/// `-wal`, `-shm` or `-journal` and read as a schema-1 database, and is opened +/// as immutable so the check never writes it. Older ones are kept files of +/// earlier restores. +async fn interrupted_kept_file(paths: &Paths) -> Result { + let refuse = |reason: String| -> StorageError { + RestoreRefusal::NotAnInterruptedRestore { + database: paths.db.clone(), + backup: paths.backup.clone(), + reason, + } + .into() + }; + let Some(kept_as) = kept_files(&paths.db).await?.pop() else { + return Err(refuse(format!( + "no {} file sits beside it", + sidecar(&paths.db, KEPT_SUFFIX).display() + ))); + }; + refuse_side_files(&paths.db, &kept_as).await?; + match inspect_db_immutable(&kept_as).await { + Ok(DbState::Current) => Ok(kept_as), + Ok(state) => Err(refuse(format!( + "{} is {}, not schema 1", + kept_as.display(), + state.describe() + ))), + Err(error) => Err(refuse(format!( + "{} could not be read as a database ({error})", + kept_as.display() + ))), + } +} + +/// The files named `.schema1-` beside `db`, oldest first. +async fn kept_files(db: &Path) -> Result, StorageError> { + let parent = db.parent().unwrap_or(Path::new("")).to_path_buf(); + let listed = if parent.as_os_str().is_empty() { + PathBuf::from(".") + } else { + parent.clone() + }; + let prefix = sidecar(db, KEPT_SUFFIX).to_string_lossy().into_owned(); + blocking(move || { + let list_error = |source| io_error("list the directory", &listed, source); + let mut kept = Vec::new(); + for entry in std::fs::read_dir(&listed).map_err(list_error)? { + let path = parent.join(entry.map_err(list_error)?.file_name()); + if path + .to_string_lossy() + .strip_prefix(prefix.as_str()) + .is_some_and(is_timestamp) + { + kept.push(path); + } + } + kept.sort(); + Ok(kept) + }) + .await +} + +/// Whether `text` has the shape [`timestamp`] writes, `YYYYMMDDTHHMMSSZ`. +fn is_timestamp(text: &str) -> bool { + let all_digits = + |part: &str, len: usize| part.len() == len && part.bytes().all(|byte| byte.is_ascii_digit()); + text + .strip_suffix('Z') + .and_then(|rest| rest.split_once('T')) + .is_some_and(|(date, time)| all_digits(date, DATE_DIGITS) && all_digits(time, TIME_DIGITS)) +} + /// Refuses if `file` has a `-wal`, `-shm` or `-journal` beside it. async fn refuse_side_files(database: &Path, file: &Path) -> Result<(), StorageError> { for suffix in SIDE_SUFFIXES { diff --git a/crates/rustmail-storage/src/migrate/tests/restore.rs b/crates/rustmail-storage/src/migrate/tests/restore.rs index 1695868..322140f 100644 --- a/crates/rustmail-storage/src/migrate/tests/restore.rs +++ b/crates/rustmail-storage/src/migrate/tests/restore.rs @@ -308,3 +308,218 @@ async fn a_restore_is_refused_beside_a_stray_migration_copy() { ); assert_untouched(&db, &db_contents, &backup_bytes).await; } + +const INTERRUPTED_KEPT_SUFFIX: &str = ".schema1-20260101T000000Z"; +const OLDER_KEPT_SUFFIX: &str = ".schema1-20250101T000000Z"; +const SECOND_CYCLE_KEPT_SUFFIX: &str = ".schema1-99991231T235959Z"; + +/// Copies the database's write-ahead log into it and closes it, as a restore +/// does before its first rename. +async fn checkpoint(db: &Path) { + let pool = legacy::open_plain(db).await; + sqlx::query("PRAGMA wal_checkpoint(TRUNCATE)") + .execute(&pool) + .await + .unwrap(); + pool.close().await; + assert_no_sidecars(db); +} + +/// Migrates a legacy database and moves the schema-1 file to a kept name by +/// hand, as a restore that stopped after its first rename leaves it, and +/// returns the kept path. +async fn interrupted_after_first_rename(db: &Path) -> PathBuf { + migrated_with_new_mail(db, 2).await; + checkpoint(db).await; + let kept = sidecar(db, INTERRUPTED_KEPT_SUFFIX); + std::fs::rename(db, &kept).unwrap(); + kept +} + +/// Leaves a valid write-ahead log beside `db` that no connection has open, as +/// a process stopped before its last checkpoint leaves it, with `db` itself as +/// it was, and returns the log's path. +async fn orphan_wal(db: &Path) -> PathBuf { + let db_bytes = std::fs::read(db).unwrap(); + let wal = sidecar(db, WAL_SUFFIX); + let mut conn = SqliteConnection::connect_with(&existing_file(db)) + .await + .unwrap(); + sqlx::query("PRAGMA wal_autocheckpoint = 0") + .execute(&mut conn) + .await + .unwrap(); + sqlx::query("CREATE TABLE only_in_the_wal (id INTEGER)") + .execute(&mut conn) + .await + .unwrap(); + let wal_bytes = std::fs::read(&wal).unwrap(); + conn.close().await.unwrap(); + assert!(!wal_bytes.is_empty()); + std::fs::write(db, db_bytes).unwrap(); + std::fs::write(&wal, wal_bytes).unwrap(); + wal +} + +/// Asserts a refused resume left the database missing and the backup and +/// kept files as they were. +fn assert_resume_untouched(db: &Path, backup_bytes: &[u8], kept: &[(PathBuf, Vec)]) { + assert!(!db.exists()); + assert_eq!( + std::fs::read(sidecar(db, BACKUP_SUFFIX)).unwrap(), + backup_bytes + ); + for (path, bytes) in kept { + assert_eq!(&std::fs::read(path).unwrap(), bytes); + } +} + +#[tokio::test] +async fn a_restore_interrupted_between_its_renames_is_finished() { + let (_dir, db) = scratch(); + let kept = interrupted_after_first_rename(&db).await; + let backup = sidecar(&db, BACKUP_SUFFIX); + let legacy_bytes = std::fs::read(&backup).unwrap(); + let kept_bytes = std::fs::read(&kept).unwrap(); + let before = legacy_snapshot(&backup).await; + + let report = restore_backup(&db).await.unwrap(); + + assert_eq!( + report, + RestoreReport { + database: db.clone(), + restored_from: backup.clone(), + kept_as: kept.clone(), + } + ); + assert_eq!(std::fs::read(&db).unwrap(), legacy_bytes); + assert!(!backup.exists()); + assert_eq!(std::fs::read(&kept).unwrap(), kept_bytes); + assert_eq!(kept_files(&db), vec![kept.clone()]); + assert_eq!(scalar(&db, "PRAGMA user_version").await, 0); + let pool = legacy::open_plain(&db).await; + legacy::initialize_as_v0_7_0(&pool).await.unwrap(); + pool.close().await; + assert_eq!(legacy_snapshot(&db).await, before); + assert_no_sidecars(&db); + assert_no_sidecars(&kept); +} + +#[tokio::test] +async fn an_interrupted_second_restore_is_finished_beside_the_first_kept_file() { + let (_dir, db) = scratch(); + migrated_with_new_mail(&db, 2).await; + let first_kept = restore_backup(&db).await.unwrap().kept_as; + let Preparation::Ready(Some(_)) = migrate(&db).await.unwrap() else { + panic!("the restored database should be migrated again"); + }; + checkpoint(&db).await; + let second_kept = sidecar(&db, SECOND_CYCLE_KEPT_SUFFIX); + std::fs::rename(&db, &second_kept).unwrap(); + let backup = sidecar(&db, BACKUP_SUFFIX); + let legacy_bytes = std::fs::read(&backup).unwrap(); + let first_kept_bytes = std::fs::read(&first_kept).unwrap(); + let second_kept_bytes = std::fs::read(&second_kept).unwrap(); + + let report = restore_backup(&db).await.unwrap(); + + assert_eq!(report.kept_as, second_kept); + assert_eq!(std::fs::read(&db).unwrap(), legacy_bytes); + assert!(!backup.exists()); + assert_eq!(scalar(&db, "PRAGMA user_version").await, 0); + assert_eq!(std::fs::read(&first_kept).unwrap(), first_kept_bytes); + assert_eq!(std::fs::read(&second_kept).unwrap(), second_kept_bytes); + assert_eq!(kept_files(&db), vec![first_kept, second_kept]); +} + +#[tokio::test] +async fn an_interrupted_restore_whose_newest_kept_file_is_not_schema_1_is_refused() { + let (_dir, db) = scratch(); + let kept = interrupted_after_first_rename(&db).await; + let older = sidecar(&db, OLDER_KEPT_SUFFIX); + std::fs::copy(&kept, &older).unwrap(); + std::fs::copy(sidecar(&db, BACKUP_SUFFIX), &kept).unwrap(); + let backup_bytes = std::fs::read(sidecar(&db, BACKUP_SUFFIX)).unwrap(); + let kept_contents = vec![ + (kept.clone(), std::fs::read(&kept).unwrap()), + (older.clone(), std::fs::read(&older).unwrap()), + ]; + + let error = restore_backup(&db).await.unwrap_err(); + + assert!( + matches!( + &error, + StorageError::RestoreRefused(RestoreRefusal::NotAnInterruptedRestore { reason, .. }) + if reason.contains(&kept.display().to_string()) && reason.contains("schema 0") + ), + "got {error:?}" + ); + assert_resume_untouched(&db, &backup_bytes, &kept_contents); +} + +#[tokio::test] +async fn a_missing_database_without_a_kept_file_is_refused() { + let (_dir, db) = scratch(); + migrated_with_new_mail(&db, 2).await; + checkpoint(&db).await; + std::fs::remove_file(&db).unwrap(); + let backup_bytes = std::fs::read(sidecar(&db, BACKUP_SUFFIX)).unwrap(); + + let error = restore_backup(&db).await.unwrap_err(); + + assert!( + matches!( + &error, + StorageError::RestoreRefused(RestoreRefusal::NotAnInterruptedRestore { .. }) + ), + "got {error:?}" + ); + assert_resume_untouched(&db, &backup_bytes, &[]); +} + +#[tokio::test] +async fn an_interrupted_restore_is_refused_while_the_database_has_a_journal_beside_it() { + let (_dir, db) = scratch(); + let kept = interrupted_after_first_rename(&db).await; + let journal = sidecar(&db, JOURNAL_SUFFIX); + std::fs::write(&journal, b"hot").unwrap(); + let backup_bytes = std::fs::read(sidecar(&db, BACKUP_SUFFIX)).unwrap(); + let kept_contents = vec![(kept.clone(), std::fs::read(&kept).unwrap())]; + + let error = restore_backup(&db).await.unwrap_err(); + + assert!( + matches!( + &error, + StorageError::RestoreRefused(RestoreRefusal::FileInUse { file, .. }) if *file == journal + ), + "got {error:?}" + ); + assert_resume_untouched(&db, &backup_bytes, &kept_contents); +} + +#[tokio::test] +async fn an_interrupted_restore_is_refused_while_the_kept_file_has_a_wal_beside_it() { + let (_dir, db) = scratch(); + let kept = interrupted_after_first_rename(&db).await; + let wal = orphan_wal(&kept).await; + let backup_bytes = std::fs::read(sidecar(&db, BACKUP_SUFFIX)).unwrap(); + let kept_contents = vec![ + (kept.clone(), std::fs::read(&kept).unwrap()), + (wal.clone(), std::fs::read(&wal).unwrap()), + ]; + + let error = restore_backup(&db).await.unwrap_err(); + + assert!( + matches!( + &error, + StorageError::RestoreRefused(RestoreRefusal::FileInUse { file, .. }) if *file == wal + ), + "got {error:?}" + ); + assert_resume_untouched(&db, &backup_bytes, &kept_contents); + assert!(!sidecar(&kept, SHM_SUFFIX).exists()); +} diff --git a/docs/getting-started/upgrading.md b/docs/getting-started/upgrading.md index 301f6a9..fd08996 100644 --- a/docs/getting-started/upgrading.md +++ b/docs/getting-started/upgrading.md @@ -91,6 +91,8 @@ Older binaries cannot read the new layout. Restore the backup **with the new bin `restore-backup` takes the same lock as the upgrade, so it refuses to run while another RustMail holds it, and it refuses while another process still has the database open. It renames the current database to `rustmail.db.schema1-` (it never deletes it) and moves `rustmail.db.schema0.bak` back to `rustmail.db`. +If `restore-backup` is stopped between those two moves, `rustmail.db` is missing; run `restore-backup` again and it finishes the restore, after the same checks. It takes the newest `rustmail.db.schema1-` as the file it moved away and leaves every kept file as it is. It refuses instead when the files do not show that state, for example when no `rustmail.db.schema1-` sits beside the backup or the newest one is not a schema-1 database, and its message says which file to move by hand. + ::: warning Mail received after the upgrade is not restored The backup holds the mailbox as it was when you upgraded. Mail received after that, and any changes such as deletions or read flags, exist only in the schema-1 file (`rustmail.db.schema1-`). Starting the new version again on the restored file upgrades it again. :::