fix: let restore-backup finish a restore interrupted between renames

crates/rustmail-storage/src/error.rs

@@ -345,6 +345,25 @@ pub enum RestoreRefusal {
     /// The name that is taken.
     kept: PathBuf,
   },
+  /// The database is missing beside its backup, but the files beside them do
+  /// not show a restore that stopped between its two renames.
+  #[error(
+    "{} is missing and its backup {} exists, but {reason}, so restore-backup cannot tell \
+     that it stopped between its two renames; nothing was changed. If {} holds the mail you \
+     want, move it to {} by hand.",
+    .database.display(),
+    .backup.display(),
+    .backup.display(),
+    .database.display()
+  )]
+  NotAnInterruptedRestore {
+    /// The missing database.
+    database: PathBuf,
+    /// The backup, `<db>.schema0.bak`.
+    backup: PathBuf,
+    /// What the files beside them show instead.
+    reason: String,
+  },
 }
 
 /// SQLite primary result code for `SQLITE_BUSY`.

crates/rustmail-storage/src/migrate.rs

@@ -416,10 +416,23 @@ fn existing_file(path: &Path) -> SqliteConnectOptions {
 }
 
 async fn inspect_db(path: &Path) -> Result<DbState, StorageError> {
+  inspect_db_with(path, existing_file(path)).await
+}
+
+/// Like [`inspect_db`], with the file opened as immutable, so SQLite neither
+/// writes to it nor creates a `-wal` or `-shm` beside it.
+async fn inspect_db_immutable(path: &Path) -> Result<DbState, StorageError> {
+  inspect_db_with(path, existing_file(path).immutable(true)).await
+}
+
+async fn inspect_db_with(
+  path: &Path,
+  options: SqliteConnectOptions,
+) -> Result<DbState, StorageError> {
   if !exists(path).await? {
     return Ok(DbState::Absent);
   }
-  let mut conn = SqliteConnection::connect_with(&existing_file(path)).await?;
+  let mut conn = SqliteConnection::connect_with(&options).await?;
   let schema = probe(&mut conn).await;
   conn.close().await?;
   Ok(match schema? {

crates/rustmail-storage/src/migrate/restore.rs

@@ -7,8 +7,8 @@ use time::OffsetDateTime;
 use tracing::info;
 
 use super::{
-  DbState, JOURNAL_SUFFIX, Paths, SHM_SUFFIX, WAL_SUFFIX, acquire_lock, existing_file, exists,
-  inspect_db, rename, sidecar, sync_parent,
+  DbState, JOURNAL_SUFFIX, Paths, SHM_SUFFIX, WAL_SUFFIX, acquire_lock, blocking, existing_file,
+  exists, inspect_db, inspect_db_immutable, io_error, rename, sidecar, sync_parent,
 };
 use crate::error::{RestoreRefusal, StorageError};
 use crate::schema::{FileSchema, probe};
@@ -21,6 +21,10 @@ const KEPT_SUFFIX: &str = ".schema1-";
 const SIDE_SUFFIXES: [&str; 3] = [WAL_SUFFIX, SHM_SUFFIX, JOURNAL_SUFFIX];
 /// What `PRAGMA quick_check` answers for an intact file.
 const QUICK_CHECK_OK: &str = "ok";
+/// Digits in the date part of a kept file's timestamp, `YYYYMMDD`.
+const DATE_DIGITS: usize = 8;
+/// Digits in the time part of a kept file's timestamp, `HHMMSS`.
+const TIME_DIGITS: usize = 6;
 
 /// What [`restore_backup`] did.
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -46,29 +50,35 @@ pub struct RestoreReport {
 /// `path` and the directory is synced. Neither file keeps a `-wal`, `-shm` or
 /// `-journal`, so nothing of one can be applied to the other.
 ///
+/// A restore that stopped between those two renames is finished instead:
+/// when `path` is missing, the backup passes the same checks, nothing has
+/// `path` or the backup open, and the newest `<path>.schema1-<UTC timestamp>`
+/// beside them has no `-wal`, `-shm` or `-journal` and reads, without being
+/// written, as a schema-1 database, the backup is moved to `path` and the
+/// directory is synced. Every kept file is left as it is.
+///
 /// # Errors
 ///
 /// Returns [`StorageError::MigrationLocked`] if another process holds the
 /// lock, [`StorageError::RestoreRefused`] for files a restore cannot start
-/// from, including a database another process still has open,
+/// from or finish, including a database another process still has open,
 /// [`StorageError::NewerSchema`] or [`StorageError::UnrecognizedSchema`] for a
 /// database rustmail cannot read, and [`StorageError::Database`] or
 /// [`StorageError::MigrationIo`] if SQLite or the filesystem fails. Nothing is
 /// renamed unless every check passed.
 pub async fn restore_backup(path: &Path) -> Result<RestoreReport, StorageError> {
   let paths = Paths::new(path);
   let _lock = acquire_lock(&paths).await?;
-  match inspect_db(&paths.db).await? {
-    DbState::Current => {}
-    state => {
-      return Err(
-        RestoreRefusal::NotMigrated {
-          database: paths.db.clone(),
-          database_state: state.describe(),
-        }
-        .into(),
-      );
-    }
+  let state = inspect_db(&paths.db).await?;
+  let interrupted = state == DbState::Absent && exists(&paths.backup).await?;
+  if !interrupted && state != DbState::Current {
+    return Err(
+      RestoreRefusal::NotMigrated {
+        database: paths.db.clone(),
+        database_state: state.describe(),
+      }
+      .into(),
+    );
   }
   if exists(&paths.migrating).await? {
     return Err(
@@ -79,6 +89,9 @@ pub async fn restore_backup(path: &Path) -> Result<RestoreReport, StorageError>
       .into(),
     );
   }
+  if interrupted {
+    return finish_interrupted_restore(paths).await;
+  }
   if !exists(&paths.backup).await? {
     return Err(
       RestoreRefusal::NoBackup {
@@ -99,6 +112,32 @@ pub async fn restore_backup(path: &Path) -> Result<RestoreReport, StorageError>
     return Err(RestoreRefusal::KeptNameTaken { kept: kept_as }.into());
   }
   rename(&paths.db, &kept_as).await?;
+  put_backup_in_place(paths, kept_as).await
+}
+
+/// Finishes a restore that moved the schema-1 database away but not the
+/// backup into its place, after the same checks a restore starts with.
+async fn finish_interrupted_restore(paths: Paths) -> Result<RestoreReport, StorageError> {
+  let kept_as = interrupted_kept_file(&paths).await?;
+  refuse_side_files(&paths.db, &paths.db).await?;
+  refuse_side_files(&paths.db, &paths.backup).await?;
+  check_backup(&paths.backup).await?;
+  refuse_side_files(&paths.db, &paths.backup).await?;
+  refuse_side_files(&paths.db, &paths.db).await?;
+  info!(
+    event = "storage_restore_resume",
+    database = %paths.db.display(),
+    kept_as = %kept_as.display(),
+    "Finishing a restore that stopped between its two renames"
+  );
+  put_backup_in_place(paths, kept_as).await
+}
+
+/// Moves the backup to the database path, now free, and syncs the directory.
+async fn put_backup_in_place(
+  paths: Paths,
+  kept_as: PathBuf,
+) -> Result<RestoreReport, StorageError> {
   rename(&paths.backup, &paths.db).await?;
   sync_parent(&paths.db).await?;
   info!(
@@ -115,6 +154,79 @@ pub async fn restore_backup(path: &Path) -> Result<RestoreReport, StorageError>
   })
 }
 
+/// The schema-1 database an interrupted restore moved away: the newest
+/// `<db>.schema1-<UTC timestamp>` beside the missing database. It must have no
+/// `-wal`, `-shm` or `-journal` and read as a schema-1 database, and is opened
+/// as immutable so the check never writes it. Older ones are kept files of
+/// earlier restores.
+async fn interrupted_kept_file(paths: &Paths) -> Result<PathBuf, StorageError> {
+  let refuse = |reason: String| -> StorageError {
+    RestoreRefusal::NotAnInterruptedRestore {
+      database: paths.db.clone(),
+      backup: paths.backup.clone(),
+      reason,
+    }
+    .into()
+  };
+  let Some(kept_as) = kept_files(&paths.db).await?.pop() else {
+    return Err(refuse(format!(
+      "no {}<UTC timestamp> file sits beside it",
+      sidecar(&paths.db, KEPT_SUFFIX).display()
+    )));
+  };
+  refuse_side_files(&paths.db, &kept_as).await?;
+  match inspect_db_immutable(&kept_as).await {
+    Ok(DbState::Current) => Ok(kept_as),
+    Ok(state) => Err(refuse(format!(
+      "{} is {}, not schema 1",
+      kept_as.display(),
+      state.describe()
+    ))),
+    Err(error) => Err(refuse(format!(
+      "{} could not be read as a database ({error})",
+      kept_as.display()
+    ))),
+  }
+}
+
+/// The files named `<db>.schema1-<UTC timestamp>` beside `db`, oldest first.
+async fn kept_files(db: &Path) -> Result<Vec<PathBuf>, StorageError> {
+  let parent = db.parent().unwrap_or(Path::new("")).to_path_buf();
+  let listed = if parent.as_os_str().is_empty() {
+    PathBuf::from(".")
+  } else {
+    parent.clone()
+  };
+  let prefix = sidecar(db, KEPT_SUFFIX).to_string_lossy().into_owned();
+  blocking(move || {
+    let list_error = |source| io_error("list the directory", &listed, source);
+    let mut kept = Vec::new();
+    for entry in std::fs::read_dir(&listed).map_err(list_error)? {
+      let path = parent.join(entry.map_err(list_error)?.file_name());
+      if path
+        .to_string_lossy()
+        .strip_prefix(prefix.as_str())
+        .is_some_and(is_timestamp)
+      {
+        kept.push(path);
+      }
+    }
+    kept.sort();
+    Ok(kept)
+  })
+  .await
+}
+
+/// Whether `text` has the shape [`timestamp`] writes, `YYYYMMDDTHHMMSSZ`.
+fn is_timestamp(text: &str) -> bool {
+  let all_digits =
+    |part: &str, len: usize| part.len() == len && part.bytes().all(|byte| byte.is_ascii_digit());
+  text
+    .strip_suffix('Z')
+    .and_then(|rest| rest.split_once('T'))
+    .is_some_and(|(date, time)| all_digits(date, DATE_DIGITS) && all_digits(time, TIME_DIGITS))
+}
+
 /// Refuses if `file` has a `-wal`, `-shm` or `-journal` beside it.
 async fn refuse_side_files(database: &Path, file: &Path) -> Result<(), StorageError> {
   for suffix in SIDE_SUFFIXES {