feat(homeserver): add backup service (#92)

* feat(homeserver): add backup service

* Fix PR review

* fix: make lmdb backup interval a config

* fix change default backup to disabled

pubky-homeserver/config.default.toml

@@ -4,6 +4,11 @@
 # "token_required" - a signup token is required to signup.
 signup_mode = "token_required"
 
+# LMDB backup interval in seconds. 0 means disabled.
+# Periodically creates a "safe to copy" compacted backup on single
+# file under `{data_dir}/data/lmdb/backup.mdb`
+lmdb_backup_interval_s = 0
+
 [drive]
 # The port number to run an HTTPS (Pkarr TLS) server on.
 # Pkarr TLS is a TLS implementation that is compatible with the Pkarr protocol.
@@ -41,7 +46,7 @@ admin_password = "admin"
 public_socket = "127.0.0.1:6286"
 
 # The interval at which user keys are republished to the DHT.
-user_keys_republisher_interval = 14400  # 4 hours in seconds
+user_keys_republisher_interval = 14400 # 4 hours in seconds
 
 # List of bootstrap nodes for the DHT.
 # domain:port format.
@@ -49,12 +54,9 @@ dht_bootstrap_nodes = [
     "router.bittorrent.com:6881",
     "dht.transmissionbt.com:6881",
     "dht.libtorrent.org:25401",
-    "relay.pkarr.org:6881"
+    "relay.pkarr.org:6881",
 ]
 
 # Relay node urls for the DHT.
 # Improves the availability of pkarr packets.
-dht_relay_nodes = [
-    "https://relay.pkarr.org", 
-    "https://pkarr.pubky.org"
-]
+dht_relay_nodes = ["https://relay.pkarr.org", "https://pkarr.pubky.org"]

pubky-homeserver/src/core/backup.rs

@@ -0,0 +1,120 @@
+use crate::core::database::DB;
+use heed::CompactionOption;
+use std::path::PathBuf;
+use std::time::Duration;
+use tokio::time::interval;
+use tracing::{error, info};
+
+/// Periodically creates a backup of the LMDB environment every 4 hours.
+///
+/// The backup process performs the following steps:
+/// 1. Copies the LMDB environment to a temporary file (with a `.tmp` extension),
+///    ensuring it’s safe for moving or copying.
+/// 2. Atomically renames the temporary file to a final backup file (with a `.mdb` extension).
+///
+/// # Arguments
+///
+/// * `db` - The LMDB database handle.
+/// * `backup_path` - The base path for the backup file (extensions will be appended).
+pub async fn backup_lmdb_periodically(db: DB, backup_path: PathBuf, period: Duration) {
+    let mut interval_timer = interval(period);
+
+    interval_timer.tick().await; // Ignore the first tick as it is instant.
+
+    loop {
+        // Wait for the next backup tick.
+        interval_timer.tick().await;
+
+        // Clone the database handle and backup path for use in the blocking task.
+        let db_clone = db.clone();
+        let backup_path_clone = backup_path.clone();
+
+        // Execute the backup operation in a blocking task.
+        tokio::task::spawn_blocking(move || {
+            do_backup(db_clone, backup_path_clone);
+        })
+        .await
+        .map_err(|e| error!("Backup task panicked: {:?}", e))
+        .ok();
+    }
+}
+
+/// Performs the actual backup of the LMDB environment.
+///
+/// It first writes the backup to a temporary file and, upon success, renames it
+/// to the final backup file. Any errors encountered during these operations are logged.
+///
+/// # Arguments
+///
+/// * `db` - The LMDB database handle.
+/// * `backup_path` - The base path for the backup file (extensions will be appended).
+fn do_backup(db: DB, backup_path: PathBuf) {
+    // Define file paths for the temporary and final backup files.
+    let final_backup_path = backup_path.with_extension("mdb");
+    let temp_backup_path = backup_path.with_extension("tmp");
+
+    // Create a backup by copying the LMDB environment to the temporary file.
+    if let Err(e) = db
+        .env
+        .copy_to_file(&temp_backup_path, CompactionOption::Enabled)
+    {
+        error!(
+            "Failed to create temporary LMDB backup at {:?}: {:?}",
+            temp_backup_path, e
+        );
+        return;
+    }
+
+    // Atomically rename the temporary file to the final backup file.
+    if let Err(e) = std::fs::rename(&temp_backup_path, &final_backup_path) {
+        error!(
+            "Failed to rename temporary backup file {:?} to final backup file {:?}: {:?}",
+            temp_backup_path, final_backup_path, e
+        );
+        return;
+    }
+
+    info!(
+        "LMDB backup successfully created at {:?}",
+        final_backup_path
+    );
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile::tempdir;
+
+    /// Tests that the backup creates the final backup file with the `.mdb` extension
+    /// and that no temporary `.tmp` file is left after the backup process.
+    #[test]
+    fn test_do_backup_creates_backup_file() {
+        // Create a test DB instance.
+        let db = DB::test();
+
+        // Create a temporary directory to store the backup.
+        let temp_dir = tempdir().expect("Failed to create temporary directory");
+        let backup_path = temp_dir.path().join("lmdb_backup");
+
+        // Perform the backup.
+        do_backup(db, backup_path.clone());
+
+        // Define the expected final and temporary backup file paths.
+        let final_backup_file = backup_path.with_extension("mdb");
+        let temp_backup_file = backup_path.with_extension("tmp");
+
+        // Assert the final backup file exists.
+        assert!(
+            final_backup_file.exists(),
+            "Expected final backup file at {:?} to exist.",
+            final_backup_file
+        );
+
+        // Assert that the temporary backup file was cleaned up.
+        assert!(
+            !temp_backup_file.exists(),
+            "Expected temporary backup file at {:?} to be removed.",
+            temp_backup_file
+        );
+    }
+}

pubky-homeserver/src/core/homeserver_core.rs

@@ -1,5 +1,6 @@
 use std::{path::PathBuf, time::Duration};
 
+use super::backup::backup_lmdb_periodically;
 use crate::core::database::DB;
 use crate::core::user_keys_republisher::UserKeysRepublisher;
 use crate::SignupMode;
@@ -47,6 +48,16 @@ impl HomeserverCore {
             admin,
         };
 
+        // Spawn the backup process. This task will run forever.
+        if let Some(backup_interval) = config.lmdb_backup_interval {
+            let backup_path = config.storage.join("backup");
+            tokio::spawn(backup_lmdb_periodically(
+                db.clone(),
+                backup_path,
+                backup_interval,
+            ));
+        }
+
         let router = super::routes::create_app(state.clone());
 
         let user_keys_republisher = UserKeysRepublisher::new(
@@ -116,6 +127,9 @@ pub struct CoreConfig {
     ///
     /// Defaults to `60*60*4` (4 hours)
     pub user_keys_republisher_interval: Option<Duration>,
+
+    /// The interval at which the LMDB backup is performed. None means disabled.
+    pub lmdb_backup_interval: Option<Duration>,
 }
 
 impl Default for CoreConfig {
@@ -129,6 +143,8 @@ impl Default for CoreConfig {
             max_list_limit: DEFAULT_MAX_LIST_LIMIT,
 
             user_keys_republisher_interval: Some(Duration::from_secs(60 * 60 * 4)),
+
+            lmdb_backup_interval: None,
         }
     }
 }
@@ -142,7 +158,7 @@ impl CoreConfig {
         Self {
             storage,
             db_map_size: 10485760,
-
+            lmdb_backup_interval: None,
             ..Default::default()
         }
     }

pubky-homeserver/src/core/mod.rs

@@ -1,3 +1,4 @@
+mod backup;
 pub mod database;
 mod error;
 mod extractors;

pubky-homeserver/src/data_directory/config_toml.rs

@@ -114,12 +114,19 @@ pub struct GeneralToml {
     /// The mode of the signup.
     #[serde(default = "default_signup_mode")]
     pub signup_mode: SignupMode,
+    /// LMDB backup interval in seconds. 0 means disabled.
+    #[serde(default = "default_lmdb_backup_interval_s")]
+    pub lmdb_backup_interval_s: u64,
 }
 
 fn default_signup_mode() -> SignupMode {
     SignupMode::TokenRequired
 }
 
+fn default_lmdb_backup_interval_s() -> u64 {
+    0
+}
+
 /// The error that can occur when reading the config file
 #[derive(Debug, thiserror::Error)]
 pub enum ConfigReadError {

pubky-homeserver/src/homeserver/server.rs

@@ -261,6 +261,11 @@ impl TryFrom<DataDir> for Config {
             user_keys_republisher_interval: Some(Duration::from_secs(
                 conf.pkdns.user_keys_republisher_interval.into(),
             )),
+            lmdb_backup_interval: if conf.general.lmdb_backup_interval_s == 0 {
+                None
+            } else {
+                Some(Duration::from_secs(conf.general.lmdb_backup_interval_s))
+            },
             ..Default::default()
         };