diff --git a/pubky-homeserver/config.default.toml b/pubky-homeserver/config.default.toml index 7373e4b..3a0a77f 100644 --- a/pubky-homeserver/config.default.toml +++ b/pubky-homeserver/config.default.toml @@ -4,6 +4,11 @@ # "token_required" - a signup token is required to signup. signup_mode = "token_required" +# LMDB backup interval in seconds. 0 means disabled. +# Periodically creates a "safe to copy" compacted backup on single +# file under `{data_dir}/data/lmdb/backup.mdb` +lmdb_backup_interval_s = 0 + [drive] # The port number to run an HTTPS (Pkarr TLS) server on. # Pkarr TLS is a TLS implementation that is compatible with the Pkarr protocol. @@ -41,7 +46,7 @@ admin_password = "admin" public_socket = "127.0.0.1:6286" # The interval at which user keys are republished to the DHT. -user_keys_republisher_interval = 14400 # 4 hours in seconds +user_keys_republisher_interval = 14400 # 4 hours in seconds # List of bootstrap nodes for the DHT. # domain:port format. @@ -49,12 +54,9 @@ dht_bootstrap_nodes = [ "router.bittorrent.com:6881", "dht.transmissionbt.com:6881", "dht.libtorrent.org:25401", - "relay.pkarr.org:6881" + "relay.pkarr.org:6881", ] # Relay node urls for the DHT. # Improves the availability of pkarr packets. -dht_relay_nodes = [ - "https://relay.pkarr.org", - "https://pkarr.pubky.org" -] \ No newline at end of file +dht_relay_nodes = ["https://relay.pkarr.org", "https://pkarr.pubky.org"] diff --git a/pubky-homeserver/src/core/backup.rs b/pubky-homeserver/src/core/backup.rs new file mode 100644 index 0000000..90f2ef9 --- /dev/null +++ b/pubky-homeserver/src/core/backup.rs @@ -0,0 +1,120 @@ +use crate::core::database::DB; +use heed::CompactionOption; +use std::path::PathBuf; +use std::time::Duration; +use tokio::time::interval; +use tracing::{error, info}; + +/// Periodically creates a backup of the LMDB environment every 4 hours. +/// +/// The backup process performs the following steps: +/// 1. Copies the LMDB environment to a temporary file (with a `.tmp` extension), +/// ensuring it’s safe for moving or copying. +/// 2. Atomically renames the temporary file to a final backup file (with a `.mdb` extension). +/// +/// # Arguments +/// +/// * `db` - The LMDB database handle. +/// * `backup_path` - The base path for the backup file (extensions will be appended). +pub async fn backup_lmdb_periodically(db: DB, backup_path: PathBuf, period: Duration) { + let mut interval_timer = interval(period); + + interval_timer.tick().await; // Ignore the first tick as it is instant. + + loop { + // Wait for the next backup tick. + interval_timer.tick().await; + + // Clone the database handle and backup path for use in the blocking task. + let db_clone = db.clone(); + let backup_path_clone = backup_path.clone(); + + // Execute the backup operation in a blocking task. + tokio::task::spawn_blocking(move || { + do_backup(db_clone, backup_path_clone); + }) + .await + .map_err(|e| error!("Backup task panicked: {:?}", e)) + .ok(); + } +} + +/// Performs the actual backup of the LMDB environment. +/// +/// It first writes the backup to a temporary file and, upon success, renames it +/// to the final backup file. Any errors encountered during these operations are logged. +/// +/// # Arguments +/// +/// * `db` - The LMDB database handle. +/// * `backup_path` - The base path for the backup file (extensions will be appended). +fn do_backup(db: DB, backup_path: PathBuf) { + // Define file paths for the temporary and final backup files. + let final_backup_path = backup_path.with_extension("mdb"); + let temp_backup_path = backup_path.with_extension("tmp"); + + // Create a backup by copying the LMDB environment to the temporary file. + if let Err(e) = db + .env + .copy_to_file(&temp_backup_path, CompactionOption::Enabled) + { + error!( + "Failed to create temporary LMDB backup at {:?}: {:?}", + temp_backup_path, e + ); + return; + } + + // Atomically rename the temporary file to the final backup file. + if let Err(e) = std::fs::rename(&temp_backup_path, &final_backup_path) { + error!( + "Failed to rename temporary backup file {:?} to final backup file {:?}: {:?}", + temp_backup_path, final_backup_path, e + ); + return; + } + + info!( + "LMDB backup successfully created at {:?}", + final_backup_path + ); +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + /// Tests that the backup creates the final backup file with the `.mdb` extension + /// and that no temporary `.tmp` file is left after the backup process. + #[test] + fn test_do_backup_creates_backup_file() { + // Create a test DB instance. + let db = DB::test(); + + // Create a temporary directory to store the backup. + let temp_dir = tempdir().expect("Failed to create temporary directory"); + let backup_path = temp_dir.path().join("lmdb_backup"); + + // Perform the backup. + do_backup(db, backup_path.clone()); + + // Define the expected final and temporary backup file paths. + let final_backup_file = backup_path.with_extension("mdb"); + let temp_backup_file = backup_path.with_extension("tmp"); + + // Assert the final backup file exists. + assert!( + final_backup_file.exists(), + "Expected final backup file at {:?} to exist.", + final_backup_file + ); + + // Assert that the temporary backup file was cleaned up. + assert!( + !temp_backup_file.exists(), + "Expected temporary backup file at {:?} to be removed.", + temp_backup_file + ); + } +} diff --git a/pubky-homeserver/src/core/homeserver_core.rs b/pubky-homeserver/src/core/homeserver_core.rs index 8bcee07..b03b8c1 100644 --- a/pubky-homeserver/src/core/homeserver_core.rs +++ b/pubky-homeserver/src/core/homeserver_core.rs @@ -1,5 +1,6 @@ use std::{path::PathBuf, time::Duration}; +use super::backup::backup_lmdb_periodically; use crate::core::database::DB; use crate::core::user_keys_republisher::UserKeysRepublisher; use crate::SignupMode; @@ -47,6 +48,16 @@ impl HomeserverCore { admin, }; + // Spawn the backup process. This task will run forever. + if let Some(backup_interval) = config.lmdb_backup_interval { + let backup_path = config.storage.join("backup"); + tokio::spawn(backup_lmdb_periodically( + db.clone(), + backup_path, + backup_interval, + )); + } + let router = super::routes::create_app(state.clone()); let user_keys_republisher = UserKeysRepublisher::new( @@ -116,6 +127,9 @@ pub struct CoreConfig { /// /// Defaults to `60*60*4` (4 hours) pub user_keys_republisher_interval: Option, + + /// The interval at which the LMDB backup is performed. None means disabled. + pub lmdb_backup_interval: Option, } impl Default for CoreConfig { @@ -129,6 +143,8 @@ impl Default for CoreConfig { max_list_limit: DEFAULT_MAX_LIST_LIMIT, user_keys_republisher_interval: Some(Duration::from_secs(60 * 60 * 4)), + + lmdb_backup_interval: None, } } } @@ -142,7 +158,7 @@ impl CoreConfig { Self { storage, db_map_size: 10485760, - + lmdb_backup_interval: None, ..Default::default() } } diff --git a/pubky-homeserver/src/core/mod.rs b/pubky-homeserver/src/core/mod.rs index 619ca61..c30a063 100644 --- a/pubky-homeserver/src/core/mod.rs +++ b/pubky-homeserver/src/core/mod.rs @@ -1,3 +1,4 @@ +mod backup; pub mod database; mod error; mod extractors; diff --git a/pubky-homeserver/src/data_directory/config_toml.rs b/pubky-homeserver/src/data_directory/config_toml.rs index 3c0f28b..a679190 100644 --- a/pubky-homeserver/src/data_directory/config_toml.rs +++ b/pubky-homeserver/src/data_directory/config_toml.rs @@ -114,12 +114,19 @@ pub struct GeneralToml { /// The mode of the signup. #[serde(default = "default_signup_mode")] pub signup_mode: SignupMode, + /// LMDB backup interval in seconds. 0 means disabled. + #[serde(default = "default_lmdb_backup_interval_s")] + pub lmdb_backup_interval_s: u64, } fn default_signup_mode() -> SignupMode { SignupMode::TokenRequired } +fn default_lmdb_backup_interval_s() -> u64 { + 0 +} + /// The error that can occur when reading the config file #[derive(Debug, thiserror::Error)] pub enum ConfigReadError { diff --git a/pubky-homeserver/src/homeserver/server.rs b/pubky-homeserver/src/homeserver/server.rs index 6170c82..dd99d16 100644 --- a/pubky-homeserver/src/homeserver/server.rs +++ b/pubky-homeserver/src/homeserver/server.rs @@ -261,6 +261,11 @@ impl TryFrom for Config { user_keys_republisher_interval: Some(Duration::from_secs( conf.pkdns.user_keys_republisher_interval.into(), )), + lmdb_backup_interval: if conf.general.lmdb_backup_interval_s == 0 { + None + } else { + Some(Duration::from_secs(conf.general.lmdb_backup_interval_s)) + }, ..Default::default() };