From 3ebdeff3540628a869cbba19d28293dfe4836bdc Mon Sep 17 00:00:00 2001 From: nazeh Date: Sat, 14 Dec 2024 12:48:53 +0300 Subject: [PATCH] feat(homeserver): use Pkarr Relay in testnet and new Config simplifications --- Cargo.lock | 163 ++++++++++++++- Cargo.toml | 2 +- examples/Cargo.toml | 12 +- pubky-homeserver/Cargo.toml | 1 + pubky-homeserver/README.md | 8 + pubky-homeserver/src/config.example.toml | 3 - pubky-homeserver/src/core/config.rs | 197 +++++------------- pubky-homeserver/src/core/database/mod.rs | 78 ++++--- .../src/core/database/tables/entries.rs | 6 +- .../src/core/database/tables/events.rs | 6 +- pubky-homeserver/src/core/error.rs | 4 +- pubky-homeserver/src/core/mod.rs | 17 +- pubky-homeserver/src/io/http.rs | 10 +- pubky-homeserver/src/io/mod.rs | 80 ++++++- pubky-homeserver/src/io/pkarr.rs | 92 ++++---- pubky-homeserver/src/main.rs | 11 +- pubky/pkg/package.json | 2 +- pubky/pkg/test/http.js | 2 +- pubky/src/native.rs | 28 +-- 19 files changed, 445 insertions(+), 277 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ce139ea..515f6f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -631,6 +631,19 @@ dependencies = [ "syn", ] +[[package]] +name = "dashmap" +version = "5.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" +dependencies = [ + "cfg-if", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "der" version = "0.7.9" @@ -834,6 +847,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "forwarded-header-value" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8835f84f38484cc86f110a805655697908257fb9a7af005234060891557198e9" +dependencies = [ + "nonempty", + "thiserror 1.0.69", +] + [[package]] name = "futures" version = "0.3.31" @@ -918,6 +941,12 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + [[package]] name = "futures-util" version = "0.3.31" @@ -982,6 +1011,26 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "governor" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68a7f542ee6b35af73b06abc0dad1c1bae89964e4e253bc4b587b91c9637867b" +dependencies = [ + "cfg-if", + "dashmap", + "futures", + "futures-timer", + "no-std-compat", + "nonzero_ext", + "parking_lot", + "portable-atomic", + "quanta", + "rand", + "smallvec", + "spinning_top", +] + [[package]] name = "h2" version = "0.4.6" @@ -1010,6 +1059,12 @@ dependencies = [ "byteorder", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + [[package]] name = "hashbrown" version = "0.15.1" @@ -1404,7 +1459,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.15.1", ] [[package]] @@ -1520,8 +1575,7 @@ checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" [[package]] name = "mainline" version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e18c8b0210572062a02c4de8c448865f4ca89824c4ac7da64a0c2669ea2c405" +source = "git+https://github.com/pubky/mainline?branch=v5#2948896ac074cd5bf9728966dcd076de35e7e763" dependencies = [ "bytes", "crc", @@ -1629,6 +1683,24 @@ dependencies = [ "tempfile", ] +[[package]] +name = "no-std-compat" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" + +[[package]] +name = "nonempty" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9e591e719385e6ebaeb5ce5d3887f7d5676fceca6411d1925ccc95745f3d6f7" + +[[package]] +name = "nonzero_ext" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -1849,7 +1921,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkarr" version = "3.0.0" -source = "git+https://github.com/Pubky/pkarr?branch=v4#caaf12b437b952cae3e189afc035e35815933eae" +source = "git+https://github.com/Pubky/pkarr?branch=v3#c3510a82c5a3a496349d8dc030edd96f750f1471" dependencies = [ "base32", "byteorder", @@ -1883,6 +1955,33 @@ dependencies = [ "wasm-bindgen-futures", ] +[[package]] +name = "pkarr-relay" +version = "0.1.0" +source = "git+https://github.com/Pubky/pkarr?branch=v3#c3510a82c5a3a496349d8dc030edd96f750f1471" +dependencies = [ + "anyhow", + "axum", + "axum-server", + "bytes", + "clap", + "dirs-next", + "governor", + "http", + "httpdate", + "pkarr", + "pubky-timestamp", + "rustls", + "serde", + "thiserror 2.0.6", + "tokio", + "toml", + "tower-http", + "tower_governor", + "tracing", + "tracing-subscriber", +] + [[package]] name = "pkcs8" version = "0.10.2" @@ -1910,6 +2009,12 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "portable-atomic" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" + [[package]] name = "postcard" version = "1.1.1" @@ -2015,6 +2120,7 @@ dependencies = [ "httpdate", "page_size", "pkarr", + "pkarr-relay", "postcard", "pubky-common", "serde", @@ -2053,6 +2159,21 @@ dependencies = [ "psl-types", ] +[[package]] +name = "quanta" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5167a477619228a0b284fac2674e3c388cba90631d7b7de620e6f1fcd08da5" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + [[package]] name = "quinn" version = "0.11.6" @@ -2144,6 +2265,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "raw-cpuid" +version = "11.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ab240315c661615f2ee9f0f2cd32d5a7343a84d5ebcccb99d46e6637565e7b0" +dependencies = [ + "bitflags", +] + [[package]] name = "redox_syscall" version = "0.5.7" @@ -2634,6 +2764,15 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spinning_top" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96d2d1d716fb500937168cc09353ffdc7a012be8475ac7308e1bdf0e3923300" +dependencies = [ + "lock_api", +] + [[package]] name = "spki" version = "0.7.3" @@ -3021,6 +3160,22 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" +[[package]] +name = "tower_governor" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aea939ea6cfa7c4880f3e7422616624f97a567c16df67b53b11f0d03917a8e46" +dependencies = [ + "axum", + "forwarded-header-value", + "governor", + "http", + "pin-project", + "thiserror 1.0.69", + "tower 0.5.1", + "tracing", +] + [[package]] name = "tracing" version = "0.1.41" diff --git a/Cargo.toml b/Cargo.toml index ff0f69b..eb66b50 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ members = [ resolver = "2" [workspace.dependencies] -pkarr = { git = "https://github.com/Pubky/pkarr", branch = "v4", package = "pkarr" } +pkarr = { git = "https://github.com/Pubky/pkarr", branch = "v3", package = "pkarr" } [profile.release] lto = true diff --git a/examples/Cargo.toml b/examples/Cargo.toml index bf61234..878c587 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -16,13 +16,13 @@ name = "request" path = "./request/main.rs" [dependencies] -anyhow = "1.0.86" +anyhow = "1.0.94" base64 = "0.22.1" -clap = { version = "4.5.16", features = ["derive"] } +clap = { version = "4.5.23", features = ["derive"] } pubky = { path = "../pubky" } pubky-common = { version = "0.1.0", path = "../pubky-common" } -reqwest = "0.12.8" +reqwest = "0.12.9" rpassword = "7.3.1" -tokio = { version = "1.40.0", features = ["macros", "rt-multi-thread"] } -tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } -url = "2.5.2" +tokio = { version = "1.42.0", features = ["macros", "rt-multi-thread"] } +tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } +url = "2.5.4" diff --git a/pubky-homeserver/Cargo.toml b/pubky-homeserver/Cargo.toml index b3050e8..333c3f9 100644 --- a/pubky-homeserver/Cargo.toml +++ b/pubky-homeserver/Cargo.toml @@ -31,3 +31,4 @@ axum-server = { version = "0.7.1", features = ["tls-rustls-no-provider"] } tower = "0.5.1" page_size = "0.6.0" +pkarr-relay = { git = "https://github.com/Pubky/pkarr", branch = "v3", package = "pkarr-relay" } diff --git a/pubky-homeserver/README.md b/pubky-homeserver/README.md index 8aa2c26..ee11bc0 100644 --- a/pubky-homeserver/README.md +++ b/pubky-homeserver/README.md @@ -23,3 +23,11 @@ Run with an optional config file ```bash ../target/release/pubky-homeserver --config=./src/config.toml ``` + +## Testnet + +To run a local homeserver for testing with an internal Pkarr Relay, hardcoded well known publickey and only connected to local Mainline testnet: + +```bash +cargo run -- --testnet +``` diff --git a/pubky-homeserver/src/config.example.toml b/pubky-homeserver/src/config.example.toml index cb47003..9584bcb 100644 --- a/pubky-homeserver/src/config.example.toml +++ b/pubky-homeserver/src/config.example.toml @@ -1,6 +1,3 @@ -# Use testnet network (local DHT) for testing. -# testnet = false - # Secret key (in hex) to generate the Homeserver's Keypair # secret_key = "0000000000000000000000000000000000000000000000000000000000000000" diff --git a/pubky-homeserver/src/core/config.rs b/pubky-homeserver/src/core/config.rs index e94a720..1727b31 100644 --- a/pubky-homeserver/src/core/config.rs +++ b/pubky-homeserver/src/core/config.rs @@ -8,9 +8,6 @@ use std::{ path::{Path, PathBuf}, time::Duration, }; -use tracing::info; - -use pubky_common::timestamp::Timestamp; // === Database === const DEFAULT_STORAGE_DIR: &str = "pubky"; @@ -22,7 +19,6 @@ pub const DEFAULT_MAX_LIST_LIMIT: u16 = 1000; #[derive(Serialize, Deserialize, Clone, PartialEq)] struct ConfigToml { - testnet: Option, port: Option, bootstrap: Option>, domain: Option, @@ -37,90 +33,42 @@ struct ConfigToml { /// Server configuration #[derive(Debug, Clone, PartialEq, Eq)] pub struct Config { - /// Whether or not this server is running in a testnet. - testnet: bool, /// The configured port for this server. - port: u16, + pub port: u16, /// Bootstrapping DHT nodes. /// /// Helpful to run the server locally or in testnet. - bootstrap: Option>, + pub bootstrap: Option>, /// A public domain for this server /// necessary for web browsers running in https environment. - domain: Option, + pub domain: Option, /// Path to the storage directory. /// /// Defaults to a directory in the OS data directory - storage: PathBuf, + pub storage: PathBuf, /// Server keypair. /// /// Defaults to a random keypair. - keypair: Keypair, - dht_request_timeout: Option, + pub keypair: Keypair, + pub dht_request_timeout: Option, /// The default limit of a list api if no `limit` query parameter is provided. /// /// Defaults to `100` - default_list_limit: u16, + pub default_list_limit: u16, /// The maximum limit of a list api, even if a `limit` query parameter is provided. /// /// Defaults to `1000` - max_list_limit: u16, + pub max_list_limit: u16, // === Database params === - db_map_size: usize, + pub db_map_size: usize, } impl Config { fn try_from_str(value: &str) -> Result { let config_toml: ConfigToml = toml::from_str(value)?; - let keypair = if let Some(secret_key) = config_toml.secret_key { - let secret_key = deserialize_secret_key(secret_key)?; - Keypair::from_secret_key(&secret_key) - } else { - Keypair::random() - }; - - let storage = { - let dir = if let Some(storage) = config_toml.storage { - storage - } else { - let path = dirs_next::data_dir().ok_or_else(|| { - anyhow!("operating environment provides no directory for application data") - })?; - path.join(DEFAULT_STORAGE_DIR) - }; - - dir.join("homeserver") - }; - - let config = Config { - testnet: config_toml.testnet.unwrap_or(false), - port: config_toml.port.unwrap_or(0), - bootstrap: config_toml.bootstrap, - domain: config_toml.domain, - keypair, - storage, - dht_request_timeout: config_toml.dht_request_timeout, - default_list_limit: config_toml.default_list_limit.unwrap_or(DEFAULT_LIST_LIMIT), - max_list_limit: config_toml - .default_list_limit - .unwrap_or(DEFAULT_MAX_LIST_LIMIT), - db_map_size: config_toml.db_map_size.unwrap_or(DEFAULT_MAP_SIZE), - }; - - if config.testnet { - let testnet_config = Config::testnet(); - - return Ok(Config { - bootstrap: testnet_config.bootstrap, - port: testnet_config.port, - keypair: testnet_config.keypair, - ..config - }); - } - - Ok(config) + config_toml.try_into() } /// Load the config from a file. @@ -132,82 +80,25 @@ impl Config { Config::try_from_str(&s) } - /// Testnet configurations - pub fn testnet() -> Self { - let testnet = pkarr::mainline::Testnet::new(10).unwrap(); - info!(?testnet.bootstrap, "Testnet bootstrap nodes"); - - Config { - port: 15411, - dht_request_timeout: None, - db_map_size: DEFAULT_MAP_SIZE, - keypair: Keypair::from_secret_key(&[0; 32]), - ..Self::test(&testnet) - } - } - /// Test configurations pub fn test(testnet: &pkarr::mainline::Testnet) -> Self { let bootstrap = Some(testnet.bootstrap.to_owned()); let storage = std::env::temp_dir() - .join(Timestamp::now().to_string()) + .join(pubky_common::timestamp::Timestamp::now().to_string()) .join(DEFAULT_STORAGE_DIR); Self { - testnet: true, bootstrap, storage, db_map_size: 10485760, ..Default::default() } } - - pub fn is_testnet(&self) -> bool { - self.testnet - } - - pub fn port(&self) -> u16 { - self.port - } - - pub fn bootstrap(&self) -> Option> { - self.bootstrap.to_owned() - } - - pub fn domain(&self) -> Option<&String> { - self.domain.as_ref() - } - - pub fn keypair(&self) -> &Keypair { - &self.keypair - } - - pub fn default_list_limit(&self) -> u16 { - self.default_list_limit - } - - pub fn max_list_limit(&self) -> u16 { - self.max_list_limit - } - - /// Get the path to the storage directory - pub fn storage(&self) -> &PathBuf { - &self.storage - } - - pub(crate) fn dht_request_timeout(&self) -> Option { - self.dht_request_timeout - } - - pub(crate) fn db_map_size(&self) -> usize { - self.db_map_size - } } impl Default for Config { fn default() -> Self { Self { - testnet: false, port: 0, bootstrap: None, domain: None, @@ -222,6 +113,44 @@ impl Default for Config { } } +impl TryFrom for Config { + type Error = anyhow::Error; + + fn try_from(value: ConfigToml) -> std::result::Result { + let keypair = if let Some(secret_key) = value.secret_key { + let secret_key = deserialize_secret_key(secret_key)?; + Keypair::from_secret_key(&secret_key) + } else { + Keypair::random() + }; + + let storage = { + let dir = if let Some(storage) = value.storage { + storage + } else { + let path = dirs_next::data_dir().ok_or_else(|| { + anyhow!("operating environment provides no directory for application data") + })?; + path.join(DEFAULT_STORAGE_DIR) + }; + + dir.join("homeserver") + }; + + Ok(Config { + port: value.port.unwrap_or(0), + bootstrap: value.bootstrap, + domain: value.domain, + keypair, + storage, + dht_request_timeout: value.dht_request_timeout, + default_list_limit: value.default_list_limit.unwrap_or(DEFAULT_LIST_LIMIT), + max_list_limit: value.default_list_limit.unwrap_or(DEFAULT_MAX_LIST_LIMIT), + db_map_size: value.db_map_size.unwrap_or(DEFAULT_MAP_SIZE), + }) + } +} + fn deserialize_secret_key(s: String) -> anyhow::Result<[u8; 32]> { let bytes = hex::decode(s).map_err(|_| anyhow!("secret_key in config.toml should hex encoded"))?; @@ -279,7 +208,6 @@ mod tests { assert_eq!( config, Config { - testnet: true, bootstrap: testnet.bootstrap.into(), db_map_size: 10485760, @@ -291,36 +219,17 @@ mod tests { } #[test] - fn config_testnet() { - let config = Config::testnet(); - - assert_eq!( - config, - Config { - testnet: true, - port: 15411, - - bootstrap: config.bootstrap.clone(), - storage: config.storage.clone(), - keypair: config.keypair.clone(), - ..Default::default() - } - ) - } - - #[test] - fn parse_with_testnet_flag() { + fn parse() { let config = Config::try_from_str( r#" # Secret key (in hex) to generate the Homeserver's Keypair - secret_key = "0123000000000000000000000000000000000000000000000000000000000000" + secret_key = "0000000000000000000000000000000000000000000000000000000000000000" # Domain to be published in Pkarr records for this server to be accessible by. domain = "localhost" # Port for the Homeserver to listen on. port = 6287 # Storage directory Defaults to storage = "/homeserver" - testnet = true bootstrap = ["foo", "bar"] @@ -332,8 +241,8 @@ mod tests { .unwrap(); assert_eq!(config.keypair, Keypair::from_secret_key(&[0; 32])); - assert_eq!(config.port, 15411); - assert_ne!( + assert_eq!(config.port, 6287); + assert_eq!( config.bootstrap, Some(vec!["foo".to_string(), "bar".to_string()]) ); diff --git a/pubky-homeserver/src/core/database/mod.rs b/pubky-homeserver/src/core/database/mod.rs index 617b766..dd05095 100644 --- a/pubky-homeserver/src/core/database/mod.rs +++ b/pubky-homeserver/src/core/database/mod.rs @@ -11,43 +11,57 @@ use crate::core::config::Config; use tables::{Tables, TABLES_COUNT}; -#[derive(Debug, Clone)] -pub struct DB { - pub(crate) env: Env, - pub(crate) tables: Tables, - pub(crate) config: Config, - pub(crate) buffers_dir: PathBuf, - pub(crate) max_chunk_size: usize, -} +pub use protected::DB; -impl DB { - /// # Safety - /// Opening [LMDB][heed::EnvOpenOptions::open] is backed by a memory map which comes with some safety precautions. - pub unsafe fn open(config: Config) -> anyhow::Result { - let buffers_dir = config.storage().clone().join("buffers"); +/// Protecting fields from being mutated by modules in crate::database +mod protected { + use super::*; - // Cleanup buffers. - let _ = fs::remove_dir(&buffers_dir); - fs::create_dir_all(&buffers_dir)?; + #[derive(Debug, Clone)] + pub struct DB { + pub(crate) env: Env, + pub(crate) tables: Tables, + pub(crate) buffers_dir: PathBuf, + pub(crate) max_chunk_size: usize, + config: Config, + } - let env = unsafe { - EnvOpenOptions::new() - .max_dbs(TABLES_COUNT) - .map_size(config.db_map_size()) - .open(config.storage()) - }?; + impl DB { + /// # Safety + /// DB uses LMDB, [opening][heed::EnvOpenOptions::open] which is marked unsafe, + /// because the possible Undefined Behavior (UB) if the lock file is broken. + pub unsafe fn open(config: Config) -> anyhow::Result { + let buffers_dir = config.storage.clone().join("buffers"); - let tables = migrations::run(&env)?; + // Cleanup buffers. + let _ = fs::remove_dir(&buffers_dir); + fs::create_dir_all(&buffers_dir)?; - let db = DB { - env, - tables, - config, - buffers_dir, - max_chunk_size: max_chunk_size(), - }; + let env = unsafe { + EnvOpenOptions::new() + .max_dbs(TABLES_COUNT) + .map_size(config.db_map_size) + .open(&config.storage) + }?; - Ok(db) + let tables = migrations::run(&env)?; + + let db = DB { + env, + tables, + config, + buffers_dir, + max_chunk_size: max_chunk_size(), + }; + + Ok(db) + } + + // === Getters === + + pub fn config(&self) -> &Config { + &self.config + } } } @@ -55,7 +69,7 @@ impl DB { /// - https://lmdb.readthedocs.io/en/release/#storage-efficiency-limits /// - https://github.com/lmdbjava/benchmarks/blob/master/results/20160710/README.md#test-2-determine-24816-kb-byte-values fn max_chunk_size() -> usize { - let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) as usize }; + let page_size = page_size::get(); // - 16 bytes Header per page (LMDB) // - Each page has to contain 2 records diff --git a/pubky-homeserver/src/core/database/tables/entries.rs b/pubky-homeserver/src/core/database/tables/entries.rs index 733833e..255b314 100644 --- a/pubky-homeserver/src/core/database/tables/entries.rs +++ b/pubky-homeserver/src/core/database/tables/entries.rs @@ -107,7 +107,7 @@ impl DB { /// Return a list of pubky urls. /// - /// - limit defaults to [Config::default_list_limit] and capped by [Config::max_list_limit] + /// - limit defaults to [crate::Config::default_list_limit] and capped by [crate::Config::max_list_limit] pub fn list( &self, txn: &RoTxn, @@ -121,8 +121,8 @@ impl DB { let mut results = Vec::new(); let limit = limit - .unwrap_or(self.config.default_list_limit()) - .min(self.config.max_list_limit()); + .unwrap_or(self.config().default_list_limit) + .min(self.config().max_list_limit); // TODO: make this more performant than split and allocations? diff --git a/pubky-homeserver/src/core/database/tables/events.rs b/pubky-homeserver/src/core/database/tables/events.rs index 6895931..39a6f73 100644 --- a/pubky-homeserver/src/core/database/tables/events.rs +++ b/pubky-homeserver/src/core/database/tables/events.rs @@ -62,7 +62,7 @@ impl Event { impl DB { /// Returns a list of events formatted as ` `. /// - /// - limit defaults to [Config::default_list_limit] and capped by [Config::max_list_limit] + /// - limit defaults to [crate::Config::default_list_limit] and capped by [crate::Config::max_list_limit] /// - cursor is a 13 character string encoding of a timestamp pub fn list_events( &self, @@ -72,8 +72,8 @@ impl DB { let txn = self.env.read_txn()?; let limit = limit - .unwrap_or(self.config.default_list_limit()) - .min(self.config.max_list_limit()); + .unwrap_or(self.config().default_list_limit) + .min(self.config().max_list_limit); let cursor = cursor.unwrap_or("0000000000000".to_string()); diff --git a/pubky-homeserver/src/core/error.rs b/pubky-homeserver/src/core/error.rs index dc52d1b..82117c3 100644 --- a/pubky-homeserver/src/core/error.rs +++ b/pubky-homeserver/src/core/error.rs @@ -78,8 +78,8 @@ impl From for Error { } } -impl From for Error { - fn from(error: pkarr::errors::SignedPacketError) -> Self { +impl From for Error { + fn from(error: pkarr::errors::SignedPacketVerifyError) -> Self { Self::new(StatusCode::BAD_REQUEST, Some(error)) } } diff --git a/pubky-homeserver/src/core/mod.rs b/pubky-homeserver/src/core/mod.rs index 5c325c9..c4cf6de 100644 --- a/pubky-homeserver/src/core/mod.rs +++ b/pubky-homeserver/src/core/mod.rs @@ -25,16 +25,19 @@ pub(crate) struct AppState { } #[derive(Debug, Clone)] -/// An I/O-less Core of the [Homeserver]. +/// A side-effect-free Core of the [Homeserver]. pub struct HomeserverCore { - pub(crate) config: Config, + config: Config, pub(crate) state: AppState, pub(crate) router: Router, } impl HomeserverCore { + /// Create a side-effect-free Homeserver core. + /// /// # Safety - /// HomeserverCore uses LMDB, [opening][heed::EnvOpenOptions::open] which comes with some safety precautions. + /// HomeserverCore uses LMDB, [opening][heed::EnvOpenOptions::open] which is marked unsafe, + /// because the possible Undefined Behavior (UB) if the lock file is broken. pub unsafe fn new(config: &Config) -> Result { let db = unsafe { DB::open(config.clone())? }; @@ -62,12 +65,16 @@ impl HomeserverCore { // === Getters === + pub fn config(&self) -> &Config { + &self.config + } + pub fn keypair(&self) -> &Keypair { - self.config.keypair() + &self.config.keypair } pub fn public_key(&self) -> PublicKey { - self.config.keypair().public_key() + self.config.keypair.public_key() } // === Public Methods === diff --git a/pubky-homeserver/src/io/http.rs b/pubky-homeserver/src/io/http.rs index 9b414ab..682c139 100644 --- a/pubky-homeserver/src/io/http.rs +++ b/pubky-homeserver/src/io/http.rs @@ -10,6 +10,7 @@ use axum_server::{ tls_rustls::{RustlsAcceptor, RustlsConfig}, Handle, }; +use futures_util::TryFutureExt; use crate::core::HomeserverCore; @@ -38,11 +39,12 @@ impl HttpServers { core.router .clone() .into_make_service_with_connect_info::(), - ), + ) + .map_err(|error| tracing::error!(?error, "Homeserver http server error")), ); let https_listener = - TcpListener::bind(SocketAddr::from(([0, 0, 0, 0], core.config.port())))?; + TcpListener::bind(SocketAddr::from(([0, 0, 0, 0], core.config().port)))?; let https_handle = Handle::new(); @@ -56,7 +58,8 @@ impl HttpServers { core.router .clone() .into_make_service_with_connect_info::(), - ), + ) + .map_err(|error| tracing::error!(?error, "Homeserver https server error")), ); // let mock_pkarr_relay_listener = TcpListener::bind(SocketAddr::from(([0, 0, 0, 0], 15411)))?; @@ -81,6 +84,7 @@ impl HttpServers { } } + /// Shutdown all HTTP servers. pub fn shutdown(&self) { self.http_handle.shutdown(); self.https_handle.shutdown(); diff --git a/pubky-homeserver/src/io/mod.rs b/pubky-homeserver/src/io/mod.rs index efb5a53..1f9a46a 100644 --- a/pubky-homeserver/src/io/mod.rs +++ b/pubky-homeserver/src/io/mod.rs @@ -1,4 +1,4 @@ -use ::pkarr::{mainline::Testnet, PublicKey}; +use ::pkarr::{Keypair, PublicKey}; use anyhow::Result; use http::HttpServers; use pkarr::PkarrServer; @@ -9,6 +9,34 @@ use crate::{Config, HomeserverCore}; mod http; mod pkarr; +#[derive(Debug, Default)] +pub struct HomeserverBuilder(Config); + +impl HomeserverBuilder { + /// Configure the Homeserver's keypair + pub fn keypair(mut self, keypair: Keypair) -> Self { + self.0.keypair = keypair; + + self + } + + /// Configure the Mainline DHT bootstrap nodes. Useful for testnet configurations. + pub fn bootstrap(mut self, bootstrap: Vec) -> Self { + self.0.bootstrap = Some(bootstrap); + + self + } + + /// Start running a Homeserver + /// + /// # Safety + /// Homeserver uses LMDB, [opening][heed::EnvOpenOptions::open] which is marked unsafe, + /// because the possible Undefined Behavior (UB) if the lock file is broken. + pub async unsafe fn build(self) -> Result { + Homeserver::start(self.0).await + } +} + #[derive(Debug)] /// Homeserver [Core][HomeserverCore] + I/O (http server and pkarr publishing). pub struct Homeserver { @@ -17,8 +45,15 @@ pub struct Homeserver { } impl Homeserver { + pub fn builder() -> HomeserverBuilder { + HomeserverBuilder::default() + } + + /// Start running a Homeserver + /// /// # Safety - /// Homeserver uses LMDB, [opening][heed::EnvOpenOptions::open] which comes with some safety precautions. + /// Homeserver uses LMDB, [opening][heed::EnvOpenOptions::open] which is marked unsafe, + /// because the possible Undefined Behavior (UB) if the lock file is broken. pub async unsafe fn start(config: Config) -> Result { tracing::debug!(?config, "Starting homeserver with configurations"); @@ -33,20 +68,47 @@ impl Homeserver { info!("Publishing Pkarr packet.."); - let pkarr_server = PkarrServer::new(config)?; - pkarr_server - .publish_server_packet(http_servers.https_address().await?.port()) - .await?; + let pkarr_server = PkarrServer::new(config, http_servers.https_address().await?.port())?; + pkarr_server.publish_server_packet().await?; info!("Homeserver listening on https://{}", core.public_key()); Ok(Self { http_servers, core }) } - /// Test version of [Homeserver::start], using mainline Testnet, and a temporary storage. - pub async fn start_test(testnet: &Testnet) -> Result { - info!("Running testnet.."); + /// Start a homeserver in a Testnet mode. + /// + /// - Homeserver address is hardcoded to `` + /// - Run a pkarr Relay on port `15411` + /// + /// # Safety + /// See [Self::start] + pub async unsafe fn start_testnet() -> Result { + let testnet = ::pkarr::mainline::Testnet::new(10)?; + let relay = unsafe { + let mut config = pkarr_relay::Config { + http_port: 15411, + ..Default::default() + }; + config.pkarr_config.dht_config.bootstrap = testnet.bootstrap.clone(); + + pkarr_relay::Relay::start(config).await? + }; + + tracing::info!(relay_address=?relay.relay_address(), bootstrap=?relay.resolver_address(),"Running in Testnet mode"); + + unsafe { + Homeserver::builder() + .keypair(Keypair::from_secret_key(&[0; 32])) + .bootstrap(testnet.bootstrap) + .build() + .await + } + } + + /// Test version of [Homeserver::start], using mainline Testnet, and a temporary storage. + pub async fn start_test(testnet: &::pkarr::mainline::Testnet) -> Result { unsafe { Homeserver::start(Config::test(testnet)).await } } diff --git a/pubky-homeserver/src/io/pkarr.rs b/pubky-homeserver/src/io/pkarr.rs index 7959354..fe46864 100644 --- a/pubky-homeserver/src/io/pkarr.rs +++ b/pubky-homeserver/src/io/pkarr.rs @@ -7,60 +7,68 @@ use crate::Config; pub struct PkarrServer { client: pkarr::Client, - config: Config, + signed_packet: SignedPacket, } impl PkarrServer { - pub fn new(config: Config) -> Result { - let mut dht_settings = pkarr::mainline::Settings::default(); + pub fn new(config: Config, port: u16) -> Result { + let mut dht_config = pkarr::mainline::Config::default(); - if let Some(bootstrap) = config.bootstrap() { - dht_settings = dht_settings.bootstrap(&bootstrap); + if let Some(bootstrap) = config.bootstrap.clone() { + dht_config.bootstrap = bootstrap; } - if let Some(request_timeout) = config.dht_request_timeout() { - dht_settings = dht_settings.request_timeout(request_timeout); + if let Some(request_timeout) = config.dht_request_timeout { + dht_config.request_timeout = request_timeout; } - let client = pkarr::Client::builder() - .dht_settings(dht_settings) - .build()?; + let client = pkarr::Client::builder().dht_config(dht_config).build()?; - Ok(Self { client, config }) + let signed_packet = create_signed_packet(config, port)?; + + Ok(Self { + client, + signed_packet, + }) } - pub async fn publish_server_packet(&self, port: u16) -> anyhow::Result<()> { - // TODO: Try to resolve first before publishing. + pub async fn publish_server_packet(&self) -> anyhow::Result<()> { + // TODO: warn if packet is not most recent, which means the + // user is publishing a Packet from somewhere else. - let default = ".".to_string(); - let target = self.config.domain().unwrap_or(&default); - let mut svcb = SVCB::new(0, target.as_str().try_into()?); - - svcb.priority = 1; - svcb.set_port(port); - - let mut signed_packet_builder = - SignedPacket::builder().https(".".try_into().unwrap(), svcb.clone(), 60 * 60); - - if self.config.domain().is_none() { - // TODO: remove after remvoing Pubky shared/public - // and add local host IP address instead. - svcb.target = "localhost".try_into().unwrap(); - - signed_packet_builder = signed_packet_builder - .https(".".try_into().unwrap(), svcb, 60 * 60) - .address( - ".".try_into().unwrap(), - "127.0.0.1".parse().unwrap(), - 60 * 60, - ); - } - - // TODO: announce A/AAAA records as well for TLS connections? - - let signed_packet = signed_packet_builder.build(self.config.keypair())?; - - self.client.publish(&signed_packet).await?; + self.client.publish(&self.signed_packet).await?; Ok(()) } } + +pub fn create_signed_packet(config: Config, port: u16) -> Result { + // TODO: Try to resolve first before publishing. + + let default = ".".to_string(); + let target = config.domain.clone().unwrap_or(default); + let mut svcb = SVCB::new(0, target.as_str().try_into()?); + + svcb.priority = 1; + svcb.set_port(port); + + let mut signed_packet_builder = + SignedPacket::builder().https(".".try_into().unwrap(), svcb.clone(), 60 * 60); + + if config.domain.is_none() { + // TODO: remove after remvoing Pubky shared/public + // and add local host IP address instead. + svcb.target = "localhost".try_into().unwrap(); + + signed_packet_builder = signed_packet_builder + .https(".".try_into().unwrap(), svcb, 60 * 60) + .address( + ".".try_into().unwrap(), + "127.0.0.1".parse().unwrap(), + 60 * 60, + ); + } + + // TODO: announce A/AAAA records as well for TLS connections? + + Ok(signed_packet_builder.build(&config.keypair)?) +} diff --git a/pubky-homeserver/src/main.rs b/pubky-homeserver/src/main.rs index 9d2363f..3d04429 100644 --- a/pubky-homeserver/src/main.rs +++ b/pubky-homeserver/src/main.rs @@ -32,14 +32,13 @@ async fn main() -> Result<()> { .init(); let server = unsafe { - Homeserver::start(if args.testnet { - Config::testnet() + if args.testnet { + Homeserver::start_testnet().await? } else if let Some(config_path) = args.config { - Config::load(config_path).await? + Homeserver::start(Config::load(config_path).await?).await? } else { - Config::default() - }) - .await? + Homeserver::builder().build().await? + } }; tokio::signal::ctrl_c().await?; diff --git a/pubky/pkg/package.json b/pubky/pkg/package.json index 24894c6..696a280 100644 --- a/pubky/pkg/package.json +++ b/pubky/pkg/package.json @@ -9,7 +9,7 @@ "url": "git+https://github.com/pubky/pubky-core.git" }, "scripts": { - "testnet": "cargo run -p pubky_homeserver -- --testnet", + "testnet": "cargo run -p pubky-homeserver -- --testnet", "test": "npm run test-nodejs && npm run test-browser", "test-nodejs": "tape test/*.js -cov", "test-browser": "browserify test/*.js -p esmify | npx tape-run", diff --git a/pubky/pkg/test/http.js b/pubky/pkg/test/http.js index 6759b9b..1edc295 100644 --- a/pubky/pkg/test/http.js +++ b/pubky/pkg/test/http.js @@ -6,7 +6,7 @@ const TLD = '8pinxxgqs41n4aididenw5apqp1urfmzdztr8jt4abrkdn435ewo'; // TODO: test HTTPs too somehow. -test.skip("basic fetch", async (t) => { +test.only("basic fetch", async (t) => { let client = Client.testnet(); // Normal TLD diff --git a/pubky/src/native.rs b/pubky/src/native.rs index 956f0d5..2d98464 100644 --- a/pubky/src/native.rs +++ b/pubky/src/native.rs @@ -1,4 +1,4 @@ -use std::{sync::Arc, time::Duration}; +use std::{net::ToSocketAddrs, sync::Arc, time::Duration}; use pkarr::mainline::Testnet; @@ -15,13 +15,14 @@ static DEFAULT_USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CAR #[derive(Debug, Default)] pub struct Settings { - pkarr_settings: pkarr::Settings, + pkarr_config: pkarr::Config, } impl Settings { /// Set Pkarr client [pkarr::Settings]. - pub fn pkarr_settings(mut self, settings: pkarr::Settings) -> Self { - self.pkarr_settings = settings; + pub fn pkarr_config(mut self, settings: pkarr::Config) -> Self { + self.pkarr_config = settings; + self } @@ -32,23 +33,26 @@ impl Settings { pub fn testnet(mut self, testnet: &Testnet) -> Self { let bootstrap = testnet.bootstrap.clone(); - let mut dht_settings = pkarr::mainline::Settings::default().bootstrap(&bootstrap); + self.pkarr_config.resolvers = Some( + bootstrap + .iter() + .flat_map(|resolver| resolver.to_socket_addrs()) + .flatten() + .collect::>(), + ); + + self.pkarr_config.dht_config.bootstrap = bootstrap; if std::env::var("CI").is_err() { - dht_settings = dht_settings.request_timeout(Duration::from_millis(500)); + self.pkarr_config.dht_config.request_timeout = Duration::from_millis(500); } - self.pkarr_settings = self - .pkarr_settings - .dht_settings(dht_settings) - .resolvers(Some(bootstrap)); - self } /// Build [Client] pub fn build(self) -> Result { - let pkarr = pkarr::Client::new(self.pkarr_settings)?; + let pkarr = pkarr::Client::new(self.pkarr_config)?; let cookie_store = Arc::new(CookieJar::default());