diff --git a/Cargo.lock b/Cargo.lock index 0a0eebb..8087df4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -157,15 +157,15 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "axum" -version = "0.7.6" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f43644eed690f5374f1af436ecd6aea01cd201f6fbdf0178adaf6907afb2cec" +checksum = "504e3947307ac8326a5437504c517c4b56716c9d98fac0028c2acc7ca47d70ae" dependencies = [ "async-trait", "axum-core", @@ -198,9 +198,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6b8ba012a258d63c9adfa28b9ddcf66149da6f986c5b5452e629d5ee64bf00" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" dependencies = [ "async-trait", "bytes", @@ -361,9 +361,9 @@ checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" [[package]] name = "cc" -version = "1.1.21" +version = "1.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0" +checksum = "3bbb537bb4a30b90362caddba8f360c0a56bc13d3a5570028e7197204cb54a17" dependencies = [ "shlex", ] @@ -1066,9 +1066,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.4" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" [[package]] name = "httpdate" @@ -1285,9 +1285,8 @@ checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904" [[package]] name = "mainline" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b751ffb57303217bcae8f490eee6044a5b40eadf6ca05ff476cad37e7b7970d" +version = "3.0.0" +source = "git+https://github.com/pubky/mainline?branch=dev#47edb4617f3ce25883b889064de5c7d257848b8f" dependencies = [ "bytes", "crc", @@ -1404,9 +1403,12 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1" +dependencies = [ + "portable-atomic", +] [[package]] name = "opaque-debug" @@ -1571,6 +1573,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkarr" version = "3.0.0" +source = "git+https://github.com/Pubky/pkarr?branch=v3#0622003a851dfc34c4c3b44daaf13c99880d300a" dependencies = [ "base32", "bytes", @@ -1621,6 +1624,12 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "portable-atomic" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" + [[package]] name = "postcard" version = "1.0.10" @@ -1670,7 +1679,6 @@ version = "0.1.0" dependencies = [ "base64 0.22.1", "bytes", - "hyper-util", "js-sys", "pkarr", "pubky-common", @@ -1828,9 +1836,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.5" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62871f2d65009c0256aed1b9cfeeb8ac272833c404e13d53d400cd0dad7a2ac0" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" dependencies = [ "bitflags", ] @@ -1848,14 +1856,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.6" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.7", - "regex-syntax 0.8.4", + "regex-automata 0.4.8", + "regex-syntax 0.8.5", ] [[package]] @@ -1869,13 +1877,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] @@ -1886,15 +1894,15 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.7" +version = "0.12.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" +checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b" dependencies = [ "base64 0.22.1", "bytes", @@ -2026,19 +2034,18 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.3" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "base64 0.22.1", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" +checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55" [[package]] name = "rustls-webpki" @@ -2185,9 +2192,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" dependencies = [ "serde", ] @@ -2344,9 +2351,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.77" +version = "2.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" dependencies = [ "proc-macro2", "quote", @@ -2400,9 +2407,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" +checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" dependencies = [ "cfg-if", "fastrand", @@ -3095,9 +3102,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.19" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c52ac009d615e79296318c1bcce2d422aaca15ad08515e344feeda07df67a587" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" dependencies = [ "memchr", ] diff --git a/docs/src/spec/auth.md b/docs/src/spec/auth.md index 62c57bf..92a3041 100644 --- a/docs/src/spec/auth.md +++ b/docs/src/spec/auth.md @@ -79,7 +79,7 @@ pubkyauth:/// ```abnf AuthToken = signature namespace version timestamp pubky capabilities -signature = 64OCTET ; ed25519 signature over encoded DNS packet +signature = 64OCTET ; ed25519 signature over the rest of the token. namespace = %x50.55.42.4b.59.3a.41.55.54.48 ; "PUBKY:AUTH" in UTF-8 (10 bytes) version = 1*OCTET ; Version of the AuthToken for future proofing. timestamp = 8OCTET ; Big-endian UNIX timestamp in microseconds diff --git a/pubky-common/src/session.rs b/pubky-common/src/session.rs index 5ce64d0..972652c 100644 --- a/pubky-common/src/session.rs +++ b/pubky-common/src/session.rs @@ -68,6 +68,10 @@ impl Session { } pub fn deserialize(bytes: &[u8]) -> Result { + if bytes.is_empty() { + return Err(Error::EmptyPayload); + } + if bytes[0] > 0 { return Err(Error::UnknownVersion); } @@ -80,8 +84,10 @@ impl Session { pub type Result = core::result::Result; -#[derive(thiserror::Error, Debug)] +#[derive(thiserror::Error, Debug, PartialEq)] pub enum Error { + #[error("Empty payload")] + EmptyPayload, #[error("Unknown version")] UnknownVersion, #[error(transparent)] @@ -123,4 +129,11 @@ mod tests { assert_eq!(deseiralized, session) } + + #[test] + fn deserialize() { + let result = Session::deserialize(&[]); + + assert_eq!(result, Err(Error::EmptyPayload)); + } } diff --git a/pubky-homeserver/src/config.rs b/pubky-homeserver/src/config.rs index b855111..ba699b7 100644 --- a/pubky-homeserver/src/config.rs +++ b/pubky-homeserver/src/config.rs @@ -2,7 +2,7 @@ use anyhow::{anyhow, Context, Result}; use pkarr::Keypair; -use serde::{Deserialize, Deserializer, Serialize}; +use serde::{Deserialize, Serialize}; use std::{ fmt::Debug, path::{Path, PathBuf}, @@ -12,34 +12,102 @@ use tracing::info; use pubky_common::timestamp::Timestamp; -const DEFAULT_HOMESERVER_PORT: u16 = 6287; +// === Database === const DEFAULT_STORAGE_DIR: &str = "pubky"; +pub const DEFAULT_MAP_SIZE: usize = 10995116277760; // 10TB (not = disk-space used) -/// Server configuration -#[derive(Serialize, Deserialize, Clone)] -pub struct Config { - testnet: bool, +// === Server == +pub const DEFAULT_LIST_LIMIT: u16 = 100; +pub const DEFAULT_MAX_LIST_LIMIT: u16 = 1000; + +#[derive(Serialize, Deserialize, Clone, PartialEq)] +struct ConfigToml { + testnet: Option, port: Option, bootstrap: Option>, domain: Option, - /// Path to the storage directory + storage: Option, + secret_key: Option, + dht_request_timeout: Option, + default_list_limit: Option, + max_list_limit: Option, + db_map_size: Option, +} + +/// Server configuration +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Config { + /// Whether or not this server is running in a testnet. + testnet: bool, + /// The configured port for this server. + port: u16, + /// Bootstrapping DHT nodes. + /// + /// Helpful to run the server locally or in testnet. + bootstrap: Option>, + /// A public domain for this server + /// necessary for web browsers running in https environment. + domain: Option, + /// Path to the storage directory. /// /// Defaults to a directory in the OS data directory - storage: Option, - #[serde(deserialize_with = "secret_key_deserialize")] - secret_key: Option<[u8; 32]>, - + storage: PathBuf, + /// Server keypair. + /// + /// Defaults to a random keypair. + keypair: Keypair, dht_request_timeout: Option, + /// The default limit of a list api if no `limit` query parameter is provided. + /// + /// Defaults to `100` + default_list_limit: u16, + /// The maximum limit of a list api, even if a `limit` query parameter is provided. + /// + /// Defaults to `1000` + max_list_limit: u16, + + // === Database params === + db_map_size: usize, } impl Config { - /// Load the config from a file. - pub async fn load(path: impl AsRef) -> Result { - let s = tokio::fs::read_to_string(path.as_ref()) - .await - .with_context(|| format!("failed to read {}", path.as_ref().to_string_lossy()))?; + fn try_from_str(value: &str) -> Result { + let config_toml: ConfigToml = toml::from_str(value)?; - let config: Config = toml::from_str(&s)?; + let keypair = if let Some(secret_key) = config_toml.secret_key { + let secret_key = deserialize_secret_key(secret_key)?; + Keypair::from_secret_key(&secret_key) + } else { + Keypair::random() + }; + + let storage = { + let dir = if let Some(storage) = config_toml.storage { + storage + } else { + let path = dirs_next::data_dir().ok_or_else(|| { + anyhow!("operating environment provides no directory for application data") + })?; + path.join(DEFAULT_STORAGE_DIR) + }; + + dir.join("homeserver") + }; + + let config = Config { + testnet: config_toml.testnet.unwrap_or(false), + port: config_toml.port.unwrap_or(0), + bootstrap: config_toml.bootstrap, + domain: config_toml.domain, + keypair, + storage, + dht_request_timeout: config_toml.dht_request_timeout, + default_list_limit: config_toml.default_list_limit.unwrap_or(DEFAULT_LIST_LIMIT), + max_list_limit: config_toml + .default_list_limit + .unwrap_or(DEFAULT_MAX_LIST_LIMIT), + db_map_size: config_toml.db_map_size.unwrap_or(DEFAULT_MAP_SIZE), + }; if config.testnet { let testnet_config = Config::testnet(); @@ -53,45 +121,47 @@ impl Config { Ok(config) } + /// Load the config from a file. + pub async fn load(path: impl AsRef) -> Result { + let s = tokio::fs::read_to_string(path.as_ref()) + .await + .with_context(|| format!("failed to read {}", path.as_ref().to_string_lossy()))?; + + Config::try_from_str(&s) + } + /// Testnet configurations pub fn testnet() -> Self { - let testnet = pkarr::mainline::Testnet::new(10); + let testnet = pkarr::mainline::Testnet::new(10).unwrap(); info!(?testnet.bootstrap, "Testnet bootstrap nodes"); - let bootstrap = Some(testnet.bootstrap.to_owned()); - let storage = Some( - std::env::temp_dir() - .join(Timestamp::now().to_string()) - .join(DEFAULT_STORAGE_DIR), - ); - - Self { - bootstrap, - storage, - port: Some(15411), - dht_request_timeout: Some(Duration::from_millis(10)), - ..Default::default() + Config { + port: 15411, + dht_request_timeout: None, + db_map_size: DEFAULT_MAP_SIZE, + ..Self::test(&testnet) } } /// Test configurations pub fn test(testnet: &pkarr::mainline::Testnet) -> Self { let bootstrap = Some(testnet.bootstrap.to_owned()); - let storage = Some( - std::env::temp_dir() - .join(Timestamp::now().to_string()) - .join(DEFAULT_STORAGE_DIR), - ); + let storage = std::env::temp_dir() + .join(Timestamp::now().to_string()) + .join(DEFAULT_STORAGE_DIR); Self { + testnet: true, bootstrap, storage, + db_map_size: 10485760, + dht_request_timeout: Some(Duration::from_millis(10)), ..Default::default() } } pub fn port(&self) -> u16 { - self.port.unwrap_or(DEFAULT_HOMESERVER_PORT) + self.port } pub fn bootstsrap(&self) -> Option> { @@ -102,73 +172,134 @@ impl Config { self.domain.as_ref() } - /// Get the path to the storage directory - pub fn storage(&self) -> Result { - let dir = if let Some(storage) = &self.storage { - PathBuf::from(storage) - } else { - let path = dirs_next::data_dir().ok_or_else(|| { - anyhow!("operating environment provides no directory for application data") - })?; - path.join(DEFAULT_STORAGE_DIR) - }; - - Ok(dir.join("homeserver")) + pub fn keypair(&self) -> &Keypair { + &self.keypair } - pub fn keypair(&self) -> Keypair { - Keypair::from_secret_key(&self.secret_key.unwrap_or_default()) + pub fn default_list_limit(&self) -> u16 { + self.default_list_limit + } + + pub fn max_list_limit(&self) -> u16 { + self.max_list_limit + } + + /// Get the path to the storage directory + pub fn storage(&self) -> &PathBuf { + &self.storage } pub(crate) fn dht_request_timeout(&self) -> Option { self.dht_request_timeout } + + pub(crate) fn db_map_size(&self) -> usize { + self.db_map_size + } } impl Default for Config { fn default() -> Self { Self { testnet: false, - port: Some(0), + port: 0, bootstrap: None, domain: None, - storage: None, - secret_key: None, + storage: storage(None) + .expect("operating environment provides no directory for application data"), + keypair: Keypair::random(), dht_request_timeout: None, + default_list_limit: DEFAULT_LIST_LIMIT, + max_list_limit: DEFAULT_MAX_LIST_LIMIT, + db_map_size: DEFAULT_MAP_SIZE, } } } -fn secret_key_deserialize<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - let opt: Option = Option::deserialize(deserializer)?; +fn deserialize_secret_key(s: String) -> anyhow::Result<[u8; 32]> { + let bytes = + hex::decode(s).map_err(|_| anyhow!("secret_key in config.toml should hex encoded"))?; - match opt { - Some(s) => { - let bytes = hex::decode(s).map_err(serde::de::Error::custom)?; + if bytes.len() != 32 { + return Err(anyhow!(format!( + "secret_key in config.toml should be 32 bytes in hex (64 characters), got: {}", + bytes.len() + ))); + } - if bytes.len() != 32 { - return Err(serde::de::Error::custom("Expected a 32-byte array")); + let mut arr = [0u8; 32]; + arr.copy_from_slice(&bytes); + + Ok(arr) +} + +fn storage(storage: Option) -> Result { + let dir = if let Some(storage) = storage { + PathBuf::from(storage) + } else { + let path = dirs_next::data_dir().ok_or_else(|| { + anyhow!("operating environment provides no directory for application data") + })?; + path.join(DEFAULT_STORAGE_DIR) + }; + + Ok(dir.join("homeserver")) +} + +#[cfg(test)] +mod tests { + use pkarr::mainline::Testnet; + + use super::*; + + #[test] + fn parse_empty() { + let config = Config::try_from_str("").unwrap(); + + assert_eq!( + config, + Config { + keypair: config.keypair.clone(), + ..Default::default() } + ) + } - let mut arr = [0u8; 32]; - arr.copy_from_slice(&bytes); - Ok(Some(arr)) - } - None => Ok(None), - } -} - -impl Debug for Config { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_map() - .entry(&"testnet", &self.testnet) - .entry(&"port", &self.port()) - .entry(&"storage", &self.storage()) - .entry(&"public_key", &self.keypair().public_key()) - .entry(&"domain", &self.domain()) - .finish() + #[test] + fn config_test() { + let testnet = Testnet::new(3).unwrap(); + let config = Config::test(&testnet); + + assert_eq!( + config, + Config { + testnet: true, + bootstrap: testnet.bootstrap.into(), + db_map_size: 10485760, + dht_request_timeout: Some(Duration::from_millis(10)), + + storage: config.storage.clone(), + keypair: config.keypair.clone(), + ..Default::default() + } + ) + } + + #[test] + fn config_testnet() { + let config = Config::testnet(); + + assert_eq!( + config, + Config { + testnet: true, + port: 15411, + + bootstrap: config.bootstrap.clone(), + storage: config.storage.clone(), + keypair: config.keypair.clone(), + ..Default::default() + } + ) } } diff --git a/pubky-homeserver/src/database.rs b/pubky-homeserver/src/database.rs index 4adc73d..7f52e16 100644 --- a/pubky-homeserver/src/database.rs +++ b/pubky-homeserver/src/database.rs @@ -1,31 +1,39 @@ use std::fs; -use std::path::Path; - use heed::{Env, EnvOpenOptions}; mod migrations; pub mod tables; -use tables::{Tables, TABLES_COUNT}; +use crate::config::Config; -pub const MAX_LIST_LIMIT: u16 = 100; +use tables::{Tables, TABLES_COUNT}; #[derive(Debug, Clone)] pub struct DB { pub(crate) env: Env, pub(crate) tables: Tables, + pub(crate) config: Config, } impl DB { - pub fn open(storage: &Path) -> anyhow::Result { - fs::create_dir_all(storage).unwrap(); + pub fn open(config: Config) -> anyhow::Result { + fs::create_dir_all(config.storage())?; - let env = unsafe { EnvOpenOptions::new().max_dbs(TABLES_COUNT).open(storage) }?; + let env = unsafe { + EnvOpenOptions::new() + .max_dbs(TABLES_COUNT) + .map_size(config.db_map_size()) + .open(config.storage()) + }?; let tables = migrations::run(&env)?; - let db = DB { env, tables }; + let db = DB { + env, + tables, + config, + }; Ok(db) } @@ -34,18 +42,15 @@ impl DB { #[cfg(test)] mod tests { use bytes::Bytes; - use pkarr::Keypair; - use pubky_common::timestamp::Timestamp; + use pkarr::{mainline::Testnet, Keypair}; + + use crate::config::Config; use super::DB; #[tokio::test] async fn entries() { - let storage = std::env::temp_dir() - .join(Timestamp::now().to_string()) - .join("pubky"); - - let db = DB::open(&storage).unwrap(); + let db = DB::open(Config::test(&Testnet::new(0).unwrap())).unwrap(); let keypair = Keypair::random(); let path = "/pub/foo.txt"; diff --git a/pubky-homeserver/src/database/tables/entries.rs b/pubky-homeserver/src/database/tables/entries.rs index 081f606..b1c7039 100644 --- a/pubky-homeserver/src/database/tables/entries.rs +++ b/pubky-homeserver/src/database/tables/entries.rs @@ -13,7 +13,7 @@ use pubky_common::{ timestamp::Timestamp, }; -use crate::database::{DB, MAX_LIST_LIMIT}; +use crate::database::DB; use super::events::Event; @@ -157,7 +157,7 @@ impl DB { /// Return a list of pubky urls. /// - /// - limit defaults to and capped by [MAX_LIST_LIMIT] + /// - limit defaults to [Config::default_list_limit] and capped by [Config::max_list_limit] pub fn list( &self, txn: &RoTxn, @@ -170,7 +170,9 @@ impl DB { // Vector to store results let mut results = Vec::new(); - let limit = limit.unwrap_or(MAX_LIST_LIMIT).min(MAX_LIST_LIMIT); + let limit = limit + .unwrap_or(self.config.default_list_limit()) + .min(self.config.max_list_limit()); // TODO: make this more performant than split and allocations? diff --git a/pubky-homeserver/src/database/tables/events.rs b/pubky-homeserver/src/database/tables/events.rs index cf82e18..76a4d46 100644 --- a/pubky-homeserver/src/database/tables/events.rs +++ b/pubky-homeserver/src/database/tables/events.rs @@ -10,6 +10,8 @@ use heed::{ use postcard::{from_bytes, to_allocvec}; use serde::{Deserialize, Serialize}; +use crate::database::DB; + /// Event [Timestamp] base32 => Encoded event. pub type EventsTable = Database; @@ -56,3 +58,48 @@ impl Event { } } } + +impl DB { + /// Returns a list of events formatted as ` `. + /// + /// - limit defaults to [Config::default_list_limit] and capped by [Config::max_list_limit] + /// - cursor is a 13 character string encoding of a timestamp + pub fn list_events( + &self, + limit: Option, + cursor: Option, + ) -> anyhow::Result> { + let txn = self.env.read_txn()?; + + let limit = limit + .unwrap_or(self.config.default_list_limit()) + .min(self.config.max_list_limit()); + + let cursor = cursor.unwrap_or("0000000000000".to_string()); + + let mut result: Vec = vec![]; + let mut next_cursor = cursor.to_string(); + + for _ in 0..limit { + match self.tables.events.get_greater_than(&txn, &next_cursor)? { + Some((timestamp, event_bytes)) => { + let event = Event::deserialize(event_bytes)?; + + let line = format!("{} {}", event.operation(), event.url()); + next_cursor = timestamp.to_string(); + + result.push(line); + } + None => break, + }; + } + + if !result.is_empty() { + result.push(format!("cursor: {next_cursor}")) + } + + txn.commit()?; + + Ok(result) + } +} diff --git a/pubky-homeserver/src/error.rs b/pubky-homeserver/src/error.rs index b6e5a14..8aa58d2 100644 --- a/pubky-homeserver/src/error.rs +++ b/pubky-homeserver/src/error.rs @@ -5,6 +5,7 @@ use axum::{ http::StatusCode, response::IntoResponse, }; +use tracing::debug; pub type Result = core::result::Result; @@ -86,36 +87,42 @@ impl From for Error { impl From for Error { fn from(error: std::io::Error) -> Self { + debug!(?error); Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into()) } } impl From for Error { fn from(error: heed::Error) -> Self { + debug!(?error); Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into()) } } impl From for Error { fn from(error: anyhow::Error) -> Self { + debug!(?error); Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into()) } } impl From for Error { fn from(error: postcard::Error) -> Self { + debug!(?error); Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into()) } } impl From for Error { fn from(error: axum::Error) -> Self { + debug!(?error); Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into()) } } impl From> for Error { fn from(error: flume::SendError) -> Self { + debug!(?error); Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into()) } } diff --git a/pubky-homeserver/src/extractors.rs b/pubky-homeserver/src/extractors.rs index 567ca6b..779ce65 100644 --- a/pubky-homeserver/src/extractors.rs +++ b/pubky-homeserver/src/extractors.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use axum::{ async_trait, - extract::{FromRequestParts, Path}, + extract::{FromRequestParts, Path, Query}, http::{request::Parts, StatusCode}, response::{IntoResponse, Response}, RequestPartsExt, @@ -74,3 +74,50 @@ where Ok(EntryPath(path.to_string())) } } + +#[derive(Debug)] +pub struct ListQueryParams { + pub limit: Option, + pub cursor: Option, + pub reverse: bool, + pub shallow: bool, +} + +#[async_trait] +impl FromRequestParts for ListQueryParams +where + S: Send + Sync, +{ + type Rejection = Response; + + async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result { + let params: Query> = + parts.extract().await.map_err(IntoResponse::into_response)?; + + let reverse = params.contains_key("reverse"); + let shallow = params.contains_key("shallow"); + let limit = params + .get("limit") + // Treat `limit=` as None + .and_then(|l| if l.is_empty() { None } else { Some(l) }) + .and_then(|l| l.parse::().ok()); + let cursor = params + .get("cursor") + .map(|c| c.as_str()) + // Treat `cursor=` as None + .and_then(|c| { + if c.is_empty() { + None + } else { + Some(c.to_string()) + } + }); + + Ok(ListQueryParams { + reverse, + shallow, + limit, + cursor, + }) + } +} diff --git a/pubky-homeserver/src/pkarr.rs b/pubky-homeserver/src/pkarr.rs index 01cb56b..26c6b66 100644 --- a/pubky-homeserver/src/pkarr.rs +++ b/pubky-homeserver/src/pkarr.rs @@ -7,13 +7,14 @@ use pkarr::{ rdata::{RData, A, SVCB}, Packet, }, - Keypair, SignedPacket, + SignedPacket, }; +use crate::config::Config; + pub async fn publish_server_packet( - pkarr_client: pkarr::Client, - keypair: &Keypair, - domain: Option<&String>, + pkarr_client: &pkarr::Client, + config: &Config, port: u16, ) -> anyhow::Result<()> { // TODO: Try to resolve first before publishing. @@ -21,7 +22,7 @@ pub async fn publish_server_packet( let mut packet = Packet::new_reply(0); let default = ".".to_string(); - let target = domain.unwrap_or(&default); + let target = config.domain().unwrap_or(&default); let mut svcb = SVCB::new(0, target.as_str().try_into()?); svcb.priority = 1; @@ -34,7 +35,7 @@ pub async fn publish_server_packet( RData::HTTPS(svcb.clone().into()), )); - if domain.is_none() { + if config.domain().is_none() { // TODO: remove after remvoing Pubky shared/public // and add local host IP address instead. svcb.target = "localhost".try_into().unwrap(); @@ -56,7 +57,7 @@ pub async fn publish_server_packet( // TODO: announce A/AAAA records as well for TLS connections? - let signed_packet = SignedPacket::from_packet(keypair, &packet)?; + let signed_packet = SignedPacket::from_packet(config.keypair(), &packet)?; pkarr_client.publish(&signed_packet).await?; diff --git a/pubky-homeserver/src/routes/feed.rs b/pubky-homeserver/src/routes/feed.rs index bd426f3..6271aeb 100644 --- a/pubky-homeserver/src/routes/feed.rs +++ b/pubky-homeserver/src/routes/feed.rs @@ -1,67 +1,37 @@ -use std::collections::HashMap; - use axum::{ body::Body, - extract::{Query, State}, + extract::State, http::{header, Response, StatusCode}, response::IntoResponse, }; +use pubky_common::timestamp::{Timestamp, TimestampError}; use crate::{ - database::{tables::events::Event, MAX_LIST_LIMIT}, - error::Result, + error::{Error, Result}, + extractors::ListQueryParams, server::AppState, }; pub async fn feed( State(state): State, - Query(params): Query>, + params: ListQueryParams, ) -> Result { - let txn = state.db.env.read_txn()?; + if let Some(ref cursor) = params.cursor { + if let Err(timestmap_error) = Timestamp::try_from(cursor.to_string()) { + let cause = match timestmap_error { + TimestampError::InvalidEncoding => { + "Cursor should be valid base32 Crockford encoding of a timestamp" + } + TimestampError::InvalidBytesLength(size) => { + &format!("Cursor should be 13 characters long, got: {size}") + } + }; - let limit = params - .get("limit") - .and_then(|l| l.parse::().ok()) - .unwrap_or(MAX_LIST_LIMIT) - .min(MAX_LIST_LIMIT); - - let mut cursor = params - .get("cursor") - .map(|c| c.as_str()) - .unwrap_or("0000000000000"); - - // Guard against bad cursor - if cursor.len() < 13 { - cursor = "0000000000000" + Err(Error::new(StatusCode::BAD_REQUEST, cause.into()))? + } } - let mut result: Vec = vec![]; - let mut next_cursor = cursor.to_string(); - - for _ in 0..limit { - match state - .db - .tables - .events - .get_greater_than(&txn, &next_cursor)? - { - Some((timestamp, event_bytes)) => { - let event = Event::deserialize(event_bytes)?; - - let line = format!("{} {}", event.operation(), event.url()); - next_cursor = timestamp.to_string(); - - result.push(line); - } - None => break, - }; - } - - if !result.is_empty() { - result.push(format!("cursor: {next_cursor}")) - } - - txn.commit()?; + let result = state.db.list_events(params.limit, params.cursor)?; Ok(Response::builder() .status(StatusCode::OK) diff --git a/pubky-homeserver/src/routes/public.rs b/pubky-homeserver/src/routes/public.rs index 4cf2eed..8c6b2b9 100644 --- a/pubky-homeserver/src/routes/public.rs +++ b/pubky-homeserver/src/routes/public.rs @@ -1,8 +1,6 @@ -use std::collections::HashMap; - use axum::{ body::{Body, Bytes}, - extract::{Query, State}, + extract::State, http::{header, Response, StatusCode}, response::IntoResponse, }; @@ -12,7 +10,7 @@ use tower_cookies::Cookies; use crate::{ error::{Error, Result}, - extractors::{EntryPath, Pubky}, + extractors::{EntryPath, ListQueryParams, Pubky}, server::AppState, }; @@ -65,7 +63,7 @@ pub async fn get( State(state): State, pubky: Pubky, path: EntryPath, - Query(params): Query>, + params: ListQueryParams, ) -> Result { verify(path.as_str())?; let public_key = pubky.public_key(); @@ -88,10 +86,10 @@ pub async fn get( let vec = state.db.list( &txn, &path, - params.contains_key("reverse"), - params.get("limit").and_then(|l| l.parse::().ok()), - params.get("cursor").map(|cursor| cursor.into()), - params.contains_key("shallow"), + params.reverse, + params.limit, + params.cursor, + params.shallow, )?; return Ok(Response::builder() diff --git a/pubky-homeserver/src/server.rs b/pubky-homeserver/src/server.rs index 7866c32..15c4b3f 100644 --- a/pubky-homeserver/src/server.rs +++ b/pubky-homeserver/src/server.rs @@ -14,25 +14,24 @@ use crate::{config::Config, database::DB, pkarr::publish_server_packet}; #[derive(Debug)] pub struct Homeserver { - port: u16, - config: Config, + state: AppState, tasks: JoinSet>, } #[derive(Clone, Debug)] pub(crate) struct AppState { - pub verifier: AuthVerifier, - pub db: DB, - pub pkarr_client: pkarr::Client, + pub(crate) verifier: AuthVerifier, + pub(crate) db: DB, + pub(crate) pkarr_client: pkarr::Client, + pub(crate) config: Config, + pub(crate) port: u16, } impl Homeserver { pub async fn start(config: Config) -> Result { debug!(?config); - let keypair = config.keypair(); - - let db = DB::open(&config.storage()?)?; + let db = DB::open(config.clone())?; let pkarr_client = pkarr::Client::new(Settings { dht: DhtSettings { @@ -43,22 +42,22 @@ impl Homeserver { ..Default::default() })?; - let state = AppState { - verifier: AuthVerifier::default(), - db, - pkarr_client: pkarr_client.clone(), - }; - - let app = crate::routes::create_app(state); - let mut tasks = JoinSet::new(); - let app = app.clone(); - let listener = TcpListener::bind(SocketAddr::from(([0, 0, 0, 0], config.port()))).await?; let port = listener.local_addr()?.port(); + let state = AppState { + verifier: AuthVerifier::default(), + db, + pkarr_client: pkarr_client.clone(), + config: config.clone(), + port, + }; + + let app = crate::routes::create_app(state.clone()); + // Spawn http server task tasks.spawn( axum::serve( @@ -71,15 +70,14 @@ impl Homeserver { info!("Homeserver listening on http://localhost:{port}"); - publish_server_packet(pkarr_client, &keypair, config.domain(), port).await?; + publish_server_packet(&pkarr_client, &config, port).await?; - info!("Homeserver listening on http://{}", keypair.public_key()); + info!( + "Homeserver listening on http://{}", + config.keypair().public_key() + ); - Ok(Self { - tasks, - config, - port, - }) + Ok(Self { tasks, state }) } /// Test version of [Homeserver::start], using mainline Testnet, and a temporary storage. @@ -92,11 +90,11 @@ impl Homeserver { // === Getters === pub fn port(&self) -> u16 { - self.port + self.state.port } pub fn public_key(&self) -> PublicKey { - self.config.keypair().public_key() + self.state.config.keypair().public_key() } // === Public Methods === diff --git a/pubky/Cargo.toml b/pubky/Cargo.toml index 6905920..31d15d2 100644 --- a/pubky/Cargo.toml +++ b/pubky/Cargo.toml @@ -18,7 +18,7 @@ bytes = "^1.7.1" base64 = "0.22.1" pubky-common = { version = "0.1.0", path = "../pubky-common" } -pkarr = { workspace = true } +pkarr = { workspace = true, features = ["endpoints"] } # Native dependencies [target.'cfg(not(target_arch = "wasm32"))'.dependencies] diff --git a/pubky/src/native/api/http.rs b/pubky/src/native/api/http.rs index ada724b..8ba46d5 100644 --- a/pubky/src/native/api/http.rs +++ b/pubky/src/native/api/http.rs @@ -29,7 +29,7 @@ mod tests { #[tokio::test] async fn http_get_pubky() { - let testnet = Testnet::new(10); + let testnet = Testnet::new(10).unwrap(); let homeserver = Homeserver::start_test(&testnet).await.unwrap(); @@ -48,7 +48,7 @@ mod tests { #[tokio::test] async fn http_get_icann() { - let testnet = Testnet::new(10); + let testnet = Testnet::new(10).unwrap(); let client = PubkyClient::builder().testnet(&testnet).build(); diff --git a/pubky/src/shared/auth.rs b/pubky/src/shared/auth.rs index 5be14d9..c545adc 100644 --- a/pubky/src/shared/auth.rs +++ b/pubky/src/shared/auth.rs @@ -235,7 +235,7 @@ mod tests { #[tokio::test] async fn basic_authn() { - let testnet = Testnet::new(10); + let testnet = Testnet::new(10).unwrap(); let server = Homeserver::start_test(&testnet).await.unwrap(); let client = PubkyClient::test(&testnet); @@ -276,7 +276,7 @@ mod tests { #[tokio::test] async fn authz() { - let testnet = Testnet::new(10); + let testnet = Testnet::new(10).unwrap(); let server = Homeserver::start_test(&testnet).await.unwrap(); let keypair = Keypair::random(); diff --git a/pubky/src/shared/pkarr.rs b/pubky/src/shared/pkarr.rs index f099932..9e2fba7 100644 --- a/pubky/src/shared/pkarr.rs +++ b/pubky/src/shared/pkarr.rs @@ -193,7 +193,7 @@ mod tests { #[tokio::test] async fn resolve_endpoint_https() { - let testnet = Testnet::new(10); + let testnet = Testnet::new(10).unwrap(); let pkarr_client = pkarr::Client::builder().testnet(&testnet).build().unwrap(); @@ -274,7 +274,7 @@ mod tests { #[tokio::test] async fn resolve_homeserver() { - let testnet = Testnet::new(10); + let testnet = Testnet::new(10).unwrap(); let server = Homeserver::start_test(&testnet).await.unwrap(); // Publish an intermediate controller of the homeserver diff --git a/pubky/src/shared/public.rs b/pubky/src/shared/public.rs index ada987d..6ded72a 100644 --- a/pubky/src/shared/public.rs +++ b/pubky/src/shared/public.rs @@ -105,7 +105,7 @@ mod tests { #[tokio::test] async fn put_get_delete() { - let testnet = Testnet::new(10); + let testnet = Testnet::new(10).unwrap(); let server = Homeserver::start_test(&testnet).await.unwrap(); let client = PubkyClient::test(&testnet); @@ -132,7 +132,7 @@ mod tests { #[tokio::test] async fn unauthorized_put_delete() { - let testnet = Testnet::new(10); + let testnet = Testnet::new(10).unwrap(); let server = Homeserver::start_test(&testnet).await.unwrap(); let client = PubkyClient::test(&testnet); @@ -198,7 +198,7 @@ mod tests { #[tokio::test] async fn list() { - let testnet = Testnet::new(10); + let testnet = Testnet::new(10).unwrap(); let server = Homeserver::start_test(&testnet).await.unwrap(); let client = PubkyClient::test(&testnet); @@ -402,7 +402,7 @@ mod tests { #[tokio::test] async fn list_shallow() { - let testnet = Testnet::new(10); + let testnet = Testnet::new(10).unwrap(); let server = Homeserver::start_test(&testnet).await.unwrap(); let client = PubkyClient::test(&testnet); @@ -613,7 +613,7 @@ mod tests { #[tokio::test] async fn list_events() { - let testnet = Testnet::new(10); + let testnet = Testnet::new(10).unwrap(); let server = Homeserver::start_test(&testnet).await.unwrap(); let client = PubkyClient::test(&testnet); @@ -710,7 +710,7 @@ mod tests { #[tokio::test] async fn read_after_event() { - let testnet = Testnet::new(10); + let testnet = Testnet::new(10).unwrap(); let server = Homeserver::start_test(&testnet).await.unwrap(); let client = PubkyClient::test(&testnet); @@ -758,7 +758,7 @@ mod tests { #[tokio::test] async fn dont_delete_shared_blobs() { - let testnet = Testnet::new(10); + let testnet = Testnet::new(10).unwrap(); let homeserver = Homeserver::start_test(&testnet).await.unwrap(); let client = PubkyClient::test(&testnet);