Merge branch 'dev' of https://github.com/pubky/pubky into feat/pkarr-reqwest

This commit is contained in:
nazeh
2024-10-01 08:34:43 +03:00
18 changed files with 475 additions and 249 deletions

91
Cargo.lock generated
View File

@@ -157,15 +157,15 @@ dependencies = [
[[package]]
name = "autocfg"
version = "1.3.0"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"
checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
[[package]]
name = "axum"
version = "0.7.6"
version = "0.7.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f43644eed690f5374f1af436ecd6aea01cd201f6fbdf0178adaf6907afb2cec"
checksum = "504e3947307ac8326a5437504c517c4b56716c9d98fac0028c2acc7ca47d70ae"
dependencies = [
"async-trait",
"axum-core",
@@ -198,9 +198,9 @@ dependencies = [
[[package]]
name = "axum-core"
version = "0.4.4"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e6b8ba012a258d63c9adfa28b9ddcf66149da6f986c5b5452e629d5ee64bf00"
checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199"
dependencies = [
"async-trait",
"bytes",
@@ -361,9 +361,9 @@ checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3"
[[package]]
name = "cc"
version = "1.1.21"
version = "1.1.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0"
checksum = "3bbb537bb4a30b90362caddba8f360c0a56bc13d3a5570028e7197204cb54a17"
dependencies = [
"shlex",
]
@@ -1066,9 +1066,9 @@ dependencies = [
[[package]]
name = "httparse"
version = "1.9.4"
version = "1.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9"
checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946"
[[package]]
name = "httpdate"
@@ -1285,9 +1285,8 @@ checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904"
[[package]]
name = "mainline"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b751ffb57303217bcae8f490eee6044a5b40eadf6ca05ff476cad37e7b7970d"
version = "3.0.0"
source = "git+https://github.com/pubky/mainline?branch=dev#47edb4617f3ce25883b889064de5c7d257848b8f"
dependencies = [
"bytes",
"crc",
@@ -1404,9 +1403,12 @@ dependencies = [
[[package]]
name = "once_cell"
version = "1.19.0"
version = "1.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1"
dependencies = [
"portable-atomic",
]
[[package]]
name = "opaque-debug"
@@ -1571,6 +1573,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "pkarr"
version = "3.0.0"
source = "git+https://github.com/Pubky/pkarr?branch=v3#0622003a851dfc34c4c3b44daaf13c99880d300a"
dependencies = [
"base32",
"bytes",
@@ -1621,6 +1624,12 @@ dependencies = [
"universal-hash",
]
[[package]]
name = "portable-atomic"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2"
[[package]]
name = "postcard"
version = "1.0.10"
@@ -1670,7 +1679,6 @@ version = "0.1.0"
dependencies = [
"base64 0.22.1",
"bytes",
"hyper-util",
"js-sys",
"pkarr",
"pubky-common",
@@ -1828,9 +1836,9 @@ dependencies = [
[[package]]
name = "redox_syscall"
version = "0.5.5"
version = "0.5.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62871f2d65009c0256aed1b9cfeeb8ac272833c404e13d53d400cd0dad7a2ac0"
checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f"
dependencies = [
"bitflags",
]
@@ -1848,14 +1856,14 @@ dependencies = [
[[package]]
name = "regex"
version = "1.10.6"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619"
checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata 0.4.7",
"regex-syntax 0.8.4",
"regex-automata 0.4.8",
"regex-syntax 0.8.5",
]
[[package]]
@@ -1869,13 +1877,13 @@ dependencies = [
[[package]]
name = "regex-automata"
version = "0.4.7"
version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df"
checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax 0.8.4",
"regex-syntax 0.8.5",
]
[[package]]
@@ -1886,15 +1894,15 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
[[package]]
name = "regex-syntax"
version = "0.8.4"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
[[package]]
name = "reqwest"
version = "0.12.7"
version = "0.12.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63"
checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b"
dependencies = [
"base64 0.22.1",
"bytes",
@@ -2026,19 +2034,18 @@ dependencies = [
[[package]]
name = "rustls-pemfile"
version = "2.1.3"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425"
checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50"
dependencies = [
"base64 0.22.1",
"rustls-pki-types",
]
[[package]]
name = "rustls-pki-types"
version = "1.8.0"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0"
checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55"
[[package]]
name = "rustls-webpki"
@@ -2185,9 +2192,9 @@ dependencies = [
[[package]]
name = "serde_spanned"
version = "0.6.7"
version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d"
checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1"
dependencies = [
"serde",
]
@@ -2344,9 +2351,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "syn"
version = "2.0.77"
version = "2.0.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed"
checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590"
dependencies = [
"proc-macro2",
"quote",
@@ -2400,9 +2407,9 @@ dependencies = [
[[package]]
name = "tempfile"
version = "3.12.0"
version = "3.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64"
checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b"
dependencies = [
"cfg-if",
"fastrand",
@@ -3095,9 +3102,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "winnow"
version = "0.6.19"
version = "0.6.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c52ac009d615e79296318c1bcce2d422aaca15ad08515e344feeda07df67a587"
checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b"
dependencies = [
"memchr",
]

View File

@@ -79,7 +79,7 @@ pubkyauth:///
```abnf
AuthToken = signature namespace version timestamp pubky capabilities
signature = 64OCTET ; ed25519 signature over encoded DNS packet
signature = 64OCTET ; ed25519 signature over the rest of the token.
namespace = %x50.55.42.4b.59.3a.41.55.54.48 ; "PUBKY:AUTH" in UTF-8 (10 bytes)
version = 1*OCTET ; Version of the AuthToken for future proofing.
timestamp = 8OCTET ; Big-endian UNIX timestamp in microseconds

View File

@@ -68,6 +68,10 @@ impl Session {
}
pub fn deserialize(bytes: &[u8]) -> Result<Self> {
if bytes.is_empty() {
return Err(Error::EmptyPayload);
}
if bytes[0] > 0 {
return Err(Error::UnknownVersion);
}
@@ -80,8 +84,10 @@ impl Session {
pub type Result<T> = core::result::Result<T, Error>;
#[derive(thiserror::Error, Debug)]
#[derive(thiserror::Error, Debug, PartialEq)]
pub enum Error {
#[error("Empty payload")]
EmptyPayload,
#[error("Unknown version")]
UnknownVersion,
#[error(transparent)]
@@ -123,4 +129,11 @@ mod tests {
assert_eq!(deseiralized, session)
}
#[test]
fn deserialize() {
let result = Session::deserialize(&[]);
assert_eq!(result, Err(Error::EmptyPayload));
}
}

View File

@@ -2,7 +2,7 @@
use anyhow::{anyhow, Context, Result};
use pkarr::Keypair;
use serde::{Deserialize, Deserializer, Serialize};
use serde::{Deserialize, Serialize};
use std::{
fmt::Debug,
path::{Path, PathBuf},
@@ -12,34 +12,102 @@ use tracing::info;
use pubky_common::timestamp::Timestamp;
const DEFAULT_HOMESERVER_PORT: u16 = 6287;
// === Database ===
const DEFAULT_STORAGE_DIR: &str = "pubky";
pub const DEFAULT_MAP_SIZE: usize = 10995116277760; // 10TB (not = disk-space used)
/// Server configuration
#[derive(Serialize, Deserialize, Clone)]
pub struct Config {
testnet: bool,
// === Server ==
pub const DEFAULT_LIST_LIMIT: u16 = 100;
pub const DEFAULT_MAX_LIST_LIMIT: u16 = 1000;
#[derive(Serialize, Deserialize, Clone, PartialEq)]
struct ConfigToml {
testnet: Option<bool>,
port: Option<u16>,
bootstrap: Option<Vec<String>>,
domain: Option<String>,
/// Path to the storage directory
storage: Option<PathBuf>,
secret_key: Option<String>,
dht_request_timeout: Option<Duration>,
default_list_limit: Option<u16>,
max_list_limit: Option<u16>,
db_map_size: Option<usize>,
}
/// Server configuration
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Config {
/// Whether or not this server is running in a testnet.
testnet: bool,
/// The configured port for this server.
port: u16,
/// Bootstrapping DHT nodes.
///
/// Helpful to run the server locally or in testnet.
bootstrap: Option<Vec<String>>,
/// A public domain for this server
/// necessary for web browsers running in https environment.
domain: Option<String>,
/// Path to the storage directory.
///
/// Defaults to a directory in the OS data directory
storage: Option<PathBuf>,
#[serde(deserialize_with = "secret_key_deserialize")]
secret_key: Option<[u8; 32]>,
storage: PathBuf,
/// Server keypair.
///
/// Defaults to a random keypair.
keypair: Keypair,
dht_request_timeout: Option<Duration>,
/// The default limit of a list api if no `limit` query parameter is provided.
///
/// Defaults to `100`
default_list_limit: u16,
/// The maximum limit of a list api, even if a `limit` query parameter is provided.
///
/// Defaults to `1000`
max_list_limit: u16,
// === Database params ===
db_map_size: usize,
}
impl Config {
/// Load the config from a file.
pub async fn load(path: impl AsRef<Path>) -> Result<Config> {
let s = tokio::fs::read_to_string(path.as_ref())
.await
.with_context(|| format!("failed to read {}", path.as_ref().to_string_lossy()))?;
fn try_from_str(value: &str) -> Result<Self> {
let config_toml: ConfigToml = toml::from_str(value)?;
let config: Config = toml::from_str(&s)?;
let keypair = if let Some(secret_key) = config_toml.secret_key {
let secret_key = deserialize_secret_key(secret_key)?;
Keypair::from_secret_key(&secret_key)
} else {
Keypair::random()
};
let storage = {
let dir = if let Some(storage) = config_toml.storage {
storage
} else {
let path = dirs_next::data_dir().ok_or_else(|| {
anyhow!("operating environment provides no directory for application data")
})?;
path.join(DEFAULT_STORAGE_DIR)
};
dir.join("homeserver")
};
let config = Config {
testnet: config_toml.testnet.unwrap_or(false),
port: config_toml.port.unwrap_or(0),
bootstrap: config_toml.bootstrap,
domain: config_toml.domain,
keypair,
storage,
dht_request_timeout: config_toml.dht_request_timeout,
default_list_limit: config_toml.default_list_limit.unwrap_or(DEFAULT_LIST_LIMIT),
max_list_limit: config_toml
.default_list_limit
.unwrap_or(DEFAULT_MAX_LIST_LIMIT),
db_map_size: config_toml.db_map_size.unwrap_or(DEFAULT_MAP_SIZE),
};
if config.testnet {
let testnet_config = Config::testnet();
@@ -53,45 +121,47 @@ impl Config {
Ok(config)
}
/// Load the config from a file.
pub async fn load(path: impl AsRef<Path>) -> Result<Config> {
let s = tokio::fs::read_to_string(path.as_ref())
.await
.with_context(|| format!("failed to read {}", path.as_ref().to_string_lossy()))?;
Config::try_from_str(&s)
}
/// Testnet configurations
pub fn testnet() -> Self {
let testnet = pkarr::mainline::Testnet::new(10);
let testnet = pkarr::mainline::Testnet::new(10).unwrap();
info!(?testnet.bootstrap, "Testnet bootstrap nodes");
let bootstrap = Some(testnet.bootstrap.to_owned());
let storage = Some(
std::env::temp_dir()
.join(Timestamp::now().to_string())
.join(DEFAULT_STORAGE_DIR),
);
Self {
bootstrap,
storage,
port: Some(15411),
dht_request_timeout: Some(Duration::from_millis(10)),
..Default::default()
Config {
port: 15411,
dht_request_timeout: None,
db_map_size: DEFAULT_MAP_SIZE,
..Self::test(&testnet)
}
}
/// Test configurations
pub fn test(testnet: &pkarr::mainline::Testnet) -> Self {
let bootstrap = Some(testnet.bootstrap.to_owned());
let storage = Some(
std::env::temp_dir()
.join(Timestamp::now().to_string())
.join(DEFAULT_STORAGE_DIR),
);
let storage = std::env::temp_dir()
.join(Timestamp::now().to_string())
.join(DEFAULT_STORAGE_DIR);
Self {
testnet: true,
bootstrap,
storage,
db_map_size: 10485760,
dht_request_timeout: Some(Duration::from_millis(10)),
..Default::default()
}
}
pub fn port(&self) -> u16 {
self.port.unwrap_or(DEFAULT_HOMESERVER_PORT)
self.port
}
pub fn bootstsrap(&self) -> Option<Vec<String>> {
@@ -102,73 +172,134 @@ impl Config {
self.domain.as_ref()
}
/// Get the path to the storage directory
pub fn storage(&self) -> Result<PathBuf> {
let dir = if let Some(storage) = &self.storage {
PathBuf::from(storage)
} else {
let path = dirs_next::data_dir().ok_or_else(|| {
anyhow!("operating environment provides no directory for application data")
})?;
path.join(DEFAULT_STORAGE_DIR)
};
Ok(dir.join("homeserver"))
pub fn keypair(&self) -> &Keypair {
&self.keypair
}
pub fn keypair(&self) -> Keypair {
Keypair::from_secret_key(&self.secret_key.unwrap_or_default())
pub fn default_list_limit(&self) -> u16 {
self.default_list_limit
}
pub fn max_list_limit(&self) -> u16 {
self.max_list_limit
}
/// Get the path to the storage directory
pub fn storage(&self) -> &PathBuf {
&self.storage
}
pub(crate) fn dht_request_timeout(&self) -> Option<Duration> {
self.dht_request_timeout
}
pub(crate) fn db_map_size(&self) -> usize {
self.db_map_size
}
}
impl Default for Config {
fn default() -> Self {
Self {
testnet: false,
port: Some(0),
port: 0,
bootstrap: None,
domain: None,
storage: None,
secret_key: None,
storage: storage(None)
.expect("operating environment provides no directory for application data"),
keypair: Keypair::random(),
dht_request_timeout: None,
default_list_limit: DEFAULT_LIST_LIMIT,
max_list_limit: DEFAULT_MAX_LIST_LIMIT,
db_map_size: DEFAULT_MAP_SIZE,
}
}
}
fn secret_key_deserialize<'de, D>(deserializer: D) -> Result<Option<[u8; 32]>, D::Error>
where
D: Deserializer<'de>,
{
let opt: Option<String> = Option::deserialize(deserializer)?;
fn deserialize_secret_key(s: String) -> anyhow::Result<[u8; 32]> {
let bytes =
hex::decode(s).map_err(|_| anyhow!("secret_key in config.toml should hex encoded"))?;
match opt {
Some(s) => {
let bytes = hex::decode(s).map_err(serde::de::Error::custom)?;
if bytes.len() != 32 {
return Err(anyhow!(format!(
"secret_key in config.toml should be 32 bytes in hex (64 characters), got: {}",
bytes.len()
)));
}
if bytes.len() != 32 {
return Err(serde::de::Error::custom("Expected a 32-byte array"));
let mut arr = [0u8; 32];
arr.copy_from_slice(&bytes);
Ok(arr)
}
fn storage(storage: Option<String>) -> Result<PathBuf> {
let dir = if let Some(storage) = storage {
PathBuf::from(storage)
} else {
let path = dirs_next::data_dir().ok_or_else(|| {
anyhow!("operating environment provides no directory for application data")
})?;
path.join(DEFAULT_STORAGE_DIR)
};
Ok(dir.join("homeserver"))
}
#[cfg(test)]
mod tests {
use pkarr::mainline::Testnet;
use super::*;
#[test]
fn parse_empty() {
let config = Config::try_from_str("").unwrap();
assert_eq!(
config,
Config {
keypair: config.keypair.clone(),
..Default::default()
}
)
}
let mut arr = [0u8; 32];
arr.copy_from_slice(&bytes);
Ok(Some(arr))
}
None => Ok(None),
}
}
impl Debug for Config {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_map()
.entry(&"testnet", &self.testnet)
.entry(&"port", &self.port())
.entry(&"storage", &self.storage())
.entry(&"public_key", &self.keypair().public_key())
.entry(&"domain", &self.domain())
.finish()
#[test]
fn config_test() {
let testnet = Testnet::new(3).unwrap();
let config = Config::test(&testnet);
assert_eq!(
config,
Config {
testnet: true,
bootstrap: testnet.bootstrap.into(),
db_map_size: 10485760,
dht_request_timeout: Some(Duration::from_millis(10)),
storage: config.storage.clone(),
keypair: config.keypair.clone(),
..Default::default()
}
)
}
#[test]
fn config_testnet() {
let config = Config::testnet();
assert_eq!(
config,
Config {
testnet: true,
port: 15411,
bootstrap: config.bootstrap.clone(),
storage: config.storage.clone(),
keypair: config.keypair.clone(),
..Default::default()
}
)
}
}

View File

@@ -1,31 +1,39 @@
use std::fs;
use std::path::Path;
use heed::{Env, EnvOpenOptions};
mod migrations;
pub mod tables;
use tables::{Tables, TABLES_COUNT};
use crate::config::Config;
pub const MAX_LIST_LIMIT: u16 = 100;
use tables::{Tables, TABLES_COUNT};
#[derive(Debug, Clone)]
pub struct DB {
pub(crate) env: Env,
pub(crate) tables: Tables,
pub(crate) config: Config,
}
impl DB {
pub fn open(storage: &Path) -> anyhow::Result<Self> {
fs::create_dir_all(storage).unwrap();
pub fn open(config: Config) -> anyhow::Result<Self> {
fs::create_dir_all(config.storage())?;
let env = unsafe { EnvOpenOptions::new().max_dbs(TABLES_COUNT).open(storage) }?;
let env = unsafe {
EnvOpenOptions::new()
.max_dbs(TABLES_COUNT)
.map_size(config.db_map_size())
.open(config.storage())
}?;
let tables = migrations::run(&env)?;
let db = DB { env, tables };
let db = DB {
env,
tables,
config,
};
Ok(db)
}
@@ -34,18 +42,15 @@ impl DB {
#[cfg(test)]
mod tests {
use bytes::Bytes;
use pkarr::Keypair;
use pubky_common::timestamp::Timestamp;
use pkarr::{mainline::Testnet, Keypair};
use crate::config::Config;
use super::DB;
#[tokio::test]
async fn entries() {
let storage = std::env::temp_dir()
.join(Timestamp::now().to_string())
.join("pubky");
let db = DB::open(&storage).unwrap();
let db = DB::open(Config::test(&Testnet::new(0).unwrap())).unwrap();
let keypair = Keypair::random();
let path = "/pub/foo.txt";

View File

@@ -13,7 +13,7 @@ use pubky_common::{
timestamp::Timestamp,
};
use crate::database::{DB, MAX_LIST_LIMIT};
use crate::database::DB;
use super::events::Event;
@@ -157,7 +157,7 @@ impl DB {
/// Return a list of pubky urls.
///
/// - limit defaults to and capped by [MAX_LIST_LIMIT]
/// - limit defaults to [Config::default_list_limit] and capped by [Config::max_list_limit]
pub fn list(
&self,
txn: &RoTxn,
@@ -170,7 +170,9 @@ impl DB {
// Vector to store results
let mut results = Vec::new();
let limit = limit.unwrap_or(MAX_LIST_LIMIT).min(MAX_LIST_LIMIT);
let limit = limit
.unwrap_or(self.config.default_list_limit())
.min(self.config.max_list_limit());
// TODO: make this more performant than split and allocations?

View File

@@ -10,6 +10,8 @@ use heed::{
use postcard::{from_bytes, to_allocvec};
use serde::{Deserialize, Serialize};
use crate::database::DB;
/// Event [Timestamp] base32 => Encoded event.
pub type EventsTable = Database<Str, Bytes>;
@@ -56,3 +58,48 @@ impl Event {
}
}
}
impl DB {
/// Returns a list of events formatted as `<OP> <url>`.
///
/// - limit defaults to [Config::default_list_limit] and capped by [Config::max_list_limit]
/// - cursor is a 13 character string encoding of a timestamp
pub fn list_events(
&self,
limit: Option<u16>,
cursor: Option<String>,
) -> anyhow::Result<Vec<String>> {
let txn = self.env.read_txn()?;
let limit = limit
.unwrap_or(self.config.default_list_limit())
.min(self.config.max_list_limit());
let cursor = cursor.unwrap_or("0000000000000".to_string());
let mut result: Vec<String> = vec![];
let mut next_cursor = cursor.to_string();
for _ in 0..limit {
match self.tables.events.get_greater_than(&txn, &next_cursor)? {
Some((timestamp, event_bytes)) => {
let event = Event::deserialize(event_bytes)?;
let line = format!("{} {}", event.operation(), event.url());
next_cursor = timestamp.to_string();
result.push(line);
}
None => break,
};
}
if !result.is_empty() {
result.push(format!("cursor: {next_cursor}"))
}
txn.commit()?;
Ok(result)
}
}

View File

@@ -5,6 +5,7 @@ use axum::{
http::StatusCode,
response::IntoResponse,
};
use tracing::debug;
pub type Result<T, E = Error> = core::result::Result<T, E>;
@@ -86,36 +87,42 @@ impl From<pkarr::Error> for Error {
impl From<std::io::Error> for Error {
fn from(error: std::io::Error) -> Self {
debug!(?error);
Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into())
}
}
impl From<heed::Error> for Error {
fn from(error: heed::Error) -> Self {
debug!(?error);
Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into())
}
}
impl From<anyhow::Error> for Error {
fn from(error: anyhow::Error) -> Self {
debug!(?error);
Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into())
}
}
impl From<postcard::Error> for Error {
fn from(error: postcard::Error) -> Self {
debug!(?error);
Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into())
}
}
impl From<axum::Error> for Error {
fn from(error: axum::Error) -> Self {
debug!(?error);
Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into())
}
}
impl<T> From<flume::SendError<T>> for Error {
fn from(error: flume::SendError<T>) -> Self {
debug!(?error);
Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into())
}
}

View File

@@ -2,7 +2,7 @@ use std::collections::HashMap;
use axum::{
async_trait,
extract::{FromRequestParts, Path},
extract::{FromRequestParts, Path, Query},
http::{request::Parts, StatusCode},
response::{IntoResponse, Response},
RequestPartsExt,
@@ -74,3 +74,50 @@ where
Ok(EntryPath(path.to_string()))
}
}
#[derive(Debug)]
pub struct ListQueryParams {
pub limit: Option<u16>,
pub cursor: Option<String>,
pub reverse: bool,
pub shallow: bool,
}
#[async_trait]
impl<S> FromRequestParts<S> for ListQueryParams
where
S: Send + Sync,
{
type Rejection = Response;
async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result<Self, Self::Rejection> {
let params: Query<HashMap<String, String>> =
parts.extract().await.map_err(IntoResponse::into_response)?;
let reverse = params.contains_key("reverse");
let shallow = params.contains_key("shallow");
let limit = params
.get("limit")
// Treat `limit=` as None
.and_then(|l| if l.is_empty() { None } else { Some(l) })
.and_then(|l| l.parse::<u16>().ok());
let cursor = params
.get("cursor")
.map(|c| c.as_str())
// Treat `cursor=` as None
.and_then(|c| {
if c.is_empty() {
None
} else {
Some(c.to_string())
}
});
Ok(ListQueryParams {
reverse,
shallow,
limit,
cursor,
})
}
}

View File

@@ -7,13 +7,14 @@ use pkarr::{
rdata::{RData, A, SVCB},
Packet,
},
Keypair, SignedPacket,
SignedPacket,
};
use crate::config::Config;
pub async fn publish_server_packet(
pkarr_client: pkarr::Client,
keypair: &Keypair,
domain: Option<&String>,
pkarr_client: &pkarr::Client,
config: &Config,
port: u16,
) -> anyhow::Result<()> {
// TODO: Try to resolve first before publishing.
@@ -21,7 +22,7 @@ pub async fn publish_server_packet(
let mut packet = Packet::new_reply(0);
let default = ".".to_string();
let target = domain.unwrap_or(&default);
let target = config.domain().unwrap_or(&default);
let mut svcb = SVCB::new(0, target.as_str().try_into()?);
svcb.priority = 1;
@@ -34,7 +35,7 @@ pub async fn publish_server_packet(
RData::HTTPS(svcb.clone().into()),
));
if domain.is_none() {
if config.domain().is_none() {
// TODO: remove after remvoing Pubky shared/public
// and add local host IP address instead.
svcb.target = "localhost".try_into().unwrap();
@@ -56,7 +57,7 @@ pub async fn publish_server_packet(
// TODO: announce A/AAAA records as well for TLS connections?
let signed_packet = SignedPacket::from_packet(keypair, &packet)?;
let signed_packet = SignedPacket::from_packet(config.keypair(), &packet)?;
pkarr_client.publish(&signed_packet).await?;

View File

@@ -1,67 +1,37 @@
use std::collections::HashMap;
use axum::{
body::Body,
extract::{Query, State},
extract::State,
http::{header, Response, StatusCode},
response::IntoResponse,
};
use pubky_common::timestamp::{Timestamp, TimestampError};
use crate::{
database::{tables::events::Event, MAX_LIST_LIMIT},
error::Result,
error::{Error, Result},
extractors::ListQueryParams,
server::AppState,
};
pub async fn feed(
State(state): State<AppState>,
Query(params): Query<HashMap<String, String>>,
params: ListQueryParams,
) -> Result<impl IntoResponse> {
let txn = state.db.env.read_txn()?;
if let Some(ref cursor) = params.cursor {
if let Err(timestmap_error) = Timestamp::try_from(cursor.to_string()) {
let cause = match timestmap_error {
TimestampError::InvalidEncoding => {
"Cursor should be valid base32 Crockford encoding of a timestamp"
}
TimestampError::InvalidBytesLength(size) => {
&format!("Cursor should be 13 characters long, got: {size}")
}
};
let limit = params
.get("limit")
.and_then(|l| l.parse::<u16>().ok())
.unwrap_or(MAX_LIST_LIMIT)
.min(MAX_LIST_LIMIT);
let mut cursor = params
.get("cursor")
.map(|c| c.as_str())
.unwrap_or("0000000000000");
// Guard against bad cursor
if cursor.len() < 13 {
cursor = "0000000000000"
Err(Error::new(StatusCode::BAD_REQUEST, cause.into()))?
}
}
let mut result: Vec<String> = vec![];
let mut next_cursor = cursor.to_string();
for _ in 0..limit {
match state
.db
.tables
.events
.get_greater_than(&txn, &next_cursor)?
{
Some((timestamp, event_bytes)) => {
let event = Event::deserialize(event_bytes)?;
let line = format!("{} {}", event.operation(), event.url());
next_cursor = timestamp.to_string();
result.push(line);
}
None => break,
};
}
if !result.is_empty() {
result.push(format!("cursor: {next_cursor}"))
}
txn.commit()?;
let result = state.db.list_events(params.limit, params.cursor)?;
Ok(Response::builder()
.status(StatusCode::OK)

View File

@@ -1,8 +1,6 @@
use std::collections::HashMap;
use axum::{
body::{Body, Bytes},
extract::{Query, State},
extract::State,
http::{header, Response, StatusCode},
response::IntoResponse,
};
@@ -12,7 +10,7 @@ use tower_cookies::Cookies;
use crate::{
error::{Error, Result},
extractors::{EntryPath, Pubky},
extractors::{EntryPath, ListQueryParams, Pubky},
server::AppState,
};
@@ -65,7 +63,7 @@ pub async fn get(
State(state): State<AppState>,
pubky: Pubky,
path: EntryPath,
Query(params): Query<HashMap<String, String>>,
params: ListQueryParams,
) -> Result<impl IntoResponse> {
verify(path.as_str())?;
let public_key = pubky.public_key();
@@ -88,10 +86,10 @@ pub async fn get(
let vec = state.db.list(
&txn,
&path,
params.contains_key("reverse"),
params.get("limit").and_then(|l| l.parse::<u16>().ok()),
params.get("cursor").map(|cursor| cursor.into()),
params.contains_key("shallow"),
params.reverse,
params.limit,
params.cursor,
params.shallow,
)?;
return Ok(Response::builder()

View File

@@ -14,25 +14,24 @@ use crate::{config::Config, database::DB, pkarr::publish_server_packet};
#[derive(Debug)]
pub struct Homeserver {
port: u16,
config: Config,
state: AppState,
tasks: JoinSet<std::io::Result<()>>,
}
#[derive(Clone, Debug)]
pub(crate) struct AppState {
pub verifier: AuthVerifier,
pub db: DB,
pub pkarr_client: pkarr::Client,
pub(crate) verifier: AuthVerifier,
pub(crate) db: DB,
pub(crate) pkarr_client: pkarr::Client,
pub(crate) config: Config,
pub(crate) port: u16,
}
impl Homeserver {
pub async fn start(config: Config) -> Result<Self> {
debug!(?config);
let keypair = config.keypair();
let db = DB::open(&config.storage()?)?;
let db = DB::open(config.clone())?;
let pkarr_client = pkarr::Client::new(Settings {
dht: DhtSettings {
@@ -43,22 +42,22 @@ impl Homeserver {
..Default::default()
})?;
let state = AppState {
verifier: AuthVerifier::default(),
db,
pkarr_client: pkarr_client.clone(),
};
let app = crate::routes::create_app(state);
let mut tasks = JoinSet::new();
let app = app.clone();
let listener = TcpListener::bind(SocketAddr::from(([0, 0, 0, 0], config.port()))).await?;
let port = listener.local_addr()?.port();
let state = AppState {
verifier: AuthVerifier::default(),
db,
pkarr_client: pkarr_client.clone(),
config: config.clone(),
port,
};
let app = crate::routes::create_app(state.clone());
// Spawn http server task
tasks.spawn(
axum::serve(
@@ -71,15 +70,14 @@ impl Homeserver {
info!("Homeserver listening on http://localhost:{port}");
publish_server_packet(pkarr_client, &keypair, config.domain(), port).await?;
publish_server_packet(&pkarr_client, &config, port).await?;
info!("Homeserver listening on http://{}", keypair.public_key());
info!(
"Homeserver listening on http://{}",
config.keypair().public_key()
);
Ok(Self {
tasks,
config,
port,
})
Ok(Self { tasks, state })
}
/// Test version of [Homeserver::start], using mainline Testnet, and a temporary storage.
@@ -92,11 +90,11 @@ impl Homeserver {
// === Getters ===
pub fn port(&self) -> u16 {
self.port
self.state.port
}
pub fn public_key(&self) -> PublicKey {
self.config.keypair().public_key()
self.state.config.keypair().public_key()
}
// === Public Methods ===

View File

@@ -18,7 +18,7 @@ bytes = "^1.7.1"
base64 = "0.22.1"
pubky-common = { version = "0.1.0", path = "../pubky-common" }
pkarr = { workspace = true }
pkarr = { workspace = true, features = ["endpoints"] }
# Native dependencies
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]

View File

@@ -29,7 +29,7 @@ mod tests {
#[tokio::test]
async fn http_get_pubky() {
let testnet = Testnet::new(10);
let testnet = Testnet::new(10).unwrap();
let homeserver = Homeserver::start_test(&testnet).await.unwrap();
@@ -48,7 +48,7 @@ mod tests {
#[tokio::test]
async fn http_get_icann() {
let testnet = Testnet::new(10);
let testnet = Testnet::new(10).unwrap();
let client = PubkyClient::builder().testnet(&testnet).build();

View File

@@ -235,7 +235,7 @@ mod tests {
#[tokio::test]
async fn basic_authn() {
let testnet = Testnet::new(10);
let testnet = Testnet::new(10).unwrap();
let server = Homeserver::start_test(&testnet).await.unwrap();
let client = PubkyClient::test(&testnet);
@@ -276,7 +276,7 @@ mod tests {
#[tokio::test]
async fn authz() {
let testnet = Testnet::new(10);
let testnet = Testnet::new(10).unwrap();
let server = Homeserver::start_test(&testnet).await.unwrap();
let keypair = Keypair::random();

View File

@@ -193,7 +193,7 @@ mod tests {
#[tokio::test]
async fn resolve_endpoint_https() {
let testnet = Testnet::new(10);
let testnet = Testnet::new(10).unwrap();
let pkarr_client = pkarr::Client::builder().testnet(&testnet).build().unwrap();
@@ -274,7 +274,7 @@ mod tests {
#[tokio::test]
async fn resolve_homeserver() {
let testnet = Testnet::new(10);
let testnet = Testnet::new(10).unwrap();
let server = Homeserver::start_test(&testnet).await.unwrap();
// Publish an intermediate controller of the homeserver

View File

@@ -105,7 +105,7 @@ mod tests {
#[tokio::test]
async fn put_get_delete() {
let testnet = Testnet::new(10);
let testnet = Testnet::new(10).unwrap();
let server = Homeserver::start_test(&testnet).await.unwrap();
let client = PubkyClient::test(&testnet);
@@ -132,7 +132,7 @@ mod tests {
#[tokio::test]
async fn unauthorized_put_delete() {
let testnet = Testnet::new(10);
let testnet = Testnet::new(10).unwrap();
let server = Homeserver::start_test(&testnet).await.unwrap();
let client = PubkyClient::test(&testnet);
@@ -198,7 +198,7 @@ mod tests {
#[tokio::test]
async fn list() {
let testnet = Testnet::new(10);
let testnet = Testnet::new(10).unwrap();
let server = Homeserver::start_test(&testnet).await.unwrap();
let client = PubkyClient::test(&testnet);
@@ -402,7 +402,7 @@ mod tests {
#[tokio::test]
async fn list_shallow() {
let testnet = Testnet::new(10);
let testnet = Testnet::new(10).unwrap();
let server = Homeserver::start_test(&testnet).await.unwrap();
let client = PubkyClient::test(&testnet);
@@ -613,7 +613,7 @@ mod tests {
#[tokio::test]
async fn list_events() {
let testnet = Testnet::new(10);
let testnet = Testnet::new(10).unwrap();
let server = Homeserver::start_test(&testnet).await.unwrap();
let client = PubkyClient::test(&testnet);
@@ -710,7 +710,7 @@ mod tests {
#[tokio::test]
async fn read_after_event() {
let testnet = Testnet::new(10);
let testnet = Testnet::new(10).unwrap();
let server = Homeserver::start_test(&testnet).await.unwrap();
let client = PubkyClient::test(&testnet);
@@ -758,7 +758,7 @@ mod tests {
#[tokio::test]
async fn dont_delete_shared_blobs() {
let testnet = Testnet::new(10);
let testnet = Testnet::new(10).unwrap();
let homeserver = Homeserver::start_test(&testnet).await.unwrap();
let client = PubkyClient::test(&testnet);