Merge branch 'main' into dev

.dockerignore

@@ -0,0 +1,3 @@
+/target
+/docs
+/.github

.gitignore

@@ -1 +1,2 @@
 target/
+config.toml

Cargo.toml

@@ -3,16 +3,12 @@ members = [
   "pubky",
   "pubky-*", 
 
-  "examples/authz/authenticator"
+  "examples"
 ]
 
 # See: https://github.com/rust-lang/rust/issues/90148#issuecomment-949194352
 resolver = "2"
 
-[workspace.dependencies]
-pkarr = { git = "https://github.com/Pubky/pkarr", branch = "v3", package = "pkarr" }
-serde = { version = "^1.0.209", features = ["derive"] }
-
 [profile.release]
 lto = true
 opt-level = 'z'

Dockerfile

@@ -0,0 +1,56 @@
+# ========================
+# Build Stage
+# ========================
+FROM rust:1.82.0-alpine3.20 AS builder
+
+# Install build dependencies, including static OpenSSL libraries
+RUN apk add --no-cache \
+    musl-dev \
+    openssl-dev \
+    openssl-libs-static \
+    pkgconfig \
+    build-base \
+    curl
+
+# Set environment variables for static linking with OpenSSL
+ENV OPENSSL_STATIC=yes
+ENV OPENSSL_LIB_DIR=/usr/lib
+ENV OPENSSL_INCLUDE_DIR=/usr/include
+
+# Add the MUSL target for static linking
+RUN rustup target add x86_64-unknown-linux-musl
+
+# Set the working directory
+WORKDIR /usr/src/app
+
+# Copy over Cargo.toml and Cargo.lock for dependency caching
+COPY Cargo.toml Cargo.lock ./
+
+# Copy over all the source code
+COPY . .
+
+# Build the project in release mode for the MUSL target
+RUN cargo build --release --bin pubky_homeserver --target x86_64-unknown-linux-musl
+
+# Strip the binary to reduce size
+RUN strip target/x86_64-unknown-linux-musl/release/pubky_homeserver
+
+# ========================
+# Runtime Stage
+# ========================
+FROM alpine:3.20
+
+# Install runtime dependencies (only ca-certificates)
+RUN apk add --no-cache ca-certificates
+
+# Copy the compiled binary from the builder stage
+COPY --from=builder /usr/src/app/target/x86_64-unknown-linux-musl/release/pubky_homeserver /usr/local/bin/homeserver
+
+# Set the working directory
+WORKDIR /usr/local/bin
+
+# Expose the port the homeserver listens on (should match that of config.toml)
+EXPOSE 6287
+
+# Set the default command to run the homeserver binary
+CMD ["homeserver"]

README.md

@@ -1,8 +1,61 @@
-# Pubky
+<h1 align="center"><a href="https://pubky.org/"><img alt="pubky" src="./.svg/pubky-core-logo.svg" width="200" /></a></h1>
+
+<h3 align="center">
+	An open protocol for per-public-key backends for censorship resistant web applications.
+</h3>
+
+<div align="center">
+  <h3>
+    <a href="https://pubky.github.io/pubky-core/">
+      Docs Site
+    </a>
+    <span> | </span>
+    <a href="https://docs.rs/pubky">
+      Rust Client's Docs
+    </a>
+    <span> | </span>
+    <a href="https://github.com/pubky/pubky-core/releases">
+      Releases
+    </a>
+    <span> | </span>
+    <a href="https://www.npmjs.com/package/@synonymdev/pubky">
+      JS bindings 
+    </a>
+  </h3>
+</div>
 
 > The Web, long centralized, must decentralize; Long decentralized, must centralize.
 
-> [!WARNING]  
-> Pubky is still under heavy development and should be considered an alpha software.
-> 
-> Features might be added, removed, or changed. Data might be lost.
+## Overview
+
+Pubky-core combines a [censorship resistant public-key based alternative to DNS](https://pkarr.org) with conventional, tried and tested web technologies, to keep users in control of their identities and data, while enabling developers to build software with as much availability as web apps, without the costs of managing a central database.
+
+## Features
+- Public key based authentication.
+- Public key based 3rd party authorization.
+- Key-value store through PUT/GET/DELETE HTTP API + pagination.
+
+## Getting started
+
+This repository contains a [Homeserver](./pubky-homeserver), and a [Client](./pubky) (both Rust and JS wasm bindings).
+You can a run a local homeserver using `cargo run` with more instructions in the README.
+Check  the [Examples](./examples) directory for small feature-focesed examples of how to use the Pubky client.
+
+### JavaScript
+If you prefer to use JavaScript in NodeJs/Browser or any runtime with Wasm support, you can either install from npm [`@synonymdev/pubky`](https://www.npmjs.com/package/@synonymdev/pubky)
+or build the bindings yourself:
+```bash
+cd pubky/pkg
+npm i
+npm run build
+```
+
+#### Testing
+There are unit tests for the JavaScript bindings in both NodeJs and headless web browser, but first you need to run a local temporary Homeserver
+```bash
+npm run testnet
+```
+Then in a different terminal window:
+```bash
+npm test
+```

examples/Cargo.toml

@@ -1,14 +1,27 @@
 [package]
-name = "authenticator"
+name = "authn"
 version = "0.1.0"
 edition = "2021"
 
+[[bin]]
+name = "signup"
+path = "./authn/signup.rs"
+
+[[bin]]
+name = "authenticator"
+path = "./authz/authenticator.rs"
+
+[[bin]]
+name = "request"
+path = "./request/main.rs"
+
 [dependencies]
 anyhow = "1.0.86"
 base64 = "0.22.1"
 clap = { version = "4.5.16", features = ["derive"] }
-pubky = { version = "0.1.0", path = "../../../pubky" }
-pubky-common = { version = "0.1.0", path = "../../../pubky-common" }
+pubky = { path = "../pubky" }
+pubky-common = { version = "0.1.0", path = "../pubky-common" }
+reqwest = "0.12.8"
 rpassword = "7.3.1"
 tokio = { version = "1.40.0", features = ["macros", "rt-multi-thread"] }
 url = "2.5.2"

examples/README.md

@@ -0,0 +1,7 @@
+# Pubky examples
+
+Minimal examples for different flows and functions you might need to implement using Pubky.
+
+- [request](./request/README.md): shows how to make direct HTTP requests to Pubky URLs.
+- [authentication](./authn/README.md): shows how to signup, signin or signout to and from a homeserver.
+- [authorization flow](./authz/README.md): shows how to setup Pubky authz for a 3rd party application and how to implement an authenticator to sign in such app.

examples/authn/README.md

@@ -0,0 +1,13 @@
+# Authentication examples
+
+You can use these examples to test Signup or Signin to a provided homeserver using a keypair,
+as opposed to using a the 3rd party [authorization flow](../authz).
+
+
+## Usage 
+
+### Signup
+
+```bash
+cargo run --bin signup <homeserver pubky> </path/to/recovery file>
+```

examples/authn/signup.rs

@@ -0,0 +1,49 @@
+use anyhow::Result;
+use clap::Parser;
+use pubky::PubkyClient;
+use std::path::PathBuf;
+
+use pubky_common::crypto::PublicKey;
+
+#[derive(Parser, Debug)]
+#[command(version, about, long_about = None)]
+struct Cli {
+    /// Homeserver Pkarr Domain (for example `5jsjx1o6fzu6aeeo697r3i5rx15zq41kikcye8wtwdqm4nb4tryo`)
+    homeserver: String,
+
+    /// Path to a recovery_file of the Pubky you want to sign in with
+    recovery_file: PathBuf,
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let cli = Cli::parse();
+
+    let recovery_file = std::fs::read(&cli.recovery_file)?;
+    println!("\nSuccessfully opened recovery file");
+
+    let homeserver = cli.homeserver;
+
+    let client = PubkyClient::builder().build();
+
+    println!("Enter your recovery_file's passphrase to signup:");
+    let passphrase = rpassword::read_password()?;
+
+    let keypair = pubky_common::recovery_file::decrypt_recovery_file(&recovery_file, &passphrase)?;
+
+    println!("Successfully decrypted the recovery file, signing up to the homeserver:");
+
+    client
+        .signup(&keypair, &PublicKey::try_from(homeserver).unwrap())
+        .await?;
+
+    println!("Successfully signed up. Checking session:");
+
+    let session = client.session(&keypair.public_key()).await?;
+
+    println!("Successfully resolved current session at the homeserver.");
+
+    println!("{:?}", session);
+
+    Ok(())
+}

examples/authz/README.md

@@ -5,7 +5,7 @@ This example shows 3rd party authorization in Pubky.
 It consists of 2 parts:
 
 1. [3rd party app](./3rd-party-app): A web component showing the how to implement a Pubky Auth widget.
-2. [Authenticator CLI](./authenticator): A CLI showing the authenticator (key chain) asking user for consent and generating the AuthToken.
+2. [Authenticator CLI](./authenticator.rs): A CLI showing the authenticator (key chain) asking user for consent and generating the AuthToken.
 
 ## Usage
 
@@ -26,4 +26,10 @@ Copy the Pubky Auth URL from the frontend.
 
 Finally run the CLI to paste the Pubky Auth in.
 
+```bash
+cargo run --bin authenticator <RECOVERY_FILE> "<Auth_URL>" [Testnet]
+```
+
+Where the auth url should be within qutoatino marks, and the Testnet is an option you can set to true to use the local homeserver
+
 You should see the frontend reacting by showing the success of authorization and session details.

examples/authz/authenticator.rs

@@ -17,6 +17,9 @@ struct Cli {
 
     /// Pubky Auth url
     url: Url,
+
+    // Whether or not to use testnet Dht network (local testing)
+    testnet: Option<bool>,
 }
 
 #[tokio::main]
@@ -62,14 +65,20 @@ async fn main() -> Result<()> {
     println!("Successfully decrypted recovery file...");
     println!("PublicKey: {}", keypair.public_key());
 
-    let client = PubkyClient::testnet();
+    let client = if cli.testnet.unwrap_or_default() {
+        let client = PubkyClient::testnet();
+
+        // For the purposes of this demo, we need to make sure
+        // the user has an account on the local homeserver.
+        if client.signin(&keypair).await.is_err() {
+            client
+                .signup(&keypair, &PublicKey::try_from(HOMESERVER).unwrap())
+                .await?;
+        };
 
-    // For the purposes of this demo, we need to make sure
-    // the user has an account on the local homeserver.
-    if client.signin(&keypair).await.is_err() {
         client
-            .signup(&keypair, &PublicKey::try_from(HOMESERVER).unwrap())
-            .await?;
+    } else {
+        PubkyClient::builder().build()
     };
 
     println!("Sending AuthToken to the 3rd party app...");

examples/request/README.md

@@ -0,0 +1,17 @@
+# Request
+
+Make HTTP requests over for Pubky authority URL.
+
+## Usage 
+
+Request data from a Pubky's data storage.
+
+```bash
+cargo run --bin request GET pubky://<user pubky>/pub/<path>
+```
+
+Or make a direct HTTP request.
+
+```bash
+cargo run --bin request GET https://<Pkarr domain>/[path]
+```

examples/request/main.rs

@@ -0,0 +1,38 @@
+use anyhow::Result;
+use clap::Parser;
+use reqwest::Method;
+use url::Url;
+
+use pubky::PubkyClient;
+
+#[derive(Parser, Debug)]
+#[command(version, about, long_about = None)]
+struct Cli {
+    /// HTTP method to use
+    method: Method,
+    /// Pubky or HTTPS url
+    url: Url,
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let cli = Cli::parse();
+
+    let client = PubkyClient::builder().build();
+
+    match cli.url.scheme() {
+        "https" => {
+            unimplemented!();
+        }
+        "pubky" => {
+            let response = client.get(cli.url).await.unwrap();
+
+            println!("Got a response: \n {:?}", response);
+        }
+        _ => {
+            panic!("Only https:// and pubky:// URL schemes are supported")
+        }
+    }
+
+    Ok(())
+}

pubky-common/Cargo.toml

@@ -2,32 +2,29 @@
 name = "pubky-common"
 version = "0.1.0"
 edition = "2021"
+description = "Types and struct in common between Pubky client and homeserver"
+license = "MIT"
+repository = "https://github.com/pubky/pubky-core"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 base32 = "0.5.0"
 blake3 = "1.5.1"
-ed25519-dalek = "2.1.1"
+ed25519-dalek = { version = "2.1.1", features = ["serde"] }
 once_cell = "1.19.0"
-pkarr = { workspace = true }
 rand = "0.8.5"
 thiserror = "1.0.60"
 postcard = { version = "1.0.8", features = ["alloc"] }
 crypto_secretbox = { version = "0.1.1", features = ["std"] }
 argon2 = { version = "0.5.3", features = ["std"] }
 
-serde = { workspace = true, optional = true }
+pubky-timestamp = { version = "0.2.0", features = ["full"] }
+serde = { version = "1.0.213", features = ["derive"] }
+pkarr = { version = "2.2.1-alpha.2", features = ["serde"] }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 js-sys = "0.3.69"
 
 [dev-dependencies]
 postcard = "1.0.8"
-
-[features]
-
-serde = ["dep:serde", "ed25519-dalek/serde", "pkarr/serde"]
-full = ['serde']
-
-default = ['full']

pubky-common/src/auth.rs

@@ -76,7 +76,7 @@ impl AuthToken {
                 let now = Timestamp::now();
 
                 // Chcek timestamp;
-                let diff = token.timestamp.difference(&now);
+                let diff = token.timestamp.as_u64() as i64 - now.as_u64() as i64;
                 if diff > TIMESTAMP_WINDOW {
                     return Err(Error::TooFarInTheFuture);
                 }
@@ -155,7 +155,7 @@ impl AuthVerifier {
 
     /// Remove all tokens older than two time intervals in the past.
     fn gc(&self) {
-        let threshold = ((Timestamp::now().into_inner() / TIME_INTERVAL) - 2).to_be_bytes();
+        let threshold = ((Timestamp::now().as_u64() / TIME_INTERVAL) - 2).to_be_bytes();
 
         let mut inner = self.seen.lock().unwrap();
 
@@ -235,7 +235,7 @@ mod tests {
 
         let verifier = AuthVerifier::default();
 
-        let timestamp = (&Timestamp::now()) - (TIMESTAMP_WINDOW as u64);
+        let timestamp = (Timestamp::now()) - (TIMESTAMP_WINDOW as u64);
 
         let mut signable = vec![];
         signable.extend_from_slice(signer.public_key().as_bytes());

pubky-common/src/capabilities.rs

@@ -9,7 +9,7 @@ pub struct Capability {
 }
 
 impl Capability {
-    /// Create a root [Capability] at the `/` path with all the available [PubkyAbility]
+    /// Create a root [Capability] at the `/` path with all the available [Action]s
     pub fn root() -> Self {
         Capability {
             scope: "/".to_string(),

pubky-common/src/lib.rs

@@ -4,4 +4,7 @@ pub mod crypto;
 pub mod namespaces;
 pub mod recovery_file;
 pub mod session;
-pub mod timestamp;
+
+pub mod timestamp {
+    pub use pubky_timestamp::*;
+}

pubky-common/src/session.rs

@@ -26,7 +26,7 @@ impl Session {
         Self {
             version: 0,
             pubky: token.pubky().to_owned(),
-            created_at: Timestamp::now().into_inner(),
+            created_at: Timestamp::now().as_u64(),
             capabilities: token.capabilities().to_vec(),
             user_agent: user_agent.as_deref().unwrap_or("").to_string(),
             name: user_agent.as_deref().unwrap_or("").to_string(),

pubky-homeserver/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "pubky_homeserver"
+name = "pubky-homeserver"
 version = "0.1.0"
 edition = "2021"
 
@@ -15,13 +15,19 @@ flume = "0.11.0"
 futures-util = "0.3.30"
 heed = "0.20.3"
 hex = "0.4.3"
-pkarr = { workspace = true }
+httpdate = "1.0.3"
+libc = "0.2.159"
 postcard = { version = "1.0.8", features = ["alloc"] }
+pkarr = { version = "2.2.1-alpha.2", features = ["serde", "async"]  }
 pubky-common = { version = "0.1.0", path = "../pubky-common" }
-serde = { workspace = true }
+serde = { version = "1.0.213", features = ["derive"] }
 tokio = { version = "1.37.0", features = ["full"] }
 toml = "0.8.19"
 tower-cookies = "0.10.0"
 tower-http = { version = "0.5.2", features = ["cors", "trace"] }
 tracing = "0.1.40"
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
+url = "2.5.2"
+
+[dev-dependencies]
+reqwest = "0.12.8"

pubky-homeserver/README.md

@@ -1,5 +1,7 @@
 # Pubky Homeserver
 
+A pubky-core homeserver that acts as users' agent on the Internet, providing data availability and more.more.more.more.
+
 ## Usage
 
 Use `cargo run`

pubky-homeserver/src/config.rs

@@ -114,6 +114,8 @@ impl Config {
 
             return Ok(Config {
                 bootstrap: testnet_config.bootstrap,
+                port: testnet_config.port,
+                keypair: testnet_config.keypair,
                 ..config
             });
         }
@@ -139,6 +141,7 @@ impl Config {
             port: 15411,
             dht_request_timeout: None,
             db_map_size: DEFAULT_MAP_SIZE,
+            keypair: Keypair::from_secret_key(&[0; 32]),
             ..Self::test(&testnet)
         }
     }
@@ -155,7 +158,6 @@ impl Config {
             bootstrap,
             storage,
             db_map_size: 10485760,
-            dht_request_timeout: Some(Duration::from_millis(10)),
             ..Default::default()
         }
     }
@@ -276,7 +278,6 @@ mod tests {
                 testnet: true,
                 bootstrap: testnet.bootstrap.into(),
                 db_map_size: 10485760,
-                dht_request_timeout: Some(Duration::from_millis(10)),
 
                 storage: config.storage.clone(),
                 keypair: config.keypair.clone(),
@@ -302,4 +303,35 @@ mod tests {
             }
         )
     }
+
+    #[test]
+    fn parse_with_testnet_flag() {
+        let config = Config::try_from_str(
+            r#"
+            # Secret key (in hex) to generate the Homeserver's Keypair
+            secret_key = "0123000000000000000000000000000000000000000000000000000000000000"
+            # Domain to be published in Pkarr records for this server to be accessible by.
+            domain = "localhost"
+            # Port for the Homeserver to listen on.
+            port = 6287
+            # Storage directory Defaults to <System's Data Directory>
+            storage = "/homeserver"
+            testnet = true
+
+            bootstrap = ["foo", "bar"]
+
+            # event stream
+            default_list_limit = 500
+            max_list_limit = 10000
+        "#,
+        )
+        .unwrap();
+
+        assert_eq!(config.keypair, Keypair::from_secret_key(&[0; 32]));
+        assert_eq!(config.port, 15411);
+        assert_ne!(
+            config.bootstrap,
+            Some(vec!["foo".to_string(), "bar".to_string()])
+        );
+    }
 }

pubky-homeserver/src/database.rs

@@ -1,4 +1,4 @@
-use std::fs;
+use std::{fs, path::PathBuf};
 
 use heed::{Env, EnvOpenOptions};
 
@@ -14,11 +14,17 @@ pub struct DB {
     pub(crate) env: Env,
     pub(crate) tables: Tables,
     pub(crate) config: Config,
+    pub(crate) buffers_dir: PathBuf,
+    pub(crate) max_chunk_size: usize,
 }
 
 impl DB {
     pub fn open(config: Config) -> anyhow::Result<Self> {
-        fs::create_dir_all(config.storage())?;
+        let buffers_dir = config.storage().clone().join("buffers");
+
+        // Cleanup buffers.
+        let _ = fs::remove_dir(&buffers_dir);
+        fs::create_dir_all(&buffers_dir)?;
 
         let env = unsafe {
             EnvOpenOptions::new()
@@ -33,46 +39,25 @@ impl DB {
             env,
             tables,
             config,
+            buffers_dir,
+            max_chunk_size: max_chunk_size(),
         };
 
         Ok(db)
     }
 }
 
-#[cfg(test)]
-mod tests {
-    use bytes::Bytes;
-    use pkarr::{mainline::Testnet, Keypair};
+/// calculate optimal chunk size:
+/// - https://lmdb.readthedocs.io/en/release/#storage-efficiency-limits
+/// - https://github.com/lmdbjava/benchmarks/blob/master/results/20160710/README.md#test-2-determine-24816-kb-byte-values
+fn max_chunk_size() -> usize {
+    let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) as usize };
 
-    use crate::config::Config;
-
-    use super::DB;
-
-    #[tokio::test]
-    async fn entries() {
-        let db = DB::open(Config::test(&Testnet::new(0).unwrap())).unwrap();
-
-        let keypair = Keypair::random();
-        let path = "/pub/foo.txt";
-
-        let (tx, rx) = flume::bounded::<Bytes>(0);
-
-        let mut cloned = db.clone();
-        let cloned_keypair = keypair.clone();
-
-        let done = tokio::task::spawn_blocking(move || {
-            cloned
-                .put_entry(&cloned_keypair.public_key(), path, rx)
-                .unwrap();
-        });
-
-        tx.send(vec![1, 2, 3, 4, 5].into()).unwrap();
-        drop(tx);
-
-        done.await.unwrap();
-
-        let blob = db.get_blob(&keypair.public_key(), path).unwrap().unwrap();
-
-        assert_eq!(blob, Bytes::from(vec![1, 2, 3, 4, 5]));
-    }
+    // - 16 bytes Header  per page (LMDB)
+    // - Each page has to contain 2 records
+    // - 8 bytes per record (LMDB) (imperically, it seems to be 10 not 8)
+    // - 12 bytes key:
+    //      - timestamp : 8 bytes
+    //      - chunk index: 4 bytes
+    ((page_size - 16) / 2) - (8 + 2) - 12
 }

pubky-homeserver/src/database/tables/blobs.rs

@@ -1,38 +1,24 @@
-use heed::{types::Bytes, Database};
-use pkarr::PublicKey;
+use heed::{types::Bytes, Database, RoTxn};
 
 use crate::database::DB;
 
 use super::entries::Entry;
 
-/// hash of the blob => bytes.
+/// (entry timestamp | chunk_index BE) => bytes
 pub type BlobsTable = Database<Bytes, Bytes>;
 
 pub const BLOBS_TABLE: &str = "blobs";
 
 impl DB {
-    pub fn get_blob(
+    pub fn read_entry_content<'txn>(
         &self,
-        public_key: &PublicKey,
-        path: &str,
-    ) -> anyhow::Result<Option<bytes::Bytes>> {
-        let rtxn = self.env.read_txn()?;
-
-        let key = format!("{public_key}/{path}");
-
-        let result = if let Some(bytes) = self.tables.entries.get(&rtxn, &key)? {
-            let entry = Entry::deserialize(bytes)?;
-
-            self.tables
-                .blobs
-                .get(&rtxn, entry.content_hash())?
-                .map(|blob| bytes::Bytes::from(blob[8..].to_vec()))
-        } else {
-            None
-        };
-
-        rtxn.commit()?;
-
-        Ok(result)
+        rtxn: &'txn RoTxn,
+        entry: &Entry,
+    ) -> anyhow::Result<impl Iterator<Item = Result<&'txn [u8], heed::Error>> + 'txn> {
+        Ok(self
+            .tables
+            .blobs
+            .prefix_iter(rtxn, &entry.timestamp().to_bytes())?
+            .map(|i| i.map(|(_, bytes)| bytes)))
     }
 }

pubky-homeserver/src/database/tables/entries.rs

@@ -1,6 +1,11 @@
 use pkarr::PublicKey;
 use postcard::{from_bytes, to_allocvec};
 use serde::{Deserialize, Serialize};
+use std::{
+    fs::File,
+    io::{Read, Write},
+    path::PathBuf,
+};
 use tracing::instrument;
 
 use heed::{
@@ -23,74 +28,12 @@ pub type EntriesTable = Database<Str, Bytes>;
 pub const ENTRIES_TABLE: &str = "entries";
 
 impl DB {
-    pub fn put_entry(
+    pub fn write_entry(
         &mut self,
         public_key: &PublicKey,
         path: &str,
-        rx: flume::Receiver<bytes::Bytes>,
-    ) -> anyhow::Result<()> {
-        let mut wtxn = self.env.write_txn()?;
-
-        let mut hasher = Hasher::new();
-        let mut bytes = vec![];
-        let mut length = 0;
-
-        while let Ok(chunk) = rx.recv() {
-            hasher.update(&chunk);
-            bytes.extend_from_slice(&chunk);
-            length += chunk.len();
-        }
-
-        let hash = hasher.finalize();
-
-        let key = hash.as_bytes();
-
-        let mut bytes_with_ref_count = Vec::with_capacity(bytes.len() + 8);
-        bytes_with_ref_count.extend_from_slice(&u64::to_be_bytes(0));
-        bytes_with_ref_count.extend_from_slice(&bytes);
-
-        // TODO: For now, we set the first 8 bytes to a reference counter
-        let exists = self
-            .tables
-            .blobs
-            .get(&wtxn, key)?
-            .unwrap_or(bytes_with_ref_count.as_slice());
-
-        let new_count = u64::from_be_bytes(exists[0..8].try_into().unwrap()) + 1;
-
-        bytes_with_ref_count[0..8].copy_from_slice(&u64::to_be_bytes(new_count));
-
-        self.tables
-            .blobs
-            .put(&mut wtxn, hash.as_bytes(), &bytes_with_ref_count)?;
-
-        let mut entry = Entry::new();
-
-        entry.set_content_hash(hash);
-        entry.set_content_length(length);
-
-        let key = format!("{public_key}/{path}");
-
-        self.tables
-            .entries
-            .put(&mut wtxn, &key, &entry.serialize())?;
-
-        if path.starts_with("pub/") {
-            let url = format!("pubky://{key}");
-            let event = Event::put(&url);
-            let value = event.serialize();
-
-            let key = entry.timestamp.to_string();
-
-            self.tables.events.put(&mut wtxn, &key, &value)?;
-
-            // TODO: delete older events.
-            // TODO: move to events.rs
-        }
-
-        wtxn.commit()?;
-
-        Ok(())
+    ) -> anyhow::Result<EntryWriter> {
+        EntryWriter::new(self, public_key, path)
     }
 
     pub fn delete_entry(&mut self, public_key: &PublicKey, path: &str) -> anyhow::Result<bool> {
@@ -101,28 +44,20 @@ impl DB {
         let deleted = if let Some(bytes) = self.tables.entries.get(&wtxn, &key)? {
             let entry = Entry::deserialize(bytes)?;
 
-            let mut bytes_with_ref_count = self
-                .tables
-                .blobs
-                .get(&wtxn, entry.content_hash())?
-                .map_or(vec![], |s| s.to_vec());
+            let mut deleted_chunks = false;
 
-            let arr: [u8; 8] = bytes_with_ref_count[0..8].try_into().unwrap_or([0; 8]);
-            let reference_count = u64::from_be_bytes(arr);
-
-            let deleted_blobs = if reference_count > 1 {
-                // decrement reference count
-
-                bytes_with_ref_count[0..8].copy_from_slice(&(reference_count - 1).to_be_bytes());
-
-                self.tables
+            {
+                let mut iter = self
+                    .tables
                     .blobs
-                    .put(&mut wtxn, entry.content_hash(), &bytes_with_ref_count)?;
+                    .prefix_iter_mut(&mut wtxn, &entry.timestamp.to_bytes())?;
 
-                true
-            } else {
-                self.tables.blobs.delete(&mut wtxn, entry.content_hash())?
-            };
+                while iter.next().is_some() {
+                    unsafe {
+                        deleted_chunks = iter.del_current()?;
+                    }
+                }
+            }
 
             let deleted_entry = self.tables.entries.delete(&mut wtxn, &key)?;
 
@@ -137,11 +72,11 @@ impl DB {
 
                 self.tables.events.put(&mut wtxn, &key, &value)?;
 
-                // TODO: delete older events.
+                // TODO: delete events older than a threshold.
                 // TODO: move to events.rs
             }
 
-            deleted_entry && deleted_blobs
+            deleted_entry && deleted_chunks
         } else {
             false
         };
@@ -151,6 +86,21 @@ impl DB {
         Ok(deleted)
     }
 
+    pub fn get_entry(
+        &self,
+        txn: &RoTxn,
+        public_key: &PublicKey,
+        path: &str,
+    ) -> anyhow::Result<Option<Entry>> {
+        let key = format!("{public_key}/{path}");
+
+        if let Some(bytes) = self.tables.entries.get(txn, &key)? {
+            return Ok(Some(Entry::deserialize(bytes)?));
+        }
+
+        Ok(None)
+    }
+
     pub fn contains_directory(&self, txn: &RoTxn, path: &str) -> anyhow::Result<bool> {
         Ok(self.tables.entries.get_greater_than(txn, path)?.is_some())
     }
@@ -268,13 +218,40 @@ pub struct Entry {
     version: usize,
     /// Modified at
     timestamp: Timestamp,
-    content_hash: [u8; 32],
+    content_hash: EntryHash,
     content_length: usize,
     content_type: String,
     // user_metadata: ?
 }
 
-// TODO: get headers like Etag
+#[derive(Clone, Debug, Eq, PartialEq)]
+struct EntryHash(Hash);
+
+impl Default for EntryHash {
+    fn default() -> Self {
+        Self(Hash::from_bytes([0; 32]))
+    }
+}
+
+impl Serialize for EntryHash {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        let bytes = self.0.as_bytes();
+        bytes.serialize(serializer)
+    }
+}
+
+impl<'de> Deserialize<'de> for EntryHash {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let bytes: [u8; 32] = Deserialize::deserialize(deserializer)?;
+        Ok(Self(Hash::from_bytes(bytes)))
+    }
+}
 
 impl Entry {
     pub fn new() -> Self {
@@ -283,8 +260,13 @@ impl Entry {
 
     // === Setters ===
 
+    pub fn set_timestamp(&mut self, timestamp: &Timestamp) -> &mut Self {
+        self.timestamp = *timestamp;
+        self
+    }
+
     pub fn set_content_hash(&mut self, content_hash: Hash) -> &mut Self {
-        content_hash.as_bytes().clone_into(&mut self.content_hash);
+        EntryHash(content_hash).clone_into(&mut self.content_hash);
         self
     }
 
@@ -295,12 +277,32 @@ impl Entry {
 
     // === Getters ===
 
-    pub fn content_hash(&self) -> &[u8; 32] {
-        &self.content_hash
+    pub fn timestamp(&self) -> &Timestamp {
+        &self.timestamp
+    }
+
+    pub fn content_hash(&self) -> &Hash {
+        &self.content_hash.0
+    }
+
+    pub fn content_length(&self) -> usize {
+        self.content_length
+    }
+
+    pub fn content_type(&self) -> &str {
+        &self.content_type
     }
 
     // === Public Method ===
 
+    pub fn read_content<'txn>(
+        &self,
+        db: &'txn DB,
+        rtxn: &'txn RoTxn,
+    ) -> anyhow::Result<impl Iterator<Item = Result<&'txn [u8], heed::Error>> + 'txn> {
+        db.read_entry_content(rtxn, self)
+    }
+
     pub fn serialize(&self) -> Vec<u8> {
         to_allocvec(self).expect("Session::serialize")
     }
@@ -313,3 +315,226 @@ impl Entry {
         from_bytes(bytes)
     }
 }
+
+pub struct EntryWriter<'db> {
+    db: &'db DB,
+    buffer: File,
+    hasher: Hasher,
+    buffer_path: PathBuf,
+    entry_key: String,
+    timestamp: Timestamp,
+    is_public: bool,
+}
+
+impl<'db> EntryWriter<'db> {
+    pub fn new(db: &'db DB, public_key: &PublicKey, path: &str) -> anyhow::Result<Self> {
+        let hasher = Hasher::new();
+
+        let timestamp = Timestamp::now();
+
+        let buffer_path = db.buffers_dir.join(timestamp.to_string());
+
+        let buffer = File::create(&buffer_path)?;
+
+        let entry_key = format!("{public_key}/{path}");
+
+        Ok(Self {
+            db,
+            buffer,
+            hasher,
+            buffer_path,
+            entry_key,
+            timestamp,
+            is_public: path.starts_with("pub/"),
+        })
+    }
+
+    /// Same ase [EntryWriter::write_all] but returns a Result of a mutable reference of itself
+    /// to enable chaining with [Self::commit].
+    pub fn update(&mut self, chunk: &[u8]) -> Result<&mut Self, std::io::Error> {
+        self.write_all(chunk)?;
+
+        Ok(self)
+    }
+
+    /// Commit blob from the filesystem buffer to LMDB,
+    /// write the [Entry], and commit the write transaction.
+    pub fn commit(&self) -> anyhow::Result<Entry> {
+        let hash = self.hasher.finalize();
+
+        let mut buffer = File::open(&self.buffer_path)?;
+
+        let mut wtxn = self.db.env.write_txn()?;
+
+        let mut chunk_key = [0; 12];
+        chunk_key[0..8].copy_from_slice(&self.timestamp.to_bytes());
+
+        let mut chunk_index: u32 = 0;
+
+        loop {
+            let mut chunk = vec![0_u8; self.db.max_chunk_size];
+
+            let bytes_read = buffer.read(&mut chunk)?;
+
+            if bytes_read == 0 {
+                break; // EOF reached
+            }
+
+            chunk_key[8..].copy_from_slice(&chunk_index.to_be_bytes());
+
+            self.db
+                .tables
+                .blobs
+                .put(&mut wtxn, &chunk_key, &chunk[..bytes_read])?;
+
+            chunk_index += 1;
+        }
+
+        let mut entry = Entry::new();
+        entry.set_timestamp(&self.timestamp);
+
+        entry.set_content_hash(hash);
+
+        let length = buffer.metadata()?.len();
+        entry.set_content_length(length as usize);
+
+        self.db
+            .tables
+            .entries
+            .put(&mut wtxn, &self.entry_key, &entry.serialize())?;
+
+        // Write a public [Event].
+        if self.is_public {
+            let url = format!("pubky://{}", self.entry_key);
+            let event = Event::put(&url);
+            let value = event.serialize();
+
+            let key = entry.timestamp.to_string();
+
+            self.db.tables.events.put(&mut wtxn, &key, &value)?;
+
+            // TODO: delete events older than a threshold.
+            // TODO: move to events.rs
+        }
+
+        wtxn.commit()?;
+
+        std::fs::remove_file(&self.buffer_path)?;
+
+        Ok(entry)
+    }
+}
+
+impl<'db> std::io::Write for EntryWriter<'db> {
+    /// Write a chunk to a Filesystem based buffer.
+    #[inline]
+    fn write(&mut self, chunk: &[u8]) -> std::io::Result<usize> {
+        self.hasher.update(chunk);
+        self.buffer.write_all(chunk)?;
+
+        Ok(chunk.len())
+    }
+
+    /// Does not do anything, you need to call [Self::commit]
+    #[inline]
+    fn flush(&mut self) -> std::io::Result<()> {
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use bytes::Bytes;
+    use pkarr::{mainline::Testnet, Keypair};
+
+    use crate::config::Config;
+
+    use super::DB;
+
+    #[tokio::test]
+    async fn entries() -> anyhow::Result<()> {
+        let mut db = DB::open(Config::test(&Testnet::new(0))).unwrap();
+
+        let keypair = Keypair::random();
+        let public_key = keypair.public_key();
+        let path = "/pub/foo.txt";
+
+        let chunk = Bytes::from(vec![1, 2, 3, 4, 5]);
+
+        db.write_entry(&public_key, path)?
+            .update(&chunk)?
+            .commit()?;
+
+        let rtxn = db.env.read_txn().unwrap();
+        let entry = db.get_entry(&rtxn, &public_key, path).unwrap().unwrap();
+
+        assert_eq!(
+            entry.content_hash(),
+            &[
+                2, 79, 103, 192, 66, 90, 61, 192, 47, 186, 245, 140, 185, 61, 229, 19, 46, 61, 117,
+                197, 25, 250, 160, 186, 218, 33, 73, 29, 136, 201, 112, 87
+            ]
+        );
+
+        let mut blob = vec![];
+
+        {
+            let mut iter = entry.read_content(&db, &rtxn).unwrap();
+
+            while let Some(Ok(chunk)) = iter.next() {
+                blob.extend_from_slice(&chunk);
+            }
+        }
+
+        assert_eq!(blob, vec![1, 2, 3, 4, 5]);
+
+        rtxn.commit().unwrap();
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn chunked_entry() -> anyhow::Result<()> {
+        let mut db = DB::open(Config::test(&Testnet::new(0))).unwrap();
+
+        let keypair = Keypair::random();
+        let public_key = keypair.public_key();
+        let path = "/pub/foo.txt";
+
+        let chunk = Bytes::from(vec![0; 1024 * 1024]);
+
+        db.write_entry(&public_key, path)?
+            .update(&chunk)?
+            .commit()?;
+
+        let rtxn = db.env.read_txn().unwrap();
+        let entry = db.get_entry(&rtxn, &public_key, path).unwrap().unwrap();
+
+        assert_eq!(
+            entry.content_hash(),
+            &[
+                72, 141, 226, 2, 247, 59, 217, 118, 222, 78, 112, 72, 244, 225, 243, 154, 119, 109,
+                134, 213, 130, 183, 52, 143, 245, 59, 244, 50, 185, 135, 252, 168
+            ]
+        );
+
+        let mut blob = vec![];
+
+        {
+            let mut iter = entry.read_content(&db, &rtxn).unwrap();
+
+            while let Some(Ok(chunk)) = iter.next() {
+                blob.extend_from_slice(&chunk);
+            }
+        }
+
+        assert_eq!(blob, vec![0; 1024 * 1024]);
+
+        let stats = db.tables.blobs.stat(&rtxn).unwrap();
+        assert_eq!(stats.overflow_pages, 0);
+
+        rtxn.commit().unwrap();
+
+        Ok(())
+    }
+}

pubky-homeserver/src/error.rs

@@ -5,6 +5,7 @@ use axum::{
     http::StatusCode,
     response::IntoResponse,
 };
+use tokio::task::JoinError;
 use tracing::debug;
 
 pub type Result<T, E = Error> = core::result::Result<T, E>;
@@ -126,3 +127,24 @@ impl<T> From<flume::SendError<T>> for Error {
         Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into())
     }
 }
+
+impl From<flume::RecvError> for Error {
+    fn from(error: flume::RecvError) -> Self {
+        debug!(?error);
+        Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into())
+    }
+}
+
+impl From<JoinError> for Error {
+    fn from(error: JoinError) -> Self {
+        debug!(?error);
+        Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into())
+    }
+}
+
+impl From<axum::http::Error> for Error {
+    fn from(error: axum::http::Error) -> Self {
+        debug!(?error);
+        Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into())
+    }
+}

pubky-homeserver/src/routes.rs

@@ -1,6 +1,6 @@
 use axum::{
     extract::DefaultBodyLimit,
-    routing::{delete, get, post, put},
+    routing::{delete, get, head, post, put},
     Router,
 };
 use tower_cookies::CookieManagerLayer;
@@ -25,12 +25,13 @@ fn base(state: AppState) -> Router {
         .route("/:pubky/session", delete(auth::signout))
         .route("/:pubky/*path", put(public::put))
         .route("/:pubky/*path", get(public::get))
+        .route("/:pubky/*path", head(public::head))
         .route("/:pubky/*path", delete(public::delete))
         .route("/events/", get(feed::feed))
         .layer(CookieManagerLayer::new())
         // TODO: revisit if we enable streaming big payloads
         // TODO: maybe add to a separate router (drive router?).
-        .layer(DefaultBodyLimit::max(16 * 1024))
+        .layer(DefaultBodyLimit::max(100 * 1024 * 1024))
         .with_state(state)
 }
 

pubky-homeserver/src/routes/auth.rs

@@ -1,7 +1,6 @@
 use axum::{
-    debug_handler,
-    extract::State,
-    http::{uri::Scheme, StatusCode, Uri},
+    extract::{Host, State},
+    http::StatusCode,
     response::IntoResponse,
 };
 use axum_extra::{headers::UserAgent, TypedHeader};
@@ -20,17 +19,16 @@ use crate::{
     server::AppState,
 };
 
-#[debug_handler]
 pub async fn signup(
     State(state): State<AppState>,
     user_agent: Option<TypedHeader<UserAgent>>,
     cookies: Cookies,
-    uri: Uri,
+    host: Host,
     body: Bytes,
 ) -> Result<impl IntoResponse> {
     // TODO: Verify invitation link.
     // TODO: add errors in case of already axisting user.
-    signin(State(state), user_agent, cookies, uri, body).await
+    signin(State(state), user_agent, cookies, host, body).await
 }
 
 pub async fn session(
@@ -89,7 +87,7 @@ pub async fn signin(
     State(state): State<AppState>,
     user_agent: Option<TypedHeader<UserAgent>>,
     cookies: Cookies,
-    uri: Uri,
+    Host(host): Host,
     body: Bytes,
 ) -> Result<impl IntoResponse> {
     let token = state.verifier.verify(&body)?;
@@ -106,7 +104,7 @@ pub async fn signin(
             &mut wtxn,
             public_key,
             &User {
-                created_at: Timestamp::now().into_inner(),
+                created_at: Timestamp::now().as_u64(),
             },
         )?;
     }
@@ -124,7 +122,8 @@ pub async fn signin(
     let mut cookie = Cookie::new(public_key.to_string(), session_secret);
 
     cookie.set_path("/");
-    if *uri.scheme().unwrap_or(&Scheme::HTTP) == Scheme::HTTPS {
+
+    if is_secure(&host) {
         cookie.set_secure(true);
         cookie.set_same_site(SameSite::None);
     }
@@ -136,3 +135,34 @@ pub async fn signin(
 
     Ok(session)
 }
+
+/// Assuming that if the server is addressed by anything other than
+/// localhost, or IP addresses, it is not addressed from a browser in an
+/// secure (HTTPs) window, thus it no need to `secure` and `same_site=none` to cookies
+fn is_secure(host: &str) -> bool {
+    url::Host::parse(host)
+        .map(|host| match host {
+            url::Host::Domain(domain) => domain != "localhost",
+            _ => false,
+        })
+        .unwrap_or_default()
+}
+
+#[cfg(test)]
+mod tests {
+    use pkarr::Keypair;
+
+    use super::*;
+
+    #[test]
+    fn test_is_secure() {
+        assert!(!is_secure(""));
+        assert!(!is_secure("127.0.0.1"));
+        assert!(!is_secure("167.86.102.121"));
+        assert!(!is_secure("[2001:0db8:0000:0000:0000:ff00:0042:8329]"));
+        assert!(!is_secure("localhost"));
+        assert!(!is_secure("localhost:23423"));
+        assert!(is_secure(&Keypair::random().public_key().to_string()));
+        assert!(is_secure("example.com"));
+    }
+}

pubky-homeserver/src/routes/feed.rs

@@ -4,7 +4,7 @@ use axum::{
     http::{header, Response, StatusCode},
     response::IntoResponse,
 };
-use pubky_common::timestamp::{Timestamp, TimestampError};
+use pubky_common::timestamp::Timestamp;
 
 use crate::{
     error::{Error, Result},
@@ -17,17 +17,11 @@ pub async fn feed(
     params: ListQueryParams,
 ) -> Result<impl IntoResponse> {
     if let Some(ref cursor) = params.cursor {
-        if let Err(timestmap_error) = Timestamp::try_from(cursor.to_string()) {
-            let cause = match timestmap_error {
-                TimestampError::InvalidEncoding => {
-                    "Cursor should be valid base32 Crockford encoding of a timestamp"
-                }
-                TimestampError::InvalidBytesLength(size) => {
-                    &format!("Cursor should be 13 characters long, got: {size}")
-                }
-            };
-
-            Err(Error::new(StatusCode::BAD_REQUEST, cause.into()))?
+        if Timestamp::try_from(cursor.to_string()).is_err() {
+            Err(Error::new(
+                StatusCode::BAD_REQUEST,
+                "Cursor should be valid base32 Crockford encoding of a timestamp".into(),
+            ))?
         }
     }
 

pubky-homeserver/src/routes/public.rs

@@ -1,14 +1,18 @@
 use axum::{
-    body::{Body, Bytes},
+    body::Body,
+    debug_handler,
     extract::State,
-    http::{header, Response, StatusCode},
+    http::{header, HeaderMap, HeaderValue, Response, StatusCode},
     response::IntoResponse,
 };
 use futures_util::stream::StreamExt;
+use httpdate::HttpDate;
 use pkarr::PublicKey;
+use std::{io::Write, str::FromStr};
 use tower_cookies::Cookies;
 
 use crate::{
+    database::tables::entries::Entry,
     error::{Error, Result},
     extractors::{EntryPath, ListQueryParams, Pubky},
     server::AppState,
@@ -22,53 +26,37 @@ pub async fn put(
     body: Body,
 ) -> Result<impl IntoResponse> {
     let public_key = pubky.public_key().clone();
-    let path = path.as_str();
+    let path = path.as_str().to_string();
 
-    verify(path)?;
-    authorize(&mut state, cookies, &public_key, path)?;
+    verify(&path)?;
+    authorize(&mut state, cookies, &public_key, &path)?;
+
+    let mut entry_writer = state.db.write_entry(&public_key, &path)?;
 
     let mut stream = body.into_data_stream();
-
-    let (tx, rx) = flume::bounded::<Bytes>(1);
-
-    let path = path.to_string();
-
-    // TODO: refactor Database to clean up this scope.
-    let done = tokio::task::spawn_blocking(move || -> Result<()> {
-        // TODO: this is a blocking operation, which is ok for small
-        // payloads (we have 16 kb limit for now) but later we need
-        // to stream this to filesystem, and keep track of any failed
-        // writes to GC these files later.
-
-        state.db.put_entry(&public_key, &path, rx)?;
-
-        Ok(())
-    });
-
     while let Some(next) = stream.next().await {
         let chunk = next?;
-
-        tx.send(chunk)?;
+        entry_writer.write_all(&chunk)?;
     }
 
-    drop(tx);
-    done.await.expect("join error")?;
+    let _entry = entry_writer.commit()?;
 
     // TODO: return relevant headers, like Etag?
 
     Ok(())
 }
 
+#[debug_handler]
 pub async fn get(
     State(state): State<AppState>,
+    headers: HeaderMap,
     pubky: Pubky,
     path: EntryPath,
     params: ListQueryParams,
 ) -> Result<impl IntoResponse> {
     verify(path.as_str())?;
-    let public_key = pubky.public_key();
-
-    let path = path.as_str();
+    let public_key = pubky.public_key().clone();
+    let path = path.as_str().to_string();
 
     if path.ends_with('/') {
         let txn = state.db.env.read_txn()?;
@@ -95,16 +83,105 @@ pub async fn get(
         return Ok(Response::builder()
             .status(StatusCode::OK)
             .header(header::CONTENT_TYPE, "text/plain")
-            .body(Body::from(vec.join("\n")))
-            .unwrap());
+            .body(Body::from(vec.join("\n")))?);
     }
 
-    // TODO: Enable streaming
+    let (entry_tx, entry_rx) = flume::bounded::<Option<Entry>>(1);
+    let (chunks_tx, chunks_rx) = flume::unbounded::<std::result::Result<Vec<u8>, heed::Error>>();
 
-    match state.db.get_blob(public_key, path) {
-        Err(error) => Err(error)?,
-        Ok(Some(bytes)) => Ok(Response::builder().body(Body::from(bytes)).unwrap()),
-        Ok(None) => Err(Error::new(StatusCode::NOT_FOUND, "File Not Found".into())),
+    tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
+        let rtxn = state.db.env.read_txn()?;
+
+        let option = state.db.get_entry(&rtxn, &public_key, &path)?;
+
+        if let Some(entry) = option {
+            let iter = entry.read_content(&state.db, &rtxn)?;
+
+            entry_tx.send(Some(entry))?;
+
+            for next in iter {
+                chunks_tx.send(next.map(|b| b.to_vec()))?;
+            }
+        };
+
+        entry_tx.send(None)?;
+
+        Ok(())
+    });
+
+    get_entry(
+        headers,
+        entry_rx.recv_async().await?,
+        Some(Body::from_stream(chunks_rx.into_stream())),
+    )
+}
+
+pub async fn head(
+    State(state): State<AppState>,
+    headers: HeaderMap,
+    pubky: Pubky,
+    path: EntryPath,
+) -> Result<impl IntoResponse> {
+    verify(path.as_str())?;
+
+    let rtxn = state.db.env.read_txn()?;
+
+    get_entry(
+        headers,
+        state
+            .db
+            .get_entry(&rtxn, pubky.public_key(), path.as_str())?,
+        None,
+    )
+}
+
+pub fn get_entry(
+    headers: HeaderMap,
+    entry: Option<Entry>,
+    body: Option<Body>,
+) -> Result<Response<Body>> {
+    if let Some(entry) = entry {
+        // TODO: Enable seek API (range requests)
+        // TODO: Gzip? or brotli?
+
+        let mut response = HeaderMap::from(&entry).into_response();
+
+        // Handle IF_MODIFIED_SINCE
+        if let Some(condition_http_date) = headers
+            .get(header::IF_MODIFIED_SINCE)
+            .and_then(|h| h.to_str().ok())
+            .and_then(|s| HttpDate::from_str(s).ok())
+        {
+            let entry_http_date: HttpDate = entry.timestamp().to_owned().into();
+
+            if condition_http_date >= entry_http_date {
+                *response.status_mut() = StatusCode::NOT_MODIFIED;
+            }
+        };
+
+        // Handle IF_NONE_MATCH
+        if let Some(str) = headers
+            .get(header::IF_NONE_MATCH)
+            .and_then(|h| h.to_str().ok())
+        {
+            let etag = format!("\"{}\"", entry.content_hash());
+            if str
+                .trim()
+                .split(',')
+                .collect::<Vec<_>>()
+                .contains(&etag.as_str())
+            {
+                *response.status_mut() = StatusCode::NOT_MODIFIED;
+            };
+        }
+
+        if let Some(body) = body {
+            *response.body_mut() = body;
+        };
+
+        Ok(response)
+    } else {
+        Err(Error::with_status(StatusCode::NOT_FOUND))?
     }
 }
 
@@ -120,14 +197,13 @@ pub async fn delete(
     authorize(&mut state, cookies, &public_key, path)?;
     verify(path)?;
 
+    // TODO: should we wrap this with `tokio::task::spawn_blocking` in case it takes too long?
     let deleted = state.db.delete_entry(&public_key, path)?;
 
     if !deleted {
         // TODO: if the path ends with `/` return a `CONFLICT` error?
         return Err(Error::with_status(StatusCode::NOT_FOUND));
-    }
-
-    // TODO: return relevant headers, like Etag?
+    };
 
     Ok(())
 }
@@ -172,3 +248,133 @@ fn verify(path: &str) -> Result<()> {
 
     Ok(())
 }
+
+impl From<&Entry> for HeaderMap {
+    fn from(entry: &Entry) -> Self {
+        let mut headers = HeaderMap::new();
+        headers.insert(header::CONTENT_LENGTH, entry.content_length().into());
+        headers.insert(
+            header::LAST_MODIFIED,
+            HeaderValue::from_str(&entry.timestamp().format_http_date())
+                .expect("http date is valid header value"),
+        );
+        headers.insert(
+            header::CONTENT_TYPE,
+            // TODO: when setting content type from user input, we should validate it as a HeaderValue
+            entry
+                .content_type()
+                .try_into()
+                .or(HeaderValue::from_str(""))
+                .expect("valid header value"),
+        );
+        headers.insert(
+            header::ETAG,
+            format!("\"{}\"", entry.content_hash())
+                .try_into()
+                .expect("hex string is valid"),
+        );
+
+        headers
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use axum::http::header;
+    use pkarr::{mainline::Testnet, Keypair};
+    use reqwest::{self, Method, StatusCode};
+
+    use crate::Homeserver;
+
+    #[tokio::test]
+    async fn if_last_modified() -> anyhow::Result<()> {
+        let testnet = Testnet::new(3);
+        let mut server = Homeserver::start_test(&testnet).await?;
+
+        let public_key = Keypair::random().public_key();
+
+        let data = &[1, 2, 3, 4, 5];
+
+        server
+            .database_mut()
+            .write_entry(&public_key, "pub/foo")?
+            .update(data)?
+            .commit()?;
+
+        let client = reqwest::Client::builder().build()?;
+
+        let url = format!("http://localhost:{}/{public_key}/pub/foo", server.port());
+
+        let response = client.request(Method::GET, &url).send().await?;
+
+        let response = client
+            .request(Method::GET, &url)
+            .header(
+                header::IF_MODIFIED_SINCE,
+                response.headers().get(header::LAST_MODIFIED).unwrap(),
+            )
+            .send()
+            .await?;
+
+        assert_eq!(response.status(), StatusCode::NOT_MODIFIED);
+
+        let response = client
+            .request(Method::HEAD, &url)
+            .header(
+                header::IF_MODIFIED_SINCE,
+                response.headers().get(header::LAST_MODIFIED).unwrap(),
+            )
+            .send()
+            .await?;
+
+        assert_eq!(response.status(), StatusCode::NOT_MODIFIED);
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn if_none_match() -> anyhow::Result<()> {
+        let testnet = Testnet::new(3);
+        let mut server = Homeserver::start_test(&testnet).await?;
+
+        let public_key = Keypair::random().public_key();
+
+        let data = &[1, 2, 3, 4, 5];
+
+        server
+            .database_mut()
+            .write_entry(&public_key, "pub/foo")?
+            .update(data)?
+            .commit()?;
+
+        let client = reqwest::Client::builder().build()?;
+
+        let url = format!("http://localhost:{}/{public_key}/pub/foo", server.port());
+
+        let response = client.request(Method::GET, &url).send().await?;
+
+        let response = client
+            .request(Method::GET, &url)
+            .header(
+                header::IF_NONE_MATCH,
+                response.headers().get(header::ETAG).unwrap(),
+            )
+            .send()
+            .await?;
+
+        assert_eq!(response.status(), StatusCode::NOT_MODIFIED);
+
+        let response = client
+            .request(Method::HEAD, &url)
+            .header(
+                header::IF_NONE_MATCH,
+                response.headers().get(header::ETAG).unwrap(),
+            )
+            .send()
+            .await?;
+
+        assert_eq!(response.status(), StatusCode::NOT_MODIFIED);
+
+        Ok(())
+    }
+}

pubky-homeserver/src/server.rs

@@ -97,6 +97,11 @@ impl Homeserver {
         self.state.config.keypair().public_key()
     }
 
+    #[cfg(test)]
+    pub(crate) fn database_mut(&mut self) -> &mut DB {
+        &mut self.state.db
+    }
+
     // === Public Methods ===
 
     /// Shutdown the server and wait for all tasks to complete.

pubky/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
 name = "pubky"
-version = "0.1.0"
+version = "0.3.0"
 edition = "2021"
-description = "Pubky client"
+description = "Pubky core client"
 license = "MIT"
 repository = "https://github.com/pubky/pubky"
 keywords = ["web", "dht", "dns", "decentralized", "identity"]
@@ -17,8 +17,8 @@ url = "2.5.2"
 bytes = "^1.7.1"
 base64 = "0.22.1"
 
+pkarr = { git = "https://github.com/Pubky/pkarr", branch = "v3", package = "pkarr" }
 pubky-common = { version = "0.1.0", path = "../pubky-common" }
-pkarr = { workspace = true, features = ["endpoints"] }
 
 # Native dependencies
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
@@ -36,7 +36,7 @@ js-sys = "0.3.69"
 web-sys = "0.3.70"
 
 [dev-dependencies]
-pubky_homeserver = { path = "../pubky-homeserver" }
+pubky-homeserver = { path = "../pubky-homeserver" }
 tokio = "1.37.0"
 
 [features]

pubky/README.md

@@ -0,0 +1,53 @@
+# Pubky
+
+Rust implementation implementation of [Pubky](https://github.com/pubky/pubky-core) client.
+
+## Quick Start
+
+```rust
+use pkarr::mainline::Testnet;
+use pkarr::Keypair;
+use pubky_homeserver::Homeserver;
+use pubky::PubkyClient;
+
+#[tokio::main]
+async fn main () {
+  // Mainline Dht testnet and a temporary homeserver for unit testing.
+  let testnet = Testnet::new(10);
+  let server = Homeserver::start_test(&testnet).await.unwrap();
+
+  let client = PubkyClient::test(&testnet);
+
+  // Uncomment the following line instead if you are not just testing.
+  // let client PubkyClient::builder().build(); 
+
+  // Generate a keypair
+  let keypair = Keypair::random();
+
+  // Signup to a Homeserver
+  let keypair = Keypair::random();
+  client.signup(&keypair, &server.public_key()).await.unwrap();
+
+  // Write data.
+  let url = format!("pubky://{}/pub/foo.txt", keypair.public_key());
+  let url = url.as_str();
+
+  client.put(url, &[0, 1, 2, 3, 4]).await.unwrap();
+
+  // Read using a Public key based link
+  let response = client.get(url).await.unwrap().unwrap();
+
+  assert_eq!(response, bytes::Bytes::from(vec![0, 1, 2, 3, 4]));
+
+  // Delet an entry.
+  client.delete(url).await.unwrap();
+
+  let response = client.get(url).await.unwrap();
+
+  assert_eq!(response, None);
+}
+```
+
+## Example code
+
+Check more [examples](https://github.com/pubky/pubky-core/tree/main/examples) for using the Pubky client.

pubky/pkg/README.md

@@ -1,6 +1,6 @@
 # Pubky
 
-JavaScript implementation of [Pubky](https://github.com/pubky/pubky).
+JavaScript implementation of [Pubky](https://github.com/pubky/pubky-core) client.
 
 ## Table of Contents
 - [Install](#install)

pubky/src/error.rs

@@ -6,7 +6,7 @@ use pkarr::dns::SimpleDnsError;
 pub type Result<T, E = Error> = core::result::Result<T, E>;
 
 #[derive(thiserror::Error, Debug)]
-/// Pk common Error
+/// Pubky crate's common Error enum
 pub enum Error {
     /// For starter, to remove as code matures.
     #[error("Generic error: {0}")]

pubky/src/lib.rs

@@ -1,3 +1,6 @@
+#![doc = include_str!("../README.md")]
+//!
+
 mod error;
 mod shared;
 
@@ -14,6 +17,7 @@ pub use error::Error;
 #[cfg(not(target_arch = "wasm32"))]
 pub use crate::shared::list_builder::ListBuilder;
 
+/// A client for Pubky homeserver API, as well as generic HTTP requests to Pubky urls.
 #[derive(Debug, Clone)]
 #[wasm_bindgen]
 pub struct PubkyClient {

pubky/src/native.rs

@@ -1,15 +1,12 @@
 use std::time::Duration;
-use std::{net::ToSocketAddrs, sync::Arc};
 
-use ::pkarr::mainline::dht::Testnet;
+use pkarr::mainline::Testnet;
 
 use crate::PubkyClient;
 
 mod api;
 mod internals;
 
-use internals::PkarrResolver;
-
 static DEFAULT_USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION"),);
 
 #[derive(Debug, Default)]
@@ -55,12 +52,11 @@ impl Settings {
         // TODO: convert to Result<PubkyClient>
 
         let pkarr = pkarr::Client::new(self.pkarr_settings).unwrap();
-        let dns_resolver: PkarrResolver = (&pkarr).into();
 
         PubkyClient {
             http: reqwest::Client::builder()
                 .cookie_store(true)
-                .dns_resolver(Arc::new(dns_resolver))
+                // .dns_resolver(Arc::new(dns_resolver))
                 .user_agent(DEFAULT_USER_AGENT)
                 .build()
                 .unwrap(),
@@ -94,7 +90,7 @@ impl PubkyClient {
     /// - DHT bootstrap nodes set to the `testnet` bootstrap nodes.
     /// - DHT request timout set to 500 milliseconds. (unless in CI, then it is left as default 2000)
     ///
-    /// For more control, you can use [PubkyClientBuilder::testnet]
+    /// For more control, you can use [PubkyClient::builder] testnet option.
     pub fn test(testnet: &Testnet) -> PubkyClient {
         let mut builder = PubkyClient::builder().testnet(testnet);
 

pubky/src/native/internals.rs

@@ -3,38 +3,6 @@ use url::Url;
 
 use crate::PubkyClient;
 
-use std::net::ToSocketAddrs;
-
-use pkarr::{Client, EndpointResolver, PublicKey};
-use reqwest::dns::{Addrs, Resolve};
-
-pub struct PkarrResolver(Client);
-
-impl Resolve for PkarrResolver {
-    fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving {
-        let client = self.0.clone();
-
-        Box::pin(async move {
-            let name = name.as_str();
-
-            if PublicKey::try_from(name).is_ok() {
-                let endpoint = client.resolve_endpoint(name).await?;
-
-                let addrs: Addrs = Box::new(endpoint.to_socket_addrs().into_iter());
-                return Ok(addrs);
-            };
-
-            Ok(Box::new(format!("{name}:0").to_socket_addrs().unwrap()))
-        })
-    }
-}
-
-impl From<&pkarr::Client> for PkarrResolver {
-    fn from(pkarr: &pkarr::Client) -> Self {
-        PkarrResolver(pkarr.clone())
-    }
-}
-
 impl PubkyClient {
     // === HTTP ===
 

pubky/src/shared/auth.rs

@@ -149,20 +149,7 @@ impl PubkyClient {
         Ok(())
     }
 
-    pub async fn inner_third_party_signin(
-        &self,
-        encrypted_token: &[u8],
-        client_secret: &[u8; 32],
-    ) -> Result<PublicKey> {
-        let decrypted = decrypt(encrypted_token, client_secret)?;
-        let token = AuthToken::deserialize(&decrypted)?;
-
-        self.signin_with_authtoken(&token).await?;
-
-        Ok(token.pubky().to_owned())
-    }
-
-    pub async fn signin_with_authtoken(&self, token: &AuthToken) -> Result<Session> {
+    pub(crate) async fn signin_with_authtoken(&self, token: &AuthToken) -> Result<Session> {
         let mut url = Url::parse(&format!("https://{}/session", token.pubky()))?;
 
         self.resolve_url(&mut url).await?;

pubky/src/shared/list_builder.rs

@@ -3,6 +3,7 @@ use url::Url;
 
 use crate::{error::Result, PubkyClient};
 
+/// Helper struct to edit Pubky homeserver's list API options before sending them.
 #[derive(Debug)]
 pub struct ListBuilder<'a> {
     url: Url,

pubky/src/shared/pkarr.rs

@@ -37,7 +37,7 @@ impl PubkyClient {
             "_pubky".try_into().unwrap(),
             pkarr::dns::CLASS::IN,
             60 * 60,
-            pkarr::dns::rdata::RData::SVCB(svcb),
+            pkarr::dns::rdata::RData::HTTPS(svcb.into()),
         ));
 
         let signed_packet = SignedPacket::from_packet(keypair, &packet)?;

pubky/src/shared/public.rs

@@ -99,6 +99,7 @@ mod tests {
 
     use crate::*;
 
+    use bytes::Bytes;
     use pkarr::{mainline::Testnet, Keypair};
     use pubky_homeserver::Homeserver;
     use reqwest::{Method, StatusCode};
@@ -751,7 +752,7 @@ mod tests {
             );
         }
 
-        let get = client.get(url.as_str()).await.unwrap().unwrap();
+        let get = client.get(url).await.unwrap().unwrap();
 
         assert_eq!(get.as_ref(), &[0]);
     }
@@ -808,4 +809,35 @@ mod tests {
             ]
         )
     }
+
+    #[tokio::test]
+    async fn stream() {
+        // TODO: test better streaming API
+
+        let testnet = Testnet::new(10);
+        let server = Homeserver::start_test(&testnet).await.unwrap();
+
+        let client = PubkyClient::test(&testnet);
+
+        let keypair = Keypair::random();
+
+        client.signup(&keypair, &server.public_key()).await.unwrap();
+
+        let url = format!("pubky://{}/pub/foo.txt", keypair.public_key());
+        let url = url.as_str();
+
+        let bytes = Bytes::from(vec![0; 1024 * 1024]);
+
+        client.put(url, &bytes).await.unwrap();
+
+        let response = client.get(url).await.unwrap().unwrap();
+
+        assert_eq!(response, bytes);
+
+        client.delete(url).await.unwrap();
+
+        let response = client.get(url).await.unwrap();
+
+        assert_eq!(response, None);
+    }
 }