From 5e88cd328e0e2b0a85f75d3fd9c8915003e5d648 Mon Sep 17 00:00:00 2001 From: kernelkind Date: Mon, 20 Oct 2025 16:36:25 -0400 Subject: [PATCH 1/8] fix(thread): remove flicker on opening thread fixed bug in `egui-nav` Signed-off-by: kernelkind --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 545167e..a63e7cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1555,7 +1555,7 @@ dependencies = [ [[package]] name = "egui_nav" version = "0.2.0" -source = "git+https://github.com/damus-io/egui-nav?rev=6b4b96bae35270434abd69b24fa9943edc3f5b0b#6b4b96bae35270434abd69b24fa9943edc3f5b0b" +source = "git+https://github.com/kernelkind/egui-nav?rev=15304033930e4cb8ccae9551b439fb958732fc66#15304033930e4cb8ccae9551b439fb958732fc66" dependencies = [ "bitflags 2.9.1", "egui", diff --git a/Cargo.toml b/Cargo.toml index 110b79c..15dd1d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ egui = { version = "0.31.1", features = ["serde"] } egui-wgpu = "0.31.1" egui_extras = { version = "0.31.1", features = ["all_loaders"] } egui-winit = { version = "0.31.1", features = ["android-game-activity", "clipboard"] } -egui_nav = { git = "https://github.com/damus-io/egui-nav", rev = "6b4b96bae35270434abd69b24fa9943edc3f5b0b" } +egui_nav = { git = "https://github.com/kernelkind/egui-nav", rev = "15304033930e4cb8ccae9551b439fb958732fc66" } egui_tabs = { git = "https://github.com/damus-io/egui-tabs", rev = "6eb91740577b374a8a6658c09c9a4181299734d0" } #egui_virtual_list = "0.6.0" egui_virtual_list = { git = "https://github.com/jb55/hello_egui", rev = "a66b6794f5e707a2f4109633770e02b02fb722e1" } From dbba2a5271b7173172fbc09aa32ed0a33fcaa25b Mon Sep 17 00:00:00 2001 From: kernelkind Date: Wed, 22 Oct 2025 16:36:18 -0400 Subject: [PATCH 2/8] Revert "fix: nav drawer shadow extends all the way vertically" This reverts commit df5cf8a1fc28a78bcb58b892717a85835da01591. --- crates/notedeck_chrome/src/chrome.rs | 39 ++++++++++++---------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/crates/notedeck_chrome/src/chrome.rs b/crates/notedeck_chrome/src/chrome.rs index 11818ad..5baceb0 100644 --- a/crates/notedeck_chrome/src/chrome.rs +++ b/crates/notedeck_chrome/src/chrome.rs @@ -301,29 +301,24 @@ impl Chrome { // if the soft keyboard is open, shrink the chrome contents let mut action: Option = None; - - if keyboard_height == 0.0 { - action = self.panel(ctx, ui, keyboard_height); - } else { - // build a strip to carve out the soft keyboard inset - StripBuilder::new(ui) - .size(Size::remainder()) - .size(Size::exact(keyboard_height)) - .vertical(|mut strip| { - // the actual content, shifted up because of the soft keyboard - strip.cell(|ui| { - action = self.panel(ctx, ui, keyboard_height); - }); - - // the filler space taken up by the soft keyboard - strip.cell(|ui| { - // keyboard-visibility virtual keyboard - if virtual_keyboard && keyboard_height > 0.0 { - virtual_keyboard_ui(ui, ui.available_rect_before_wrap()) - } - }); + // build a strip to carve out the soft keyboard inset + StripBuilder::new(ui) + .size(Size::remainder()) + .size(Size::exact(keyboard_height)) + .vertical(|mut strip| { + // the actual content, shifted up because of the soft keyboard + strip.cell(|ui| { + action = self.panel(ctx, ui, keyboard_height); }); - } + + // the filler space taken up by the soft keyboard + strip.cell(|ui| { + // keyboard-visibility virtual keyboard + if virtual_keyboard && keyboard_height > 0.0 { + virtual_keyboard_ui(ui, ui.available_rect_before_wrap()) + } + }); + }); // hovering virtual keyboard if virtual_keyboard { From db9005e4034268b0504c4bf133f82a5f16d920f3 Mon Sep 17 00:00:00 2001 From: kernelkind Date: Wed, 22 Oct 2025 16:40:22 -0400 Subject: [PATCH 3/8] fix(nav-drawer): shadow extends all the way vertically df5cf8a1fc28a78bcb58b892717a85835da01591 caused a regression making the soft keyboard auto close. This patch extends the shadow all the way vertically without triggering the regression Signed-off-by: kernelkind --- crates/notedeck_chrome/src/chrome.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/notedeck_chrome/src/chrome.rs b/crates/notedeck_chrome/src/chrome.rs index 5baceb0..f734523 100644 --- a/crates/notedeck_chrome/src/chrome.rs +++ b/crates/notedeck_chrome/src/chrome.rs @@ -302,6 +302,8 @@ impl Chrome { // if the soft keyboard is open, shrink the chrome contents let mut action: Option = None; // build a strip to carve out the soft keyboard inset + let prev_spacing = ui.spacing().item_spacing; + ui.spacing_mut().item_spacing.y = 0.0; StripBuilder::new(ui) .size(Size::remainder()) .size(Size::exact(keyboard_height)) @@ -319,6 +321,7 @@ impl Chrome { } }); }); + ui.spacing_mut().item_spacing = prev_spacing; // hovering virtual keyboard if virtual_keyboard { From 9ccbaf2db8721331b49c975118168a23e7b57d0f Mon Sep 17 00:00:00 2001 From: kernelkind Date: Sun, 19 Oct 2025 20:21:18 -0400 Subject: [PATCH 4/8] chore(tracy): repaint every frame since we stop rendering when there is no user input, tracy sees big hangs, and it's annoying to parse through which frames are actual performance issues and which are due to no user input. So just repaint every frame while using tracy. Signed-off-by: kernelkind --- crates/notedeck_chrome/src/chrome.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/notedeck_chrome/src/chrome.rs b/crates/notedeck_chrome/src/chrome.rs index 11818ad..830f3cb 100644 --- a/crates/notedeck_chrome/src/chrome.rs +++ b/crates/notedeck_chrome/src/chrome.rs @@ -345,6 +345,11 @@ impl Chrome { impl notedeck::App for Chrome { fn update(&mut self, ctx: &mut notedeck::AppContext, ui: &mut egui::Ui) -> AppResponse { + #[cfg(feature = "tracy")] + { + ui.ctx().request_repaint(); + } + if let Some(action) = self.show(ctx, ui) { action.process(ctx, self, ui); self.nav.close(); From 892d77d4e3f576bc9103398ba24f947d701fbb0d Mon Sep 17 00:00:00 2001 From: kernelkind Date: Fri, 24 Oct 2025 10:43:31 -0400 Subject: [PATCH 5/8] chore(profiling): markup composite render path Signed-off-by: kernelkind --- crates/notedeck/src/imgcache.rs | 1 + crates/notedeck/src/urls.rs | 9 +++++- crates/notedeck_columns/src/ui/timeline.rs | 33 +++++++++++++++------- crates/notedeck_ui/src/profile/picture.rs | 3 ++ 4 files changed, 35 insertions(+), 11 deletions(-) diff --git a/crates/notedeck/src/imgcache.rs b/crates/notedeck/src/imgcache.rs index ab3f5e3..3efebc0 100644 --- a/crates/notedeck/src/imgcache.rs +++ b/crates/notedeck/src/imgcache.rs @@ -545,6 +545,7 @@ pub struct LatestTexture { pub request_next_repaint: Option, } +#[profiling::function] pub fn get_render_state<'a>( ctx: &egui::Context, images: &'a mut Images, diff --git a/crates/notedeck/src/urls.rs b/crates/notedeck/src/urls.rs index 9024900..67ce6e9 100644 --- a/crates/notedeck/src/urls.rs +++ b/crates/notedeck/src/urls.rs @@ -231,6 +231,7 @@ pub struct SupportedMimeType { } impl SupportedMimeType { + #[profiling::function] pub fn from_extension(extension: &str) -> Result { if let Some(mime) = mime_guess::from_ext(extension) .first() @@ -269,8 +270,13 @@ fn is_mime_supported(mime: &mime_guess::Mime) -> bool { mime.type_() == mime_guess::mime::IMAGE } +#[profiling::function] fn url_has_supported_mime(url: &str) -> MimeHostedAtUrl { - if let Ok(url) = Url::parse(url) { + let url = { + profiling::scope!("url parse"); + Url::parse(url) + }; + if let Ok(url) = url { if let Some(mut path) = url.path_segments() { if let Some(file_name) = path.next_back() { if let Some(ext) = std::path::Path::new(file_name) @@ -289,6 +295,7 @@ fn url_has_supported_mime(url: &str) -> MimeHostedAtUrl { MimeHostedAtUrl::Maybe } +#[profiling::function] pub fn supported_mime_hosted_at_url(urls: &mut UrlMimes, url: &str) -> Option { match url_has_supported_mime(url) { MimeHostedAtUrl::Yes(cache_type) => Some(cache_type), diff --git a/crates/notedeck_columns/src/ui/timeline.rs b/crates/notedeck_columns/src/ui/timeline.rs index b526f6e..914a5e9 100644 --- a/crates/notedeck_columns/src/ui/timeline.rs +++ b/crates/notedeck_columns/src/ui/timeline.rs @@ -705,16 +705,22 @@ fn render_reaction_cluster( underlying_note: &Note, reaction: &ReactionUnit, ) -> RenderEntryResponse { - let profiles_to_show: Vec = reaction - .reactions - .values() - .filter(|r| !mute.is_pk_muted(r.sender.bytes())) - .map(|r| &r.sender) - .map(|p| ProfileEntry { - record: note_context.ndb.get_profile_by_pubkey(txn, p.bytes()).ok(), - pk: p, - }) - .collect(); + let profiles_to_show: Vec = { + profiling::scope!("vec profile entries"); + reaction + .reactions + .values() + .filter(|r| !mute.is_pk_muted(r.sender.bytes())) + .map(|r| &r.sender) + .map(|p| { + profiling::scope!("ndb by pubkey"); + ProfileEntry { + record: note_context.ndb.get_profile_by_pubkey(txn, p.bytes()).ok(), + pk: p, + } + }) + .collect() + }; render_composite_entry( ui, @@ -728,6 +734,7 @@ fn render_reaction_cluster( } #[allow(clippy::too_many_arguments)] +#[profiling::function] fn render_composite_entry( ui: &mut egui::Ui, note_context: &mut NoteContext, @@ -772,6 +779,7 @@ fn render_composite_entry( .show(ui, |ui| { let show_label_newline = ui .horizontal_wrapped(|ui| { + profiling::scope!("header"); let pfps_resp = ui .allocate_ui_with_layout( vec2(ui.available_width(), 32.0), @@ -832,6 +840,7 @@ fn render_composite_entry( .inner; if let Some(desc) = show_label_newline { + profiling::scope!("description"); ui.add_space(4.0); ui.horizontal(|ui| { ui.add_space(48.0); @@ -868,6 +877,7 @@ fn render_composite_entry( RenderEntryResponse::Success(action) } +#[profiling::function] fn render_profiles( ui: &mut egui::Ui, profiles_to_show: Vec, @@ -895,11 +905,14 @@ fn render_profiles( } let resp = ui.horizontal(|ui| { + profiling::scope!("scroll area"); ScrollArea::horizontal() .scroll_bar_visibility(ScrollBarVisibility::AlwaysHidden) .show(ui, |ui| { + profiling::scope!("scroll area closure"); let mut last_pfp_resp = None; for entry in profiles_to_show { + profiling::scope!("actual rendering individual pfp"); let mut resp = ui.add( &mut ProfilePic::from_profile_or_default(img_cache, entry.record.as_ref()) .size(24.0) diff --git a/crates/notedeck_ui/src/profile/picture.rs b/crates/notedeck_ui/src/profile/picture.rs index b6a8fb4..d0ce095 100644 --- a/crates/notedeck_ui/src/profile/picture.rs +++ b/crates/notedeck_ui/src/profile/picture.rs @@ -144,9 +144,11 @@ fn render_pfp( match cur_state.texture_state { notedeck::TextureState::Pending => { + profiling::scope!("Render pending"); egui::InnerResponse::new(None, paint_circle(ui, ui_size, border, sense)) } notedeck::TextureState::Error(e) => { + profiling::scope!("Render error"); let r = paint_circle(ui, ui_size, border, sense); show_one_error_message(ui, &format!("Failed to fetch profile at url {url}: {e}")); egui::InnerResponse::new( @@ -159,6 +161,7 @@ fn render_pfp( ) } notedeck::TextureState::Loaded(textured_image) => { + profiling::scope!("Render loaded"); let texture_handle = ensure_latest_texture(ui, url, cur_state.gifs, textured_image, animation_mode); From fdde0244e284fc8f8a061f67658bbe2b036c726f Mon Sep 17 00:00:00 2001 From: kernelkind Date: Fri, 24 Oct 2025 10:58:48 -0400 Subject: [PATCH 6/8] feat(reactions): use ProfileKey when possible for performance Signed-off-by: kernelkind --- .../notedeck_columns/src/timeline/note_units.rs | 1 + .../src/timeline/timeline_units.rs | 6 ++++++ crates/notedeck_columns/src/timeline/unit.rs | 2 ++ crates/notedeck_columns/src/ui/timeline.rs | 17 ++++++++++------- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/crates/notedeck_columns/src/timeline/note_units.rs b/crates/notedeck_columns/src/timeline/note_units.rs index bdd61f4..6d74788 100644 --- a/crates/notedeck_columns/src/timeline/note_units.rs +++ b/crates/notedeck_columns/src/timeline/note_units.rs @@ -370,6 +370,7 @@ mod tests { reaction: Reaction { reaction: "+".to_owned(), sender: self.random_sender(), + sender_profilekey: None, }, })) } diff --git a/crates/notedeck_columns/src/timeline/timeline_units.rs b/crates/notedeck_columns/src/timeline/timeline_units.rs index cc10e35..74ee558 100644 --- a/crates/notedeck_columns/src/timeline/timeline_units.rs +++ b/crates/notedeck_columns/src/timeline/timeline_units.rs @@ -169,6 +169,11 @@ fn to_reaction<'a>( created_at: reacted_to_note.created_at(), }; + let sender_profilekey = ndb + .get_profile_by_pubkey(txn, payload.note.pubkey()) + .ok() + .and_then(|p| p.key()); + Some(ReactionResponse { fragment: ReactionFragment { noteref_reacted_to, @@ -176,6 +181,7 @@ fn to_reaction<'a>( reaction: Reaction { reaction: reaction.to_string(), sender: Pubkey::new(*payload.note.pubkey()), + sender_profilekey, }, }, pk: payload.note.pubkey(), diff --git a/crates/notedeck_columns/src/timeline/unit.rs b/crates/notedeck_columns/src/timeline/unit.rs index 1c4a8d4..49fca28 100644 --- a/crates/notedeck_columns/src/timeline/unit.rs +++ b/crates/notedeck_columns/src/timeline/unit.rs @@ -1,6 +1,7 @@ use std::collections::{BTreeMap, HashSet}; use enostr::Pubkey; +use nostrdb::ProfileKey; use notedeck::NoteRef; use crate::timeline::note_units::{CompositeKey, CompositeType, UnitKey}; @@ -275,6 +276,7 @@ impl ReactionFragment { pub struct Reaction { pub reaction: String, // can't use char because some emojis are 'grapheme clusters' pub sender: Pubkey, + pub sender_profilekey: Option, } /// Represents a singular repost diff --git a/crates/notedeck_columns/src/ui/timeline.rs b/crates/notedeck_columns/src/ui/timeline.rs index 914a5e9..cd2c33b 100644 --- a/crates/notedeck_columns/src/ui/timeline.rs +++ b/crates/notedeck_columns/src/ui/timeline.rs @@ -711,13 +711,16 @@ fn render_reaction_cluster( .reactions .values() .filter(|r| !mute.is_pk_muted(r.sender.bytes())) - .map(|r| &r.sender) - .map(|p| { - profiling::scope!("ndb by pubkey"); - ProfileEntry { - record: note_context.ndb.get_profile_by_pubkey(txn, p.bytes()).ok(), - pk: p, - } + .map(|r| (&r.sender, r.sender_profilekey)) + .map(|(p, key)| { + let record = if let Some(key) = key { + profiling::scope!("ndb by key"); + note_context.ndb.get_profile_by_key(txn, key).ok() + } else { + profiling::scope!("ndb by pubkey"); + note_context.ndb.get_profile_by_pubkey(txn, p.bytes()).ok() + }; + ProfileEntry { record, pk: p } }) .collect() }; From 1244be4481c51e82cda12f302c7c647c1c55dd4b Mon Sep 17 00:00:00 2001 From: kernelkind Date: Fri, 24 Oct 2025 10:59:32 -0400 Subject: [PATCH 7/8] feat(composite-cluster): do culling for pfps Signed-off-by: kernelkind --- crates/notedeck_columns/src/ui/timeline.rs | 34 ++++++++++++++++------ 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/crates/notedeck_columns/src/ui/timeline.rs b/crates/notedeck_columns/src/ui/timeline.rs index cd2c33b..9aeaf64 100644 --- a/crates/notedeck_columns/src/ui/timeline.rs +++ b/crates/notedeck_columns/src/ui/timeline.rs @@ -913,14 +913,30 @@ fn render_profiles( .scroll_bar_visibility(ScrollBarVisibility::AlwaysHidden) .show(ui, |ui| { profiling::scope!("scroll area closure"); - let mut last_pfp_resp = None; + let clip_rect = ui.clip_rect(); + let mut last_resp = None; + + let mut rendered = false; for entry in profiles_to_show { + let (rect, _) = ui.allocate_exact_size(vec2(24.0, 24.0), Sense::click()); + let should_render = rect.intersects(clip_rect); + + if !should_render { + if rendered { + break; + } else { + continue; + } + } + profiling::scope!("actual rendering individual pfp"); - let mut resp = ui.add( - &mut ProfilePic::from_profile_or_default(img_cache, entry.record.as_ref()) + + let mut widget = + ProfilePic::from_profile_or_default(img_cache, entry.record.as_ref()) .size(24.0) - .sense(Sense::click()), - ); + .sense(Sense::click()); + let mut resp = ui.put(rect, &mut widget); + rendered = true; if let Some(record) = entry.record.as_ref() { resp = resp.on_hover_ui_at_pointer(|ui| { @@ -929,14 +945,14 @@ fn render_profiles( }); } - last_pfp_resp = Some(resp.clone()); - if resp.clicked() { - action = Some(NoteAction::Profile(*entry.pk)) + action = Some(NoteAction::Profile(*entry.pk)); } + + last_resp = Some(resp); } - last_pfp_resp + last_resp }) .inner }); From 3e39cf785b8d98bde8e58c53bf1de1abe7cf16ac Mon Sep 17 00:00:00 2001 From: kernelkind Date: Thu, 23 Oct 2025 21:10:47 -0400 Subject: [PATCH 8/8] feat(mime-cache): upgrade UrlMimes 1. more performant. No more deserialization every frame 2. employs TTL (so cache doesn't grow unbounded) 3. exponential backoff to retry on error Signed-off-by: kernelkind --- crates/notedeck/src/urls.rs | 375 ++++++++++++++++++++++++++++++------ 1 file changed, 321 insertions(+), 54 deletions(-) diff --git a/crates/notedeck/src/urls.rs b/crates/notedeck/src/urls.rs index 67ce6e9..5920a14 100644 --- a/crates/notedeck/src/urls.rs +++ b/crates/notedeck/src/urls.rs @@ -4,23 +4,90 @@ use std::{ io::{Read, Write}, path::PathBuf, sync::{Arc, RwLock}, - time::{Duration, SystemTime}, + time::{Duration, SystemTime, UNIX_EPOCH}, }; -use egui::TextBuffer; +use mime_guess::Mime; use poll_promise::Promise; +use serde::{Deserialize, Serialize}; +use tracing::trace; use url::Url; use crate::{Error, MediaCacheType}; const FILE_NAME: &str = "urls.bin"; const SAVE_INTERVAL: Duration = Duration::from_secs(60); +const MIME_TTL: Duration = Duration::from_secs(60 * 60 * 24 * 7); // one week +const FAILURE_BACKOFF_BASE: Duration = Duration::from_secs(4); +const FAILURE_BACKOFF_MAX: Duration = Duration::from_secs(60 * 60 * 6); +const FAILURE_BACKOFF_EXPONENT_LIMIT: u32 = 10; -type UrlsToMime = HashMap; +type UrlsToMime = HashMap; + +#[derive(Clone, Serialize, Deserialize)] +struct StoredMimeEntry { + entry: MimeEntry, + last_updated_secs: u64, +} + +#[derive(Clone, Serialize, Deserialize, Debug)] +enum MimeEntry { + Mime(String), + Fail { count: u32 }, +} + +impl StoredMimeEntry { + fn new_mime(mime: String, last_updated: SystemTime) -> Self { + Self { + entry: MimeEntry::Mime(mime), + last_updated_secs: system_time_to_secs(last_updated), + } + } + + fn new_failure(count: u32, last_updated: SystemTime) -> Self { + Self { + entry: MimeEntry::Fail { count }, + last_updated_secs: system_time_to_secs(last_updated), + } + } + + fn last_updated(&self) -> SystemTime { + UNIX_EPOCH + Duration::from_secs(self.last_updated_secs) + } + + fn expires_at(&self) -> SystemTime { + let ttl = match &self.entry { + MimeEntry::Mime(_) => MIME_TTL, + MimeEntry::Fail { count } => failure_backoff_duration(*count), + }; + + self.last_updated() + .checked_add(ttl) + .unwrap_or(SystemTime::UNIX_EPOCH) + } + + fn is_expired(&self, now: SystemTime) -> bool { + self.expires_at() <= now + } + + fn failure_count(&self) -> Option { + match &self.entry { + MimeEntry::Fail { count } => Some(*count), + _ => None, + } + } +} + +#[derive(Clone)] +struct CachedMime { + mime: Option, + expires_at: SystemTime, +} /// caches mime type for a URL. saves to disk on interval [`SAVE_INTERVAL`] pub struct UrlCache { last_saved: SystemTime, + last_pruned: SystemTime, path: PathBuf, cache: Arc>, from_disk_promise: Option>>, @@ -34,19 +101,29 @@ impl UrlCache { pub fn new(path: PathBuf) -> Self { Self { last_saved: SystemTime::now(), + last_pruned: SystemTime::now(), path: path.clone(), cache: Default::default(), from_disk_promise: Some(read_from_disk(path)), } } - pub fn get_type(&self, url: &str) -> Option { + fn get_entry(&self, url: &str) -> Option { self.cache.read().ok()?.get(url).cloned() } - pub fn set_type(&mut self, url: String, mime_type: String) { + fn set_entry(&mut self, url: String, entry: StoredMimeEntry) { + if url.is_empty() { + return; + } if let Ok(mut locked_cache) = self.cache.write() { - locked_cache.insert(url, mime_type); + locked_cache.insert(url, entry); + } + } + + fn remove(&mut self, url: &str) { + if let Ok(mut locked_cache) = self.cache.write() { + locked_cache.remove(url); } } @@ -67,6 +144,13 @@ impl UrlCache { self.last_saved = SystemTime::now(); } } + + if let Ok(cur_duration) = SystemTime::now().duration_since(self.last_pruned) { + if cur_duration >= SAVE_INTERVAL { + self.purge_expired(SystemTime::now()); + self.last_pruned = SystemTime::now(); + } + } } pub fn clear(&mut self) { @@ -79,10 +163,22 @@ impl UrlCache { }); } } + + fn purge_expired(&self, now: SystemTime) { + let cache = self.cache.clone(); + std::thread::spawn(move || { + if let Ok(mut locked_cache) = cache.write() { + locked_cache.retain(|_, entry| !entry.is_expired(now)); + } + }); + } } -fn merge_cache(cur_cache: Arc>, from_disk: UrlsToMime) { +fn merge_cache(cur_cache: Arc>, mut from_disk: UrlsToMime) { std::thread::spawn(move || { + let now = SystemTime::now(); + from_disk.retain(|_, entry| !entry.is_expired(now)); + if let Ok(mut locked_cache) = cur_cache.write() { locked_cache.extend(from_disk); } @@ -97,9 +193,28 @@ fn read_from_disk(path: PathBuf) -> Promise> { let mut file = File::open(path)?; let mut buffer = Vec::new(); file.read_to_end(&mut buffer)?; - let data: UrlsToMime = - bincode::deserialize(&buffer).map_err(|e| Error::Generic(e.to_string()))?; - Ok(data) + if buffer.is_empty() { + return Ok(Default::default()); + } + + match bincode::deserialize::(&buffer) { + Ok(data) => { + trace!("Got {} mime entries", data.len()); + Ok(data) + } + Err(err) => { + tracing::debug!("Unable to deserialize UrlMimes with new format: {err}. Attempting legacy fallback."); + let legacy: HashMap = + bincode::deserialize(&buffer).map_err(|e| Error::Generic(e.to_string()))?; + trace!("legacy fallback has {} entries", legacy.len()); + let now = SystemTime::now(); + let migrated = legacy + .into_iter() + .map(|(url, mime)| (url, StoredMimeEntry::new_mime(mime, now))) + .collect(); + Ok(migrated) + } + } })(); match result { @@ -119,12 +234,13 @@ fn save_to_disk(path: PathBuf, cache: Arc>) { let result: Result<(), Error> = (|| { if let Ok(cache) = cache.read() { let cache = &*cache; + let num_items = cache.len(); let encoded = bincode::serialize(cache).map_err(|e| Error::Generic(e.to_string()))?; let mut file = File::create(&path)?; file.write_all(&encoded)?; file.sync_all()?; - tracing::debug!("Saved UrlCache to disk."); + tracing::debug!("Saved UrlCache with {num_items} mimes to disk."); Ok(()) } else { Err(Error::Generic( @@ -139,6 +255,26 @@ fn save_to_disk(path: PathBuf, cache: Arc>) { }); } +fn system_time_to_secs(time: SystemTime) -> u64 { + time.duration_since(UNIX_EPOCH) + .unwrap_or_else(|_| Duration::from_secs(0)) + .as_secs() +} + +fn failure_backoff_duration(count: u32) -> Duration { + if count == 0 { + return FAILURE_BACKOFF_BASE; + } + + let exponent = count.saturating_sub(1).min(FAILURE_BACKOFF_EXPONENT_LIMIT); + let base_secs = FAILURE_BACKOFF_BASE.as_secs().max(1); + let multiplier = 1u64 << exponent; + let delay_secs = base_secs.saturating_mul(multiplier); + let max_secs = FAILURE_BACKOFF_MAX.as_secs(); + + Duration::from_secs(delay_secs.min(max_secs)) +} + fn ehttp_get_mime_type(url: &str, sender: poll_promise::Sender) { let request = ehttp::Request::head(url); @@ -181,6 +317,7 @@ fn extract_mime_type(content_type: &str) -> &str { pub struct UrlMimes { pub cache: UrlCache, in_flight: HashMap>, + mime_cache: HashMap, } impl UrlMimes { @@ -188,41 +325,169 @@ impl UrlMimes { Self { cache: url_cache, in_flight: Default::default(), + mime_cache: Default::default(), } } - pub fn get(&mut self, url: &str) -> Option { - if let Some(mime_type) = self.cache.get_type(url) { - Some(mime_type) - } else if let Some(promise) = self.in_flight.get_mut(url) { - if let Some(mime_result) = promise.ready_mut() { - match mime_result { - Ok(mime_type) => { - let mime_type = mime_type.take(); - self.cache.set_type(url.to_owned(), mime_type.clone()); - self.in_flight.remove(url); - Some(mime_type) - } - Err(HttpError::HttpFailure) => { - // allow retrying - //self.in_flight.remove(url); - None - } - Err(HttpError::MissingHeader) => { - // response was malformed, don't retry - None - } - } - } else { - None + pub fn get_or_fetch(&mut self, url: &str) -> Option<&Mime> { + let now = SystemTime::now(); + + if let Some(cached) = self.mime_cache.get(url) { + if cached.expires_at > now { + return self + .mime_cache + .get(url) + .and_then(|cached| cached.mime.as_ref()); + } + + tracing::trace!("mime {:?} at url {url} has expired", cached.mime); + + self.mime_cache.remove(url); + } + + let stored_entry = self.cache.get_entry(url); + let previous_failure_count = stored_entry + .as_ref() + .and_then(|entry| entry.failure_count()) + .unwrap_or(0); + + if let Some(entry) = stored_entry.as_ref() { + if !entry.is_expired(now) { + return match &entry.entry { + MimeEntry::Mime(mime_string) => match mime_string.parse::() { + Ok(mime) => { + let expires_at = entry.expires_at(); + trace!("inserted {mime:?} in mime cache for {url}"); + self.mime_cache.insert( + url.to_owned(), + CachedMime { + mime: Some(mime), + expires_at, + }, + ); + self.mime_cache + .get(url) + .and_then(|cached| cached.mime.as_ref()) + } + Err(err) => { + tracing::warn!("Failed to parse mime '{mime_string}' for {url}: {err}"); + self.record_failure( + url, + previous_failure_count.saturating_add(1), + SystemTime::now(), + ); + None + } + }, + MimeEntry::Fail { .. } => { + trace!("Read failure from storage for {url}, wrote None to cache"); + + let expires_at = entry.expires_at(); + self.mime_cache.insert( + url.to_owned(), + CachedMime { + mime: None, + expires_at, + }, + ); + None + } + }; + } + + if !matches!(entry.entry, MimeEntry::Fail { count: _ }) { + self.cache.remove(url); + } + } + + let Some(promise) = self.in_flight.get_mut(url) else { + if Url::parse(url).is_err() { + trace!("Found invalid url: {url}"); + self.mime_cache.insert( + url.to_owned(), + CachedMime { + mime: None, + expires_at: SystemTime::UNIX_EPOCH + Duration::from_secs(u64::MAX / 2), // never expire... + }, + ); } - } else { let (sender, promise) = Promise::new(); ehttp_get_mime_type(url, sender); self.in_flight.insert(url.to_owned(), promise); - None + return None; + }; + + let Ok(mime_type) = promise.ready_mut()? else { + self.in_flight.remove(url); + self.record_failure( + url, + previous_failure_count.saturating_add(1), + SystemTime::now(), + ); + return None; + }; + + let mime_string = std::mem::take(mime_type); + self.in_flight.remove(url); + + match mime_string.parse::() { + Ok(mime) => { + let fetched_at = SystemTime::now(); + let prev_entry = stored_entry; + let entry = StoredMimeEntry::new_mime(mime_string, fetched_at); + let expires_at = entry.expires_at(); + if let Some(Some(failed_count)) = prev_entry.map(|p| { + if let MimeEntry::Fail { count } = p.entry { + Some(count) + } else { + None + } + }) { + trace!("found {mime:?} for {url}, inserting in cache & storage AFTER FAILING {failed_count} TIMES"); + } else { + trace!("found {mime:?} for {url}, inserting in cache & storage"); + } + self.cache.set_entry(url.to_owned(), entry); + self.mime_cache.insert( + url.to_owned(), + CachedMime { + mime: Some(mime), + expires_at, + }, + ); + self.mime_cache + .get(url) + .and_then(|cached| cached.mime.as_ref()) + } + Err(err) => { + tracing::warn!("Unable to parse mime type returned for {url}: {err}"); + self.record_failure( + url, + previous_failure_count.saturating_add(1), + SystemTime::now(), + ); + None + } } } + + fn record_failure(&mut self, url: &str, count: u32, timestamp: SystemTime) { + let count = count.max(1); + let entry = StoredMimeEntry::new_failure(count, timestamp); + let expires_at = entry.expires_at(); + trace!( + "failed to get mime for {url} {count} times. next request in {:?}", + failure_backoff_duration(count) + ); + self.cache.set_entry(url.to_owned(), entry); + self.mime_cache.insert( + url.to_owned(), + CachedMime { + mime: None, + expires_at, + }, + ); + } } #[derive(Debug)] @@ -258,11 +523,15 @@ impl SupportedMimeType { } pub fn to_cache_type(&self) -> MediaCacheType { - if self.mime == mime_guess::mime::IMAGE_GIF { - MediaCacheType::Gif - } else { - MediaCacheType::Image - } + mime_to_cache_type(&self.mime) + } +} + +fn mime_to_cache_type(mime: &Mime) -> MediaCacheType { + if *mime == mime_guess::mime::IMAGE_GIF { + MediaCacheType::Gif + } else { + MediaCacheType::Image } } @@ -297,18 +566,16 @@ fn url_has_supported_mime(url: &str) -> MimeHostedAtUrl { #[profiling::function] pub fn supported_mime_hosted_at_url(urls: &mut UrlMimes, url: &str) -> Option { - match url_has_supported_mime(url) { - MimeHostedAtUrl::Yes(cache_type) => Some(cache_type), - MimeHostedAtUrl::Maybe => urls - .get(url) - .and_then(|s| s.parse::().ok()) - .and_then(|mime: mime_guess::mime::Mime| { - SupportedMimeType::from_mime(mime) - .ok() - .map(|s| s.to_cache_type()) - }), - MimeHostedAtUrl::No => None, - } + let Some(mime) = urls.get_or_fetch(url) else { + return match url_has_supported_mime(url) { + MimeHostedAtUrl::Yes(media_cache_type) => Some(media_cache_type), + MimeHostedAtUrl::Maybe | MimeHostedAtUrl::No => None, + }; + }; + + Some(mime) + .filter(|mime| is_mime_supported(mime)) + .map(mime_to_cache_type) } enum MimeHostedAtUrl {