From 4b89445b2aaaab353026fbf14d65923cb8452beb Mon Sep 17 00:00:00 2001 From: decentclock Date: Tue, 12 Jul 2022 12:03:27 -0600 Subject: [PATCH 01/10] Initialize persister module --- Cargo.toml | 3 ++- persister/Cargo.toml | 14 ++++++++++++++ persister/src/lib.rs | 14 ++++++++++++++ sphinx-key/Cargo.toml | 1 + 4 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 persister/Cargo.toml create mode 100644 persister/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index ad25896..c6d525e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,8 @@ members = [ exclude = [ "sphinx-key", "crypter", - "crypter-ffi" + "crypter-ffi", + "persister", ] [patch.crates-io] diff --git a/persister/Cargo.toml b/persister/Cargo.toml new file mode 100644 index 0000000..a56ad39 --- /dev/null +++ b/persister/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "sphinx-key-persister" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +lightning-signer-core = { git = "https://gitlab.com/Evanfeenstra/validating-lightning-signer", branch = "partial-std", default-features = false, features = ["std", "secp-lowmemory"] } +lightning-signer-server = { git = "https://gitlab.com/Evanfeenstra/validating-lightning-signer", branch = "partial-std", default-features = false, features = ["persist"] } +serde = { version = "1.0.105" } +postcard = "1.0.0" +hex = "0.4.3" +anyhow = { version = "1.0.58", features = ["backtrace"] } diff --git a/persister/src/lib.rs b/persister/src/lib.rs new file mode 100644 index 0000000..7d12d9a --- /dev/null +++ b/persister/src/lib.rs @@ -0,0 +1,14 @@ +pub fn add(left: usize, right: usize) -> usize { + left + right +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn it_works() { + let result = add(2, 2); + assert_eq!(result, 4); + } +} diff --git a/sphinx-key/Cargo.toml b/sphinx-key/Cargo.toml index 8dc89c1..45db74e 100644 --- a/sphinx-key/Cargo.toml +++ b/sphinx-key/Cargo.toml @@ -22,6 +22,7 @@ pingpong = [] esp-idf-sys = { version = "0.31.5", features = ["binstart"] } sphinx-key-signer = { path = "../signer", optional = true } sphinx-key-crypter = { path = "../crypter" } +sphinx-key-persister = { path = "../persister" } embedded-svc = { version = "0.21.2" } esp-idf-svc = "0.41" esp-idf-hal = "0.37" From 06bf823b37dcfda8e83961e649eaf4a93ff83908 Mon Sep 17 00:00:00 2001 From: decentclock Date: Fri, 15 Jul 2022 14:55:48 -0600 Subject: [PATCH 02/10] Set all esp32 branches to patch-sign-chan --- parser/Cargo.toml | 2 +- persister/Cargo.toml | 4 ++-- signer/Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/parser/Cargo.toml b/parser/Cargo.toml index da7f061..064cd6f 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" [dependencies] # vls-protocol = { path = "../../../evanf/validating-lightning-signer/vls-protocol" } -vls-protocol = { git = "https://gitlab.com/Evanfeenstra/validating-lightning-signer", branch = "sphinx-key-1" } +vls-protocol = { git = "https://gitlab.com/Evanfeenstra/validating-lightning-signer", branch = "patch-sign-chan" } serde = { version = "1.0", default-features = false } serde_bolt = { version = "0.2", default-features = false } diff --git a/persister/Cargo.toml b/persister/Cargo.toml index a56ad39..ca2dfe7 100644 --- a/persister/Cargo.toml +++ b/persister/Cargo.toml @@ -6,8 +6,8 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -lightning-signer-core = { git = "https://gitlab.com/Evanfeenstra/validating-lightning-signer", branch = "partial-std", default-features = false, features = ["std", "secp-lowmemory"] } -lightning-signer-server = { git = "https://gitlab.com/Evanfeenstra/validating-lightning-signer", branch = "partial-std", default-features = false, features = ["persist"] } +lightning-signer-core = { git = "https://gitlab.com/Evanfeenstra/validating-lightning-signer", branch = "patch-sign-chan", default-features = false, features = ["std", "secp-lowmemory"] } +lightning-signer-server = { git = "https://gitlab.com/Evanfeenstra/validating-lightning-signer", branch = "patch-sign-chan", default-features = false, features = ["persist"] } serde = { version = "1.0.105" } postcard = "1.0.0" hex = "0.4.3" diff --git a/signer/Cargo.toml b/signer/Cargo.toml index 064908c..7732b29 100644 --- a/signer/Cargo.toml +++ b/signer/Cargo.toml @@ -8,7 +8,7 @@ edition = "2018" sphinx-key-parser = { path = "../parser" } # vls-protocol-signer = { path = "../../../evanf/validating-lightning-signer/vls-protocol-signer", default-features = false, features = ["std", "secp-lowmemory"] } # vls-protocol-signer = { git = "https://gitlab.com/lightning-signer/validating-lightning-signer", default-features = false, features = ["secp-lowmemory"] } -vls-protocol-signer = { git = "https://gitlab.com/Evanfeenstra/validating-lightning-signer", branch = "sphinx-key-1", default-features = false, features = ["std", "secp-lowmemory"] } +vls-protocol-signer = { git = "https://gitlab.com/Evanfeenstra/validating-lightning-signer", branch = "patch-sign-chan", default-features = false, features = ["std", "secp-lowmemory"] } anyhow = {version = "1", features = ["backtrace"]} log = "0.4" From d83bf92602869ba9c7b3ce52df9451901fcb73a2 Mon Sep 17 00:00:00 2001 From: decentclock Date: Tue, 12 Jul 2022 11:55:39 -0600 Subject: [PATCH 03/10] Create sd card setup and simple test functions --- sphinx-key/Cargo.toml | 1 + sphinx-key/src/main.rs | 5 +- sphinx-key/src/periph/sd.rs | 203 +++++++++++++++++++++++++++++++++++- 3 files changed, 203 insertions(+), 6 deletions(-) diff --git a/sphinx-key/Cargo.toml b/sphinx-key/Cargo.toml index 45db74e..854895a 100644 --- a/sphinx-key/Cargo.toml +++ b/sphinx-key/Cargo.toml @@ -19,6 +19,7 @@ pio = ["esp-idf-sys/pio"] pingpong = [] [dependencies] +bitflags = "1.3.2" esp-idf-sys = { version = "0.31.5", features = ["binstart"] } sphinx-key-signer = { path = "../signer", optional = true } sphinx-key-crypter = { path = "../crypter" } diff --git a/sphinx-key/src/main.rs b/sphinx-key/src/main.rs index 5429f2a..d06d19d 100644 --- a/sphinx-key/src/main.rs +++ b/sphinx-key/src/main.rs @@ -5,7 +5,7 @@ mod periph; use crate::core::{config::*, events::*}; use crate::periph::led::led_control_loop; -use crate::periph::sd::sd_card; +use crate::periph::sd::{sd_card, simple_fs_test}; use anyhow::Result; use esp_idf_sys as _; // If using the `binstart` feature of `esp-idf-sys`, always keep this module imported @@ -43,7 +43,8 @@ fn main() -> Result<()> { led_control_loop(pins.gpio8, peripherals.rmt.channel0, led_rx); // sd card - sd_card(peripherals.spi2); + sd_card(); + simple_fs_test(); let default_nvs = Arc::new(EspDefaultNvs::new()?); let mut store = diff --git a/sphinx-key/src/periph/sd.rs b/sphinx-key/src/periph/sd.rs index 6067fcc..173a82e 100644 --- a/sphinx-key/src/periph/sd.rs +++ b/sphinx-key/src/periph/sd.rs @@ -1,6 +1,201 @@ - +use bitflags::bitflags; use esp_idf_hal::spi; +use esp_idf_sys::c_types::c_char; +use esp_idf_sys::{ + esp, esp_vfs_fat_sdmmc_mount_config_t, esp_vfs_fat_sdspi_mount, gpio_num_t, sdmmc_card_t, + sdmmc_host_t, sdspi_device_config_t, spi_bus_config_t, spi_bus_initialize, spi_host_device_t, + spi_host_device_t_SPI2_HOST, +}; +use std::fs; +use std::io::BufReader; +use std::io::BufWriter; +use std::io::Read; +use std::io::Write; +use std::ptr; +use std::string::String; -pub fn sd_card(_spi: spi::SPI2) { - log::info!("sd_card"); -} \ No newline at end of file +pub const MOUNT_POINT: &'static str = "/sdcard"; +const C_MOUNT_POINT: &'static [u8] = b"/sdcard\0"; + +const SPI_HOST_SLOT: spi_host_device_t = spi_host_device_t_SPI2_HOST; +const SPI_GPIO_MOSI: gpio_num_t = 4; +const SPI_GPIO_CLK: gpio_num_t = 5; +const SPI_GPIO_MISO: gpio_num_t = 6; +const SPI_GPIO_CS: gpio_num_t = 1; + + +bitflags! { + struct SDMMCHostFlag: u32 { + /// host supports 1-line SD and MMC protocol + const BIT1 = 1 << 0; + /// host supports 4-line SD and MMC protocol + const BIT4 = 1 << 1; + /// host supports 8-line MMC protocol + const BIT8 = 1 << 2; + /// host supports SPI protocol + const SPI = 1 << 3; + /// host supports DDR mode for SD/MMC + const DDR = 1 << 4; + /// host `deinit` function called with the slot argument + const DEINIT_ARG = 1 << 5; + } +} + +enum SDMMCFreq { + /// SD/MMC Default speed (limited by clock divider) + Default = 20000, + /// SD High speed (limited by clock divider) + HighSPeed = 40000, + /// SD/MMC probing speed + Probing = 400, + /// MMC 52MHz speed + _52M = 52000, + /// MMC 26MHz speed + _26M = 26000, +} + + +pub fn sd_card() { + let mount_config = esp_vfs_fat_sdmmc_mount_config_t { + format_if_mount_failed: false, + max_files: 5, + allocation_unit_size: 16 * 1024, + }; + + let mut card: *mut sdmmc_card_t = ptr::null_mut(); + + let bus_cfg = spi_bus_config_t { + __bindgen_anon_1: esp_idf_sys::spi_bus_config_t__bindgen_ty_1 { + mosi_io_num: SPI_GPIO_MOSI, + }, + __bindgen_anon_2: esp_idf_sys::spi_bus_config_t__bindgen_ty_2 { + miso_io_num: SPI_GPIO_MISO, + }, + sclk_io_num: SPI_GPIO_CLK, + __bindgen_anon_3: esp_idf_sys::spi_bus_config_t__bindgen_ty_3 { quadwp_io_num: -1 }, + __bindgen_anon_4: esp_idf_sys::spi_bus_config_t__bindgen_ty_4 { quadhd_io_num: -1 }, + data4_io_num: -1, + data5_io_num: -1, + data6_io_num: -1, + data7_io_num: -1, + max_transfer_sz: 4000, + flags: 0, + intr_flags: 0, + }; + + let res = esp!(unsafe { + spi_bus_initialize( + SPI_HOST_SLOT as u32, + &bus_cfg, + esp_idf_sys::spi_common_dma_t_SPI_DMA_CH_AUTO, + ) + }); + + match res { + Ok(_) => (), + Err(e) => { + println!("Failed to initialize SPI Bus: {}", e); + } + } + + println!("Initialized SPI BUS!"); + + let slot_config = sdspi_device_config_t { + host_id: SPI_HOST_SLOT, + gpio_cs: SPI_GPIO_CS, + gpio_cd: -1, + gpio_wp: -1, + gpio_int: -1, + }; + + let host = sdmmc_host_t { + flags: (SDMMCHostFlag::SPI | SDMMCHostFlag::DEINIT_ARG).bits, //SDMMC_HOST_FLAG_SPI | SDMMC_HOST_FLAG_DEINIT_ARG, + slot: SPI_HOST_SLOT as i32, + max_freq_khz: SDMMCFreq::Default as i32, //SDMMC_FREQ_DEFAULT, + io_voltage: 3.3f32, + init: Some(esp_idf_sys::sdspi_host_init), + set_bus_width: None, + get_bus_width: None, + set_bus_ddr_mode: None, + set_card_clk: Some(esp_idf_sys::sdspi_host_set_card_clk), + do_transaction: Some(esp_idf_sys::sdspi_host_do_transaction), + __bindgen_anon_1: esp_idf_sys::sdmmc_host_t__bindgen_ty_1 { + deinit_p: Some(esp_idf_sys::sdspi_host_remove_device), + }, + io_int_enable: Some(esp_idf_sys::sdspi_host_io_int_enable), + io_int_wait: Some(esp_idf_sys::sdspi_host_io_int_wait), + command_timeout_ms: 0, + }; + + let res = esp!(unsafe { + esp_vfs_fat_sdspi_mount( + C_MOUNT_POINT.as_ptr() as *const c_char, + &host, + &slot_config, + &mount_config, + &mut card as *mut *mut sdmmc_card_t, + ) + }); + + match res { + Ok(_) => (), + Err(e) => { + println!("Failed to mount filesystem: {}", e); + } + } +} + + +pub fn simple_fs_test() { + // Create and write a file, append a file, read a file, delete a file, create a directory, write a file + + let file = fs::File::create("/sdcard/sphinx.txt").expect("create failed"); + let mut writer = BufWriter::new(file); + writer + .write_all("Hello World".as_bytes()) + .expect("write failed"); + writer + .write_all("\nWelcome to your own network".as_bytes()) + .expect("write failed"); + writer.flush().expect("Problem flushing"); + drop(writer); + println!("Data written to file"); + + let file = fs::OpenOptions::new() + .append(true) + .open("/sdcard/sphinx.txt") + .expect("cannot open file"); + let mut writer = BufWriter::new(file); + writer + .write_all("\nDepeche Mode".as_bytes()) + .expect("write failed"); + writer + .write_all("\nWelcome to my World".as_bytes()) + .expect("write failed"); + writer.flush().expect("Problem flushing"); + drop(writer); + println!("File append success"); + + let file = fs::File::open("/sdcard/sphinx.txt").expect("Cannot open file"); + let mut reader = BufReader::new(file); + let mut contents = String::new(); + reader + .read_to_string(&mut contents) + .expect("could not read file"); + drop(reader); + + fs::remove_file("/sdcard/sphinx.txt").expect("could not remove file"); + println!("File is removed"); + + let _ = fs::create_dir("/sdcard/data"); + + let file = fs::File::create("/sdcard/data/sphinx.txt").expect("Cannot open file"); + let mut writer = BufWriter::new(file); + writer + .write_all(contents.as_bytes()) + .expect("Could not write"); + writer.flush().expect("Problem flushing"); + drop(writer); + println!("Wrote file in data directory"); + println!("Simple fs test complete!"); +} From cd5af40c3489ef2d6e7c915ae69df4bf9df95239 Mon Sep 17 00:00:00 2001 From: decentclock Date: Fri, 15 Jul 2022 19:38:46 -0600 Subject: [PATCH 04/10] FsPersister successfully restores channels on an ubuntu machine --- persister/src/lib.rs | 15 +- persister/src/persist_fs.rs | 286 ++++++++++++++++++++++++++++++++++++ signer/Cargo.toml | 1 + signer/src/lib.rs | 4 +- sphinx-key/Cargo.toml | 1 - 5 files changed, 291 insertions(+), 16 deletions(-) create mode 100644 persister/src/persist_fs.rs diff --git a/persister/src/lib.rs b/persister/src/lib.rs index 7d12d9a..f485c10 100644 --- a/persister/src/lib.rs +++ b/persister/src/lib.rs @@ -1,14 +1 @@ -pub fn add(left: usize, right: usize) -> usize { - left + right -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn it_works() { - let result = add(2, 2); - assert_eq!(result, 4); - } -} +pub mod persist_fs; diff --git a/persister/src/persist_fs.rs b/persister/src/persist_fs.rs new file mode 100644 index 0000000..e61550f --- /dev/null +++ b/persister/src/persist_fs.rs @@ -0,0 +1,286 @@ +use lightning_signer::persist::Persist; +use lightning_signer_server::persist::model::{ChannelEntry, NodeEntry}; +use std::fs; +use std::io::BufReader; +use std::io::BufWriter; +use std::io::Read; +use std::io::Write; +use std::path::Path; +use std::ptr; +use std::string::String; + +use lightning_signer::bitcoin::secp256k1::PublicKey; +use lightning_signer::chain::tracker::ChainTracker; +use lightning_signer::channel::Channel; +use lightning_signer::channel::ChannelId; +use lightning_signer::channel::ChannelStub; +use lightning_signer::monitor::ChainMonitor; +use lightning_signer::node::NodeConfig; +use lightning_signer::policy::validator::EnforcementState; +use lightning_signer_server::persist::model::AllowlistItemEntry; +use lightning_signer_server::persist::model::ChainTrackerEntry; +use lightning_signer_server::persist::model::NodeChannelId; +use serde::Deserialize; +use serde::Serialize; + +use lightning_signer::persist::model::{ + ChannelEntry as CoreChannelEntry, NodeEntry as CoreNodeEntry, +}; + +const FAT32_MAXFILENAMESIZE: usize = 8; + +const NODE: &str = "/home/ubuntu/sdcard/nodes"; +const CHAN: &str = "/home/ubuntu/sdcard/channel"; +const ALLO: &str = "/home/ubuntu/sdcard/allowlis"; +const CHAI: &str = "/home/ubuntu/sdcard/chaintra"; +const PUBS: &str = "/home/ubuntu/sdcard/pubkey"; + +pub struct FsPersister { + node_path: String, + channel_path: String, + allowlist_path: String, + chaintracker_path: String, + pubkey_path: String, +} + +impl FsPersister { + pub fn new() -> Self { + let _ = fs::create_dir(NODE); + let _ = fs::create_dir(CHAN); + let _ = fs::create_dir(ALLO); + let _ = fs::create_dir(CHAI); + let _ = fs::create_dir(PUBS); + + let mut node_path = String::with_capacity(NODE.len() + 1 + FAT32_MAXFILENAMESIZE); + node_path.push_str(NODE); + node_path.push_str("/"); + let mut channel_path = String::with_capacity(CHAN.len() + 2 + 2 * FAT32_MAXFILENAMESIZE); + channel_path.push_str(CHAN); + channel_path.push_str("/"); + let mut allowlist_path = String::with_capacity(ALLO.len() + 1 + FAT32_MAXFILENAMESIZE); + allowlist_path.push_str(ALLO); + allowlist_path.push_str("/"); + let mut chaintracker_path = String::with_capacity(CHAI.len() + 1 + FAT32_MAXFILENAMESIZE); + chaintracker_path.push_str(CHAI); + chaintracker_path.push_str("/"); + let mut pubkey_path = String::with_capacity(PUBS.len() + 1 + FAT32_MAXFILENAMESIZE); + pubkey_path.push_str(PUBS); + pubkey_path.push_str("/"); + + Self { + node_path, + channel_path, + allowlist_path, + chaintracker_path, + pubkey_path, + } + } +} + +fn write(path: String, entry: T) -> anyhow::Result<()> { + let mut buf = [0u8; 10000]; + let used = postcard::to_slice(&entry, &mut buf)?; + println!("WROTE: {:?} BYTES", used.len()); + let file = fs::File::create(path)?; + let mut writer = BufWriter::new(file); + writer.write_all(used)?; + writer.flush()?; + drop(writer); + Ok(()) +} + +fn read<'a, T: Deserialize<'a>>(path: String, buf: &'a mut Vec) -> T { + let file = fs::File::open(path).expect("Cannot open file"); + let mut reader = BufReader::new(file); + reader.read_to_end(buf).expect("Could not read"); + println!("READ: {:?} BYTES", buf.len()); + postcard::from_bytes(buf).unwrap() +} + +impl Persist for FsPersister { + fn new_node(&self, node_id: &PublicKey, config: &NodeConfig, seed: &[u8]) { + let mut node_path = self.node_path.clone(); + let mut pubkey_path = self.pubkey_path.clone(); + let filename = &hex::encode(node_id.serialize())[..FAT32_MAXFILENAMESIZE]; + node_path.push_str(filename); + pubkey_path.push_str(filename); + let entry = NodeEntry { + seed: seed.to_vec(), + key_derivation_style: config.key_derivation_style as u8, + network: config.network.to_string(), + }; + if let Err(e) = write(node_path, entry) { + println!("Write error: {:?}", e); + } + if let Err(e) = write(pubkey_path, node_id) { + println!("Write error: {:?}", e); + } + } + fn delete_node(&self, node_id: &PublicKey) { + let mut channel_path = self.channel_path.clone(); + let key_a = &hex::encode(node_id.serialize())[..FAT32_MAXFILENAMESIZE]; + channel_path.push_str(key_a); + fs::remove_dir_all(channel_path).unwrap(); + let mut node_path = self.node_path.clone(); + let mut pubkey_path = self.pubkey_path.clone(); + let filename = &hex::encode(node_id.serialize())[..FAT32_MAXFILENAMESIZE]; + node_path.push_str(filename); + pubkey_path.push_str(filename); + fs::remove_file(node_path).unwrap(); + fs::remove_file(pubkey_path).unwrap(); + } + fn new_channel(&self, node_id: &PublicKey, stub: &ChannelStub) -> Result<(), ()> { + let mut channel_path = self.channel_path.clone(); + let key_a = &hex::encode(node_id.serialize())[..FAT32_MAXFILENAMESIZE]; + channel_path.push_str(&key_a); + fs::create_dir(channel_path.clone()).expect("Problem creating a directory"); + channel_path.push_str("/"); + + let id = NodeChannelId::new(node_id, &stub.id0); + channel_path.push_str(&hex::encode( + &id.channel_id().as_slice()[..FAT32_MAXFILENAMESIZE], + )); + let channel_value_satoshis = 0; + let entry = ChannelEntry { + channel_value_satoshis, + channel_setup: None, + id: Some(id.channel_id()), + enforcement_state: EnforcementState::new(0), + }; + if let Err(e) = write(channel_path, entry) { + println!("Write error: {:?}", e); + } + Ok(()) + } + fn new_chain_tracker(&self, node_id: &PublicKey, tracker: &ChainTracker) { + let mut chaintracker_path = self.chaintracker_path.clone(); + chaintracker_path.push_str(&hex::encode(node_id.serialize())[..FAT32_MAXFILENAMESIZE]); + let entry: ChainTrackerEntry = tracker.into(); + if let Err(e) = write(chaintracker_path, entry) { + println!("Write error: {:?}", e); + } + } + fn update_tracker( + &self, + node_id: &PublicKey, + tracker: &ChainTracker, + ) -> Result<(), ()> { + let mut chaintracker_path = self.chaintracker_path.clone(); + chaintracker_path.push_str(&hex::encode(node_id.serialize())[..FAT32_MAXFILENAMESIZE]); + let entry: ChainTrackerEntry = tracker.into(); + if let Err(e) = write(chaintracker_path, entry) { + println!("Write error: {:?}", e); + } + Ok(()) + } + fn get_tracker(&self, node_id: &PublicKey) -> Result, ()> { + let mut chaintracker_path = self.chaintracker_path.clone(); + chaintracker_path.push_str(&hex::encode(node_id.serialize())[..FAT32_MAXFILENAMESIZE]); + let mut buf = Vec::new(); + let ret: ChainTrackerEntry = read(chaintracker_path, &mut buf); + Ok(ret.into()) + } + fn update_channel(&self, node_id: &PublicKey, channel: &Channel) -> Result<(), ()> { + println!("UPDATING CHANNEL: {:?}", channel.id); + let channel_value_satoshis = channel.setup.channel_value_sat; + let mut channel_path = self.channel_path.clone(); + let key_a = &hex::encode(node_id.serialize())[..FAT32_MAXFILENAMESIZE]; + channel_path.push_str(&key_a); + channel_path.push_str("/"); + + let id = NodeChannelId::new(node_id, &channel.id0); + channel_path.push_str(&hex::encode( + &id.channel_id().as_slice()[..FAT32_MAXFILENAMESIZE], + )); + let entry = ChannelEntry { + channel_value_satoshis, + channel_setup: Some(channel.setup.clone()), + //id: channel.id.clone(), + id: if channel.id.is_none() { + Some(id.channel_id()) + } else { + channel.id.clone() + }, + enforcement_state: channel.enforcement_state.clone(), + }; + if let Err(e) = write(channel_path, entry) { + println!("Write error: {:?}", e); + } + Ok(()) + } + fn get_channel( + &self, + node_id: &PublicKey, + channel_id: &ChannelId, + ) -> Result { + let mut channel_path = self.channel_path.clone(); + let key_a = &hex::encode(node_id.serialize())[..FAT32_MAXFILENAMESIZE]; + channel_path.push_str(&key_a); + channel_path.push_str("/"); + let id = NodeChannelId::new(node_id, channel_id); + channel_path.push_str(&hex::encode( + &id.channel_id().as_slice()[..FAT32_MAXFILENAMESIZE], + )); + let mut buf = Vec::new(); + let ret: ChannelEntry = read(channel_path, &mut buf); + Ok(ret.into()) + } + fn get_node_channels(&self, node_id: &PublicKey) -> Vec<(ChannelId, CoreChannelEntry)> { + let mut res = Vec::new(); + let mut channel_path = self.channel_path.clone(); + let key_a = &hex::encode(node_id.serialize())[..FAT32_MAXFILENAMESIZE]; + channel_path.push_str(&key_a); + if !Path::new(&channel_path).exists() { + return res; + } + for channel in fs::read_dir(channel_path).unwrap() { + let channel = channel.unwrap(); + let mut buf = Vec::new(); + let entry: ChannelEntry = read(channel.path().to_str().unwrap().to_string(), &mut buf); + let id = entry.id.clone().unwrap(); + res.push((id, entry.into())) + } + res + } + fn update_node_allowlist(&self, node_id: &PublicKey, allowlist: Vec) -> Result<(), ()> { + let mut allowlist_path = self.allowlist_path.clone(); + allowlist_path.push_str(&hex::encode(node_id.serialize())[..FAT32_MAXFILENAMESIZE]); + let entry = AllowlistItemEntry { allowlist }; + if let Err(e) = write(allowlist_path, entry) { + println!("Write error: {:?}", e); + } + Ok(()) + } + fn get_node_allowlist(&self, node_id: &PublicKey) -> Vec { + let mut allowlist_path = self.allowlist_path.clone(); + allowlist_path.push_str(&hex::encode(node_id.serialize())[..FAT32_MAXFILENAMESIZE]); + let mut buf = Vec::new(); + let entry: AllowlistItemEntry = read(allowlist_path, &mut buf); + entry.allowlist + } + fn get_nodes(&self) -> Vec<(PublicKey, CoreNodeEntry)> { + let mut res = Vec::new(); + let pubkey_path = self.pubkey_path.clone(); + let node_path = self.node_path.clone(); + for (pubkey, node) in fs::read_dir(pubkey_path) + .unwrap() + .zip(fs::read_dir(node_path).unwrap()) + { + let node = node.unwrap(); + let pubkey = pubkey.unwrap(); + let mut buf_a = Vec::new(); + let mut buf_b = Vec::new(); + let pubkey: PublicKey = read(pubkey.path().to_str().unwrap().to_string(), &mut buf_a); + let entry: NodeEntry = read(node.path().to_str().unwrap().to_string(), &mut buf_b); + res.push((pubkey, entry.into())) + } + res + } + fn clear_database(&self) { + fs::remove_dir_all(NODE).unwrap(); + fs::remove_dir_all(CHAN).unwrap(); + fs::remove_dir_all(ALLO).unwrap(); + fs::remove_dir_all(CHAI).unwrap(); + fs::remove_dir_all(PUBS).unwrap(); + } +} diff --git a/signer/Cargo.toml b/signer/Cargo.toml index 7732b29..c6e4150 100644 --- a/signer/Cargo.toml +++ b/signer/Cargo.toml @@ -6,6 +6,7 @@ edition = "2018" [dependencies] sphinx-key-parser = { path = "../parser" } +sphinx-key-persister = { path = "../persister" } # vls-protocol-signer = { path = "../../../evanf/validating-lightning-signer/vls-protocol-signer", default-features = false, features = ["std", "secp-lowmemory"] } # vls-protocol-signer = { git = "https://gitlab.com/lightning-signer/validating-lightning-signer", default-features = false, features = ["secp-lowmemory"] } vls-protocol-signer = { git = "https://gitlab.com/Evanfeenstra/validating-lightning-signer", branch = "patch-sign-chan", default-features = false, features = ["std", "secp-lowmemory"] } diff --git a/signer/src/lib.rs b/signer/src/lib.rs index 644b3fb..904910c 100644 --- a/signer/src/lib.rs +++ b/signer/src/lib.rs @@ -10,6 +10,7 @@ pub use vls_protocol_signer::lightning_signer; pub use vls_protocol_signer::lightning_signer::bitcoin::Network; pub use vls_protocol_signer::vls_protocol; pub use sphinx_key_parser::MsgDriver; +pub use sphinx_key_persister::persist_fs::FsPersister; pub struct InitResponse { pub root_handler: RootHandler, @@ -17,7 +18,8 @@ pub struct InitResponse { } pub fn init(bytes: Vec, network: Network) -> anyhow::Result { - let persister: Arc = Arc::new(DummyPersister); + //let persister: Arc = Arc::new(DummyPersister); + let persister: Arc = Arc::new(FsPersister::new()); let mut md = MsgDriver::new(bytes); let (sequence, dbid) = read_serial_request_header(&mut md).expect("read init header"); assert_eq!(dbid, 0); diff --git a/sphinx-key/Cargo.toml b/sphinx-key/Cargo.toml index 854895a..89b3cd6 100644 --- a/sphinx-key/Cargo.toml +++ b/sphinx-key/Cargo.toml @@ -23,7 +23,6 @@ bitflags = "1.3.2" esp-idf-sys = { version = "0.31.5", features = ["binstart"] } sphinx-key-signer = { path = "../signer", optional = true } sphinx-key-crypter = { path = "../crypter" } -sphinx-key-persister = { path = "../persister" } embedded-svc = { version = "0.21.2" } esp-idf-svc = "0.41" esp-idf-hal = "0.37" From 0552fc36eaf3c1582440c14972d5d238fd76a540 Mon Sep 17 00:00:00 2001 From: Evan Feenstra Date: Sun, 17 Jul 2022 17:43:55 -0700 Subject: [PATCH 05/10] fsdb-based persistence --- persister/Cargo.toml | 3 +- persister/src/persist_fs.rs | 269 ++++++++++++------------------------ 2 files changed, 87 insertions(+), 185 deletions(-) diff --git a/persister/Cargo.toml b/persister/Cargo.toml index ca2dfe7..9305d10 100644 --- a/persister/Cargo.toml +++ b/persister/Cargo.toml @@ -9,6 +9,5 @@ edition = "2021" lightning-signer-core = { git = "https://gitlab.com/Evanfeenstra/validating-lightning-signer", branch = "patch-sign-chan", default-features = false, features = ["std", "secp-lowmemory"] } lightning-signer-server = { git = "https://gitlab.com/Evanfeenstra/validating-lightning-signer", branch = "patch-sign-chan", default-features = false, features = ["persist"] } serde = { version = "1.0.105" } -postcard = "1.0.0" hex = "0.4.3" -anyhow = { version = "1.0.58", features = ["backtrace"] } +fsdb = "0.1.4" \ No newline at end of file diff --git a/persister/src/persist_fs.rs b/persister/src/persist_fs.rs index e61550f..e78ad94 100644 --- a/persister/src/persist_fs.rs +++ b/persister/src/persist_fs.rs @@ -1,12 +1,6 @@ +use fsdb::{Bucket, Fsdb}; use lightning_signer::persist::Persist; use lightning_signer_server::persist::model::{ChannelEntry, NodeEntry}; -use std::fs; -use std::io::BufReader; -use std::io::BufWriter; -use std::io::Read; -use std::io::Write; -use std::path::Path; -use std::ptr; use std::string::String; use lightning_signer::bitcoin::secp256k1::PublicKey; @@ -20,8 +14,6 @@ use lightning_signer::policy::validator::EnforcementState; use lightning_signer_server::persist::model::AllowlistItemEntry; use lightning_signer_server::persist::model::ChainTrackerEntry; use lightning_signer_server::persist::model::NodeChannelId; -use serde::Deserialize; -use serde::Serialize; use lightning_signer::persist::model::{ ChannelEntry as CoreChannelEntry, NodeEntry as CoreNodeEntry, @@ -29,183 +21,104 @@ use lightning_signer::persist::model::{ const FAT32_MAXFILENAMESIZE: usize = 8; -const NODE: &str = "/home/ubuntu/sdcard/nodes"; -const CHAN: &str = "/home/ubuntu/sdcard/channel"; -const ALLO: &str = "/home/ubuntu/sdcard/allowlis"; -const CHAI: &str = "/home/ubuntu/sdcard/chaintra"; -const PUBS: &str = "/home/ubuntu/sdcard/pubkey"; - pub struct FsPersister { - node_path: String, - channel_path: String, - allowlist_path: String, - chaintracker_path: String, - pubkey_path: String, + nodes: Bucket, + channels: Bucket, + allowlist: Bucket, + chaintracker: Bucket, + pubkeys: Bucket, } impl FsPersister { pub fn new() -> Self { - let _ = fs::create_dir(NODE); - let _ = fs::create_dir(CHAN); - let _ = fs::create_dir(ALLO); - let _ = fs::create_dir(CHAI); - let _ = fs::create_dir(PUBS); - - let mut node_path = String::with_capacity(NODE.len() + 1 + FAT32_MAXFILENAMESIZE); - node_path.push_str(NODE); - node_path.push_str("/"); - let mut channel_path = String::with_capacity(CHAN.len() + 2 + 2 * FAT32_MAXFILENAMESIZE); - channel_path.push_str(CHAN); - channel_path.push_str("/"); - let mut allowlist_path = String::with_capacity(ALLO.len() + 1 + FAT32_MAXFILENAMESIZE); - allowlist_path.push_str(ALLO); - allowlist_path.push_str("/"); - let mut chaintracker_path = String::with_capacity(CHAI.len() + 1 + FAT32_MAXFILENAMESIZE); - chaintracker_path.push_str(CHAI); - chaintracker_path.push_str("/"); - let mut pubkey_path = String::with_capacity(PUBS.len() + 1 + FAT32_MAXFILENAMESIZE); - pubkey_path.push_str(PUBS); - pubkey_path.push_str("/"); - + let db = Fsdb::new("home/ubuntu/sdcard").expect("could not create db"); + let mut nodes = db.bucket("nodes").expect("fail nodes"); + nodes.set_max_file_name(FAT32_MAXFILENAMESIZE); + let mut channels = db.bucket("channel").expect("fail channel"); + channels.set_max_file_name(FAT32_MAXFILENAMESIZE); + let mut allowlist = db.bucket("allowlis").expect("fail allowlis"); + allowlist.set_max_file_name(FAT32_MAXFILENAMESIZE); + let mut chaintracker = db.bucket("chaintra").expect("fail chaintra"); + chaintracker.set_max_file_name(FAT32_MAXFILENAMESIZE); + let mut pubkeys = db.bucket("pubkey").expect("fail pubkey"); + pubkeys.set_max_file_name(FAT32_MAXFILENAMESIZE); Self { - node_path, - channel_path, - allowlist_path, - chaintracker_path, - pubkey_path, + nodes, + channels, + allowlist, + chaintracker, + pubkeys, } } } -fn write(path: String, entry: T) -> anyhow::Result<()> { - let mut buf = [0u8; 10000]; - let used = postcard::to_slice(&entry, &mut buf)?; - println!("WROTE: {:?} BYTES", used.len()); - let file = fs::File::create(path)?; - let mut writer = BufWriter::new(file); - writer.write_all(used)?; - writer.flush()?; - drop(writer); - Ok(()) -} - -fn read<'a, T: Deserialize<'a>>(path: String, buf: &'a mut Vec) -> T { - let file = fs::File::open(path).expect("Cannot open file"); - let mut reader = BufReader::new(file); - reader.read_to_end(buf).expect("Could not read"); - println!("READ: {:?} BYTES", buf.len()); - postcard::from_bytes(buf).unwrap() -} - impl Persist for FsPersister { fn new_node(&self, node_id: &PublicKey, config: &NodeConfig, seed: &[u8]) { - let mut node_path = self.node_path.clone(); - let mut pubkey_path = self.pubkey_path.clone(); - let filename = &hex::encode(node_id.serialize())[..FAT32_MAXFILENAMESIZE]; - node_path.push_str(filename); - pubkey_path.push_str(filename); + let pk = hex::encode(node_id.serialize()); let entry = NodeEntry { seed: seed.to_vec(), key_derivation_style: config.key_derivation_style as u8, network: config.network.to_string(), }; - if let Err(e) = write(node_path, entry) { - println!("Write error: {:?}", e); - } - if let Err(e) = write(pubkey_path, node_id) { - println!("Write error: {:?}", e); - } + let _ = self.nodes.put(&pk, entry); + let _ = self.pubkeys.put(&pk, node_id.clone()); } fn delete_node(&self, node_id: &PublicKey) { - let mut channel_path = self.channel_path.clone(); - let key_a = &hex::encode(node_id.serialize())[..FAT32_MAXFILENAMESIZE]; - channel_path.push_str(key_a); - fs::remove_dir_all(channel_path).unwrap(); - let mut node_path = self.node_path.clone(); - let mut pubkey_path = self.pubkey_path.clone(); - let filename = &hex::encode(node_id.serialize())[..FAT32_MAXFILENAMESIZE]; - node_path.push_str(filename); - pubkey_path.push_str(filename); - fs::remove_file(node_path).unwrap(); - fs::remove_file(pubkey_path).unwrap(); + let pk = hex::encode(node_id.serialize()); + // clear all channel entries within "pk" sub-bucket + let _ = self.channels.clear_within(&pk); + let _ = self.nodes.remove(&pk); + let _ = self.pubkeys.remove(&pk); } fn new_channel(&self, node_id: &PublicKey, stub: &ChannelStub) -> Result<(), ()> { - let mut channel_path = self.channel_path.clone(); - let key_a = &hex::encode(node_id.serialize())[..FAT32_MAXFILENAMESIZE]; - channel_path.push_str(&key_a); - fs::create_dir(channel_path.clone()).expect("Problem creating a directory"); - channel_path.push_str("/"); - + let pk = hex::encode(node_id.serialize()); let id = NodeChannelId::new(node_id, &stub.id0); - channel_path.push_str(&hex::encode( - &id.channel_id().as_slice()[..FAT32_MAXFILENAMESIZE], - )); - let channel_value_satoshis = 0; + let chan_id = hex::encode(id.channel_id().as_slice()); let entry = ChannelEntry { - channel_value_satoshis, + channel_value_satoshis: 0, channel_setup: None, id: Some(id.channel_id()), enforcement_state: EnforcementState::new(0), }; - if let Err(e) = write(channel_path, entry) { - println!("Write error: {:?}", e); - } + let _ = self.channels.put_within(&chan_id, entry, &pk); Ok(()) } fn new_chain_tracker(&self, node_id: &PublicKey, tracker: &ChainTracker) { - let mut chaintracker_path = self.chaintracker_path.clone(); - chaintracker_path.push_str(&hex::encode(node_id.serialize())[..FAT32_MAXFILENAMESIZE]); - let entry: ChainTrackerEntry = tracker.into(); - if let Err(e) = write(chaintracker_path, entry) { - println!("Write error: {:?}", e); - } + let pk = hex::encode(node_id.serialize()); + let _ = self.chaintracker.put(&pk, tracker.into()); } fn update_tracker( &self, node_id: &PublicKey, tracker: &ChainTracker, ) -> Result<(), ()> { - let mut chaintracker_path = self.chaintracker_path.clone(); - chaintracker_path.push_str(&hex::encode(node_id.serialize())[..FAT32_MAXFILENAMESIZE]); - let entry: ChainTrackerEntry = tracker.into(); - if let Err(e) = write(chaintracker_path, entry) { - println!("Write error: {:?}", e); - } + let pk = hex::encode(node_id.serialize()); + let _ = self.chaintracker.put(&pk, tracker.into()); Ok(()) } fn get_tracker(&self, node_id: &PublicKey) -> Result, ()> { - let mut chaintracker_path = self.chaintracker_path.clone(); - chaintracker_path.push_str(&hex::encode(node_id.serialize())[..FAT32_MAXFILENAMESIZE]); - let mut buf = Vec::new(); - let ret: ChainTrackerEntry = read(chaintracker_path, &mut buf); + let pk = hex::encode(node_id.serialize()); + let ret: ChainTrackerEntry = match self.chaintracker.get(&pk) { + Ok(ct) => ct, + Err(_) => return Err(()), + }; Ok(ret.into()) } fn update_channel(&self, node_id: &PublicKey, channel: &Channel) -> Result<(), ()> { - println!("UPDATING CHANNEL: {:?}", channel.id); - let channel_value_satoshis = channel.setup.channel_value_sat; - let mut channel_path = self.channel_path.clone(); - let key_a = &hex::encode(node_id.serialize())[..FAT32_MAXFILENAMESIZE]; - channel_path.push_str(&key_a); - channel_path.push_str("/"); - + let pk = hex::encode(node_id.serialize()); let id = NodeChannelId::new(node_id, &channel.id0); - channel_path.push_str(&hex::encode( - &id.channel_id().as_slice()[..FAT32_MAXFILENAMESIZE], - )); + let chan_id = hex::encode(id.channel_id().as_slice()); let entry = ChannelEntry { - channel_value_satoshis, - channel_setup: Some(channel.setup.clone()), - //id: channel.id.clone(), id: if channel.id.is_none() { Some(id.channel_id()) } else { channel.id.clone() }, + channel_value_satoshis: channel.setup.channel_value_sat, + channel_setup: Some(channel.setup.clone()), enforcement_state: channel.enforcement_state.clone(), }; - if let Err(e) = write(channel_path, entry) { - println!("Write error: {:?}", e); - } + let _ = self.channels.put_within(&chan_id, entry, &pk); Ok(()) } fn get_channel( @@ -213,74 +126,64 @@ impl Persist for FsPersister { node_id: &PublicKey, channel_id: &ChannelId, ) -> Result { - let mut channel_path = self.channel_path.clone(); - let key_a = &hex::encode(node_id.serialize())[..FAT32_MAXFILENAMESIZE]; - channel_path.push_str(&key_a); - channel_path.push_str("/"); + let pk = hex::encode(node_id.serialize()); let id = NodeChannelId::new(node_id, channel_id); - channel_path.push_str(&hex::encode( - &id.channel_id().as_slice()[..FAT32_MAXFILENAMESIZE], - )); - let mut buf = Vec::new(); - let ret: ChannelEntry = read(channel_path, &mut buf); + let chan_id = hex::encode(id.channel_id().as_slice()); + let ret: ChannelEntry = match self.channels.get_within(&chan_id, &pk) { + Ok(ce) => ce, + Err(_) => return Err(()), + }; Ok(ret.into()) } fn get_node_channels(&self, node_id: &PublicKey) -> Vec<(ChannelId, CoreChannelEntry)> { let mut res = Vec::new(); - let mut channel_path = self.channel_path.clone(); - let key_a = &hex::encode(node_id.serialize())[..FAT32_MAXFILENAMESIZE]; - channel_path.push_str(&key_a); - if !Path::new(&channel_path).exists() { - return res; - } - for channel in fs::read_dir(channel_path).unwrap() { - let channel = channel.unwrap(); - let mut buf = Vec::new(); - let entry: ChannelEntry = read(channel.path().to_str().unwrap().to_string(), &mut buf); - let id = entry.id.clone().unwrap(); - res.push((id, entry.into())) + let pk = hex::encode(node_id.serialize()); + let list = match self.channels.list_within(&pk) { + Ok(l) => l, + Err(_) => return res, + }; + for channel in list { + if let Ok(entry) = self.channels.get_within(&channel, &pk) { + let id = entry.id.clone().unwrap(); + res.push((id, entry.into())); + }; } res } fn update_node_allowlist(&self, node_id: &PublicKey, allowlist: Vec) -> Result<(), ()> { - let mut allowlist_path = self.allowlist_path.clone(); - allowlist_path.push_str(&hex::encode(node_id.serialize())[..FAT32_MAXFILENAMESIZE]); + let pk = hex::encode(node_id.serialize()); let entry = AllowlistItemEntry { allowlist }; - if let Err(e) = write(allowlist_path, entry) { - println!("Write error: {:?}", e); - } + let _ = self.allowlist.put(&pk, entry); Ok(()) } fn get_node_allowlist(&self, node_id: &PublicKey) -> Vec { - let mut allowlist_path = self.allowlist_path.clone(); - allowlist_path.push_str(&hex::encode(node_id.serialize())[..FAT32_MAXFILENAMESIZE]); - let mut buf = Vec::new(); - let entry: AllowlistItemEntry = read(allowlist_path, &mut buf); + let pk = hex::encode(node_id.serialize()); + let entry: AllowlistItemEntry = match self.allowlist.get(&pk) { + Ok(e) => e, + Err(_) => return Vec::new(), + }; entry.allowlist } fn get_nodes(&self) -> Vec<(PublicKey, CoreNodeEntry)> { let mut res = Vec::new(); - let pubkey_path = self.pubkey_path.clone(); - let node_path = self.node_path.clone(); - for (pubkey, node) in fs::read_dir(pubkey_path) - .unwrap() - .zip(fs::read_dir(node_path).unwrap()) - { - let node = node.unwrap(); - let pubkey = pubkey.unwrap(); - let mut buf_a = Vec::new(); - let mut buf_b = Vec::new(); - let pubkey: PublicKey = read(pubkey.path().to_str().unwrap().to_string(), &mut buf_a); - let entry: NodeEntry = read(node.path().to_str().unwrap().to_string(), &mut buf_b); - res.push((pubkey, entry.into())) + let list = match self.nodes.list() { + Ok(ns) => ns, + Err(_) => return res, + }; + for pk in list { + if let Ok(pubkey) = self.pubkeys.get(&pk) { + if let Ok(node) = self.nodes.get(&pk) { + res.push((pubkey, node.into())); + } + } } res } fn clear_database(&self) { - fs::remove_dir_all(NODE).unwrap(); - fs::remove_dir_all(CHAN).unwrap(); - fs::remove_dir_all(ALLO).unwrap(); - fs::remove_dir_all(CHAI).unwrap(); - fs::remove_dir_all(PUBS).unwrap(); + let _ = self.nodes.clear(); + let _ = self.channels.clear(); + let _ = self.allowlist.clear(); + let _ = self.chaintracker.clear(); + let _ = self.pubkeys.clear(); } } From 203c87ac4c8ca5a032d64bae4c427d0bc2dec997 Mon Sep 17 00:00:00 2001 From: Evan Feenstra Date: Mon, 18 Jul 2022 09:08:44 -0700 Subject: [PATCH 06/10] fsdb 0.1.7 --- persister/Cargo.toml | 2 +- persister/src/persist_fs.rs | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/persister/Cargo.toml b/persister/Cargo.toml index 9305d10..27c3f42 100644 --- a/persister/Cargo.toml +++ b/persister/Cargo.toml @@ -10,4 +10,4 @@ lightning-signer-core = { git = "https://gitlab.com/Evanfeenstra/validating-ligh lightning-signer-server = { git = "https://gitlab.com/Evanfeenstra/validating-lightning-signer", branch = "patch-sign-chan", default-features = false, features = ["persist"] } serde = { version = "1.0.105" } hex = "0.4.3" -fsdb = "0.1.4" \ No newline at end of file +fsdb = "0.1.7" \ No newline at end of file diff --git a/persister/src/persist_fs.rs b/persister/src/persist_fs.rs index e78ad94..4049175 100644 --- a/persister/src/persist_fs.rs +++ b/persister/src/persist_fs.rs @@ -1,4 +1,4 @@ -use fsdb::{Bucket, Fsdb}; +use fsdb::{Bucket, DoubleBucket, Fsdb}; use lightning_signer::persist::Persist; use lightning_signer_server::persist::model::{ChannelEntry, NodeEntry}; use std::string::String; @@ -23,7 +23,7 @@ const FAT32_MAXFILENAMESIZE: usize = 8; pub struct FsPersister { nodes: Bucket, - channels: Bucket, + channels: DoubleBucket, allowlist: Bucket, chaintracker: Bucket, pubkeys: Bucket, @@ -34,7 +34,7 @@ impl FsPersister { let db = Fsdb::new("home/ubuntu/sdcard").expect("could not create db"); let mut nodes = db.bucket("nodes").expect("fail nodes"); nodes.set_max_file_name(FAT32_MAXFILENAMESIZE); - let mut channels = db.bucket("channel").expect("fail channel"); + let mut channels = db.double_bucket("channel").expect("fail channel"); channels.set_max_file_name(FAT32_MAXFILENAMESIZE); let mut allowlist = db.bucket("allowlis").expect("fail allowlis"); allowlist.set_max_file_name(FAT32_MAXFILENAMESIZE); @@ -66,7 +66,7 @@ impl Persist for FsPersister { fn delete_node(&self, node_id: &PublicKey) { let pk = hex::encode(node_id.serialize()); // clear all channel entries within "pk" sub-bucket - let _ = self.channels.clear_within(&pk); + let _ = self.channels.clear(&pk); let _ = self.nodes.remove(&pk); let _ = self.pubkeys.remove(&pk); } @@ -80,7 +80,7 @@ impl Persist for FsPersister { id: Some(id.channel_id()), enforcement_state: EnforcementState::new(0), }; - let _ = self.channels.put_within(&chan_id, entry, &pk); + let _ = self.channels.put(&pk, &chan_id, entry); Ok(()) } fn new_chain_tracker(&self, node_id: &PublicKey, tracker: &ChainTracker) { @@ -118,7 +118,7 @@ impl Persist for FsPersister { channel_setup: Some(channel.setup.clone()), enforcement_state: channel.enforcement_state.clone(), }; - let _ = self.channels.put_within(&chan_id, entry, &pk); + let _ = self.channels.put(&pk, &chan_id, entry); Ok(()) } fn get_channel( @@ -129,7 +129,7 @@ impl Persist for FsPersister { let pk = hex::encode(node_id.serialize()); let id = NodeChannelId::new(node_id, channel_id); let chan_id = hex::encode(id.channel_id().as_slice()); - let ret: ChannelEntry = match self.channels.get_within(&chan_id, &pk) { + let ret: ChannelEntry = match self.channels.get(&pk, &chan_id) { Ok(ce) => ce, Err(_) => return Err(()), }; @@ -138,12 +138,12 @@ impl Persist for FsPersister { fn get_node_channels(&self, node_id: &PublicKey) -> Vec<(ChannelId, CoreChannelEntry)> { let mut res = Vec::new(); let pk = hex::encode(node_id.serialize()); - let list = match self.channels.list_within(&pk) { + let list = match self.channels.list(&pk) { Ok(l) => l, Err(_) => return res, }; for channel in list { - if let Ok(entry) = self.channels.get_within(&channel, &pk) { + if let Ok(entry) = self.channels.get(&pk, &channel) { let id = entry.id.clone().unwrap(); res.push((id, entry.into())); }; @@ -181,7 +181,7 @@ impl Persist for FsPersister { } fn clear_database(&self) { let _ = self.nodes.clear(); - let _ = self.channels.clear(); + let _ = self.channels.clear_all(); let _ = self.allowlist.clear(); let _ = self.chaintracker.clear(); let _ = self.pubkeys.clear(); From a1769feb16d34d26fdc6f2544803328b416dca89 Mon Sep 17 00:00:00 2001 From: Evan Feenstra Date: Mon, 18 Jul 2022 09:48:18 -0700 Subject: [PATCH 07/10] dbfs 0.1.8 --- persister/Cargo.toml | 2 +- persister/src/persist_fs.rs | 21 ++++++--------------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/persister/Cargo.toml b/persister/Cargo.toml index 27c3f42..e5ecff7 100644 --- a/persister/Cargo.toml +++ b/persister/Cargo.toml @@ -10,4 +10,4 @@ lightning-signer-core = { git = "https://gitlab.com/Evanfeenstra/validating-ligh lightning-signer-server = { git = "https://gitlab.com/Evanfeenstra/validating-lightning-signer", branch = "patch-sign-chan", default-features = false, features = ["persist"] } serde = { version = "1.0.105" } hex = "0.4.3" -fsdb = "0.1.7" \ No newline at end of file +fsdb = "0.1.8" \ No newline at end of file diff --git a/persister/src/persist_fs.rs b/persister/src/persist_fs.rs index 4049175..e7a5a18 100644 --- a/persister/src/persist_fs.rs +++ b/persister/src/persist_fs.rs @@ -32,22 +32,13 @@ pub struct FsPersister { impl FsPersister { pub fn new() -> Self { let db = Fsdb::new("home/ubuntu/sdcard").expect("could not create db"); - let mut nodes = db.bucket("nodes").expect("fail nodes"); - nodes.set_max_file_name(FAT32_MAXFILENAMESIZE); - let mut channels = db.double_bucket("channel").expect("fail channel"); - channels.set_max_file_name(FAT32_MAXFILENAMESIZE); - let mut allowlist = db.bucket("allowlis").expect("fail allowlis"); - allowlist.set_max_file_name(FAT32_MAXFILENAMESIZE); - let mut chaintracker = db.bucket("chaintra").expect("fail chaintra"); - chaintracker.set_max_file_name(FAT32_MAXFILENAMESIZE); - let mut pubkeys = db.bucket("pubkey").expect("fail pubkey"); - pubkeys.set_max_file_name(FAT32_MAXFILENAMESIZE); + let max = Some(FAT32_MAXFILENAMESIZE); Self { - nodes, - channels, - allowlist, - chaintracker, - pubkeys, + nodes: db.bucket("nodes", max).expect("fail nodes"), + channels: db.double_bucket("channel", max).expect("fail channel"), + allowlist: db.bucket("allowlis", max).expect("fail allowlis"), + chaintracker: db.bucket("chaintra", max).expect("fail chaintra"), + pubkeys: db.bucket("pubkey", max).expect("fail pubkey"), } } } From b17d991fe7079348d7d9a09becf2b167d2947a48 Mon Sep 17 00:00:00 2001 From: decentclock Date: Mon, 18 Jul 2022 10:57:36 -0600 Subject: [PATCH 08/10] Set directory of FsPersister in constructor --- persister/src/persist_fs.rs | 4 ++-- signer/src/lib.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/persister/src/persist_fs.rs b/persister/src/persist_fs.rs index e7a5a18..74e7bf8 100644 --- a/persister/src/persist_fs.rs +++ b/persister/src/persist_fs.rs @@ -30,8 +30,8 @@ pub struct FsPersister { } impl FsPersister { - pub fn new() -> Self { - let db = Fsdb::new("home/ubuntu/sdcard").expect("could not create db"); + pub fn new(dir: &str) -> Self { + let db = Fsdb::new(dir).expect("could not create db"); let max = Some(FAT32_MAXFILENAMESIZE); Self { nodes: db.bucket("nodes", max).expect("fail nodes"), diff --git a/signer/src/lib.rs b/signer/src/lib.rs index 904910c..0d665c3 100644 --- a/signer/src/lib.rs +++ b/signer/src/lib.rs @@ -19,7 +19,7 @@ pub struct InitResponse { pub fn init(bytes: Vec, network: Network) -> anyhow::Result { //let persister: Arc = Arc::new(DummyPersister); - let persister: Arc = Arc::new(FsPersister::new()); + let persister: Arc = Arc::new(FsPersister::new("/sdcard/store")); let mut md = MsgDriver::new(bytes); let (sequence, dbid) = read_serial_request_header(&mut md).expect("read init header"); assert_eq!(dbid, 0); From 00969c0ba4e1ffe24e68e671f9b72e8853c954bd Mon Sep 17 00:00:00 2001 From: Evan Feenstra Date: Mon, 18 Jul 2022 10:10:41 -0700 Subject: [PATCH 09/10] validate chan existence --- persister/src/persist_fs.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/persister/src/persist_fs.rs b/persister/src/persist_fs.rs index 74e7bf8..1c30322 100644 --- a/persister/src/persist_fs.rs +++ b/persister/src/persist_fs.rs @@ -65,6 +65,10 @@ impl Persist for FsPersister { let pk = hex::encode(node_id.serialize()); let id = NodeChannelId::new(node_id, &stub.id0); let chan_id = hex::encode(id.channel_id().as_slice()); + // should not exist + if let Ok(_) = self.channels.get(&pk, &chan_id) { + return Err(()); // already exists + } let entry = ChannelEntry { channel_value_satoshis: 0, channel_setup: None, @@ -99,6 +103,10 @@ impl Persist for FsPersister { let pk = hex::encode(node_id.serialize()); let id = NodeChannelId::new(node_id, &channel.id0); let chan_id = hex::encode(id.channel_id().as_slice()); + // should exist + if let Err(_) = self.channels.get(&pk, &chan_id) { + return Err(()); // not found + } let entry = ChannelEntry { id: if channel.id.is_none() { Some(id.channel_id()) From 8b46e2c582eaf0c9c971ca01e3b3ed2fb32037f9 Mon Sep 17 00:00:00 2001 From: decentclock Date: Mon, 18 Jul 2022 13:55:42 -0600 Subject: [PATCH 10/10] Persist multiple channels to the same peer --- persister/src/persist_fs.rs | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/persister/src/persist_fs.rs b/persister/src/persist_fs.rs index 1c30322..f25b95e 100644 --- a/persister/src/persist_fs.rs +++ b/persister/src/persist_fs.rs @@ -43,6 +43,11 @@ impl FsPersister { } } +fn get_channel_key(channel_id: &[u8]) -> &[u8] { + let length = channel_id.len(); + channel_id.get(length-11..length-7).unwrap() +} + impl Persist for FsPersister { fn new_node(&self, node_id: &PublicKey, config: &NodeConfig, seed: &[u8]) { let pk = hex::encode(node_id.serialize()); @@ -63,16 +68,15 @@ impl Persist for FsPersister { } fn new_channel(&self, node_id: &PublicKey, stub: &ChannelStub) -> Result<(), ()> { let pk = hex::encode(node_id.serialize()); - let id = NodeChannelId::new(node_id, &stub.id0); - let chan_id = hex::encode(id.channel_id().as_slice()); + let chan_id = hex::encode(get_channel_key(stub.id0.as_slice())); // should not exist if let Ok(_) = self.channels.get(&pk, &chan_id) { return Err(()); // already exists } let entry = ChannelEntry { + id: Some(stub.id0.clone()), channel_value_satoshis: 0, channel_setup: None, - id: Some(id.channel_id()), enforcement_state: EnforcementState::new(0), }; let _ = self.channels.put(&pk, &chan_id, entry); @@ -101,18 +105,13 @@ impl Persist for FsPersister { } fn update_channel(&self, node_id: &PublicKey, channel: &Channel) -> Result<(), ()> { let pk = hex::encode(node_id.serialize()); - let id = NodeChannelId::new(node_id, &channel.id0); - let chan_id = hex::encode(id.channel_id().as_slice()); + let chan_id = hex::encode(get_channel_key(channel.id0.as_slice())); // should exist if let Err(_) = self.channels.get(&pk, &chan_id) { return Err(()); // not found } let entry = ChannelEntry { - id: if channel.id.is_none() { - Some(id.channel_id()) - } else { - channel.id.clone() - }, + id: Some(channel.id0.clone()), channel_value_satoshis: channel.setup.channel_value_sat, channel_setup: Some(channel.setup.clone()), enforcement_state: channel.enforcement_state.clone(), @@ -126,8 +125,7 @@ impl Persist for FsPersister { channel_id: &ChannelId, ) -> Result { let pk = hex::encode(node_id.serialize()); - let id = NodeChannelId::new(node_id, channel_id); - let chan_id = hex::encode(id.channel_id().as_slice()); + let chan_id = hex::encode(get_channel_key(channel_id.as_slice())); let ret: ChannelEntry = match self.channels.get(&pk, &chan_id) { Ok(ce) => ce, Err(_) => return Err(()),