Merge pull request #45 from stakwork/persistance2

Persistance2

Cargo.toml

@@ -11,7 +11,8 @@ members = [
 exclude = [
     "sphinx-key",
     "crypter",
-    "crypter-ffi"
+    "crypter-ffi",
+    "persister",
 ]
 
 [patch.crates-io]

parser/Cargo.toml

@@ -5,7 +5,7 @@ edition = "2021"
 
 [dependencies]
 # vls-protocol = { path = "../../../evanf/validating-lightning-signer/vls-protocol" }
-vls-protocol = { git = "https://gitlab.com/Evanfeenstra/validating-lightning-signer", branch = "sphinx-key-1" }
+vls-protocol = { git = "https://gitlab.com/Evanfeenstra/validating-lightning-signer", branch = "patch-sign-chan" }
 serde = { version = "1.0", default-features = false }
 serde_bolt = { version = "0.2", default-features = false }
 

persister/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "sphinx-key-persister"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+lightning-signer-core = { git = "https://gitlab.com/Evanfeenstra/validating-lightning-signer", branch = "patch-sign-chan", default-features = false, features = ["std", "secp-lowmemory"] }
+lightning-signer-server = { git = "https://gitlab.com/Evanfeenstra/validating-lightning-signer", branch = "patch-sign-chan", default-features = false, features = ["persist"] }
+serde = { version = "1.0.105" }
+hex = "0.4.3"
+fsdb = "0.1.8"

persister/src/lib.rs

@@ -0,0 +1 @@
+pub mod persist_fs;

persister/src/persist_fs.rs

@@ -0,0 +1,186 @@
+use fsdb::{Bucket, DoubleBucket, Fsdb};
+use lightning_signer::persist::Persist;
+use lightning_signer_server::persist::model::{ChannelEntry, NodeEntry};
+use std::string::String;
+
+use lightning_signer::bitcoin::secp256k1::PublicKey;
+use lightning_signer::chain::tracker::ChainTracker;
+use lightning_signer::channel::Channel;
+use lightning_signer::channel::ChannelId;
+use lightning_signer::channel::ChannelStub;
+use lightning_signer::monitor::ChainMonitor;
+use lightning_signer::node::NodeConfig;
+use lightning_signer::policy::validator::EnforcementState;
+use lightning_signer_server::persist::model::AllowlistItemEntry;
+use lightning_signer_server::persist::model::ChainTrackerEntry;
+use lightning_signer_server::persist::model::NodeChannelId;
+
+use lightning_signer::persist::model::{
+    ChannelEntry as CoreChannelEntry, NodeEntry as CoreNodeEntry,
+};
+
+const FAT32_MAXFILENAMESIZE: usize = 8;
+
+pub struct FsPersister {
+    nodes: Bucket<NodeEntry>,
+    channels: DoubleBucket<ChannelEntry>,
+    allowlist: Bucket<AllowlistItemEntry>,
+    chaintracker: Bucket<ChainTrackerEntry>,
+    pubkeys: Bucket<PublicKey>,
+}
+
+impl FsPersister {
+    pub fn new(dir: &str) -> Self {
+        let db = Fsdb::new(dir).expect("could not create db");
+        let max = Some(FAT32_MAXFILENAMESIZE);
+        Self {
+            nodes: db.bucket("nodes", max).expect("fail nodes"),
+            channels: db.double_bucket("channel", max).expect("fail channel"),
+            allowlist: db.bucket("allowlis", max).expect("fail allowlis"),
+            chaintracker: db.bucket("chaintra", max).expect("fail chaintra"),
+            pubkeys: db.bucket("pubkey", max).expect("fail pubkey"),
+        }
+    }
+}
+
+fn get_channel_key(channel_id: &[u8]) -> &[u8] {
+        let length = channel_id.len();
+        channel_id.get(length-11..length-7).unwrap()
+}
+
+impl Persist for FsPersister {
+    fn new_node(&self, node_id: &PublicKey, config: &NodeConfig, seed: &[u8]) {
+        let pk = hex::encode(node_id.serialize());
+        let entry = NodeEntry {
+            seed: seed.to_vec(),
+            key_derivation_style: config.key_derivation_style as u8,
+            network: config.network.to_string(),
+        };
+        let _ = self.nodes.put(&pk, entry);
+        let _ = self.pubkeys.put(&pk, node_id.clone());
+    }
+    fn delete_node(&self, node_id: &PublicKey) {
+        let pk = hex::encode(node_id.serialize());
+        // clear all channel entries within "pk" sub-bucket
+        let _ = self.channels.clear(&pk);
+        let _ = self.nodes.remove(&pk);
+        let _ = self.pubkeys.remove(&pk);
+    }
+    fn new_channel(&self, node_id: &PublicKey, stub: &ChannelStub) -> Result<(), ()> {
+        let pk = hex::encode(node_id.serialize());
+        let chan_id = hex::encode(get_channel_key(stub.id0.as_slice()));
+        // should not exist
+        if let Ok(_) = self.channels.get(&pk, &chan_id) {
+            return Err(()); // already exists
+        }
+        let entry = ChannelEntry {
+            id: Some(stub.id0.clone()),
+            channel_value_satoshis: 0,
+            channel_setup: None,
+            enforcement_state: EnforcementState::new(0),
+        };
+        let _ = self.channels.put(&pk, &chan_id, entry);
+        Ok(())
+    }
+    fn new_chain_tracker(&self, node_id: &PublicKey, tracker: &ChainTracker<ChainMonitor>) {
+        let pk = hex::encode(node_id.serialize());
+        let _ = self.chaintracker.put(&pk, tracker.into());
+    }
+    fn update_tracker(
+        &self,
+        node_id: &PublicKey,
+        tracker: &ChainTracker<ChainMonitor>,
+    ) -> Result<(), ()> {
+        let pk = hex::encode(node_id.serialize());
+        let _ = self.chaintracker.put(&pk, tracker.into());
+        Ok(())
+    }
+    fn get_tracker(&self, node_id: &PublicKey) -> Result<ChainTracker<ChainMonitor>, ()> {
+        let pk = hex::encode(node_id.serialize());
+        let ret: ChainTrackerEntry = match self.chaintracker.get(&pk) {
+            Ok(ct) => ct,
+            Err(_) => return Err(()),
+        };
+        Ok(ret.into())
+    }
+    fn update_channel(&self, node_id: &PublicKey, channel: &Channel) -> Result<(), ()> {
+        let pk = hex::encode(node_id.serialize());
+        let chan_id = hex::encode(get_channel_key(channel.id0.as_slice()));
+        // should exist
+        if let Err(_) = self.channels.get(&pk, &chan_id) {
+            return Err(()); // not found
+        }
+        let entry = ChannelEntry {
+            id: Some(channel.id0.clone()),
+            channel_value_satoshis: channel.setup.channel_value_sat,
+            channel_setup: Some(channel.setup.clone()),
+            enforcement_state: channel.enforcement_state.clone(),
+        };
+        let _ = self.channels.put(&pk, &chan_id, entry);
+        Ok(())
+    }
+    fn get_channel(
+        &self,
+        node_id: &PublicKey,
+        channel_id: &ChannelId,
+    ) -> Result<CoreChannelEntry, ()> {
+        let pk = hex::encode(node_id.serialize());
+        let chan_id = hex::encode(get_channel_key(channel_id.as_slice()));
+        let ret: ChannelEntry = match self.channels.get(&pk, &chan_id) {
+            Ok(ce) => ce,
+            Err(_) => return Err(()),
+        };
+        Ok(ret.into())
+    }
+    fn get_node_channels(&self, node_id: &PublicKey) -> Vec<(ChannelId, CoreChannelEntry)> {
+        let mut res = Vec::new();
+        let pk = hex::encode(node_id.serialize());
+        let list = match self.channels.list(&pk) {
+            Ok(l) => l,
+            Err(_) => return res,
+        };
+        for channel in list {
+            if let Ok(entry) = self.channels.get(&pk, &channel) {
+                let id = entry.id.clone().unwrap();
+                res.push((id, entry.into()));
+            };
+        }
+        res
+    }
+    fn update_node_allowlist(&self, node_id: &PublicKey, allowlist: Vec<String>) -> Result<(), ()> {
+        let pk = hex::encode(node_id.serialize());
+        let entry = AllowlistItemEntry { allowlist };
+        let _ = self.allowlist.put(&pk, entry);
+        Ok(())
+    }
+    fn get_node_allowlist(&self, node_id: &PublicKey) -> Vec<String> {
+        let pk = hex::encode(node_id.serialize());
+        let entry: AllowlistItemEntry = match self.allowlist.get(&pk) {
+            Ok(e) => e,
+            Err(_) => return Vec::new(),
+        };
+        entry.allowlist
+    }
+    fn get_nodes(&self) -> Vec<(PublicKey, CoreNodeEntry)> {
+        let mut res = Vec::new();
+        let list = match self.nodes.list() {
+            Ok(ns) => ns,
+            Err(_) => return res,
+        };
+        for pk in list {
+            if let Ok(pubkey) = self.pubkeys.get(&pk) {
+                if let Ok(node) = self.nodes.get(&pk) {
+                    res.push((pubkey, node.into()));
+                }
+            }
+        }
+        res
+    }
+    fn clear_database(&self) {
+        let _ = self.nodes.clear();
+        let _ = self.channels.clear_all();
+        let _ = self.allowlist.clear();
+        let _ = self.chaintracker.clear();
+        let _ = self.pubkeys.clear();
+    }
+}

signer/Cargo.toml

@@ -6,9 +6,10 @@ edition = "2018"
 
 [dependencies]
 sphinx-key-parser = { path = "../parser" }
+sphinx-key-persister = { path = "../persister" }
 # vls-protocol-signer = { path = "../../../evanf/validating-lightning-signer/vls-protocol-signer", default-features = false, features = ["std", "secp-lowmemory"] }
 # vls-protocol-signer = { git = "https://gitlab.com/lightning-signer/validating-lightning-signer", default-features = false, features = ["secp-lowmemory"] }
-vls-protocol-signer = { git = "https://gitlab.com/Evanfeenstra/validating-lightning-signer", branch = "sphinx-key-1", default-features = false, features = ["std", "secp-lowmemory"] }
+vls-protocol-signer = { git = "https://gitlab.com/Evanfeenstra/validating-lightning-signer", branch = "patch-sign-chan", default-features = false, features = ["std", "secp-lowmemory"] }
 anyhow = {version = "1", features = ["backtrace"]}
 log = "0.4"
 

signer/src/lib.rs

@@ -10,6 +10,7 @@ pub use vls_protocol_signer::lightning_signer;
 pub use vls_protocol_signer::lightning_signer::bitcoin::Network;
 pub use vls_protocol_signer::vls_protocol;
 pub use sphinx_key_parser::MsgDriver;
+pub use sphinx_key_persister::persist_fs::FsPersister;
 
 pub struct InitResponse {
     pub root_handler: RootHandler,
@@ -17,7 +18,8 @@ pub struct InitResponse {
 }
 
 pub fn init(bytes: Vec<u8>, network: Network) -> anyhow::Result<InitResponse> {
-    let persister: Arc<dyn Persist> = Arc::new(DummyPersister);
+    //let persister: Arc<dyn Persist> = Arc::new(DummyPersister);
+    let persister: Arc<dyn Persist> = Arc::new(FsPersister::new("/sdcard/store"));
     let mut md = MsgDriver::new(bytes);
     let (sequence, dbid) = read_serial_request_header(&mut md).expect("read init header");
     assert_eq!(dbid, 0);

sphinx-key/Cargo.toml

@@ -19,6 +19,7 @@ pio = ["esp-idf-sys/pio"]
 pingpong = []
 
 [dependencies]
+bitflags = "1.3.2"
 esp-idf-sys = { version = "0.31.5", features = ["binstart"] }
 sphinx-key-signer = { path = "../signer", optional = true }
 sphinx-key-crypter = { path = "../crypter" }

sphinx-key/src/main.rs

@@ -5,7 +5,7 @@ mod periph;
 
 use crate::core::{config::*, events::*};
 use crate::periph::led::led_control_loop;
-use crate::periph::sd::sd_card;
+use crate::periph::sd::{sd_card, simple_fs_test};
 
 use anyhow::Result;
 use esp_idf_sys as _; // If using the `binstart` feature of `esp-idf-sys`, always keep this module imported
@@ -43,7 +43,8 @@ fn main() -> Result<()> {
     led_control_loop(pins.gpio8, peripherals.rmt.channel0, led_rx);
 
     // sd card
-    sd_card(peripherals.spi2);
+    sd_card();
+    simple_fs_test();
 
     let default_nvs = Arc::new(EspDefaultNvs::new()?);
     let mut store =

sphinx-key/src/periph/sd.rs

@@ -1,6 +1,201 @@
-
+use bitflags::bitflags;
 use esp_idf_hal::spi;
+use esp_idf_sys::c_types::c_char;
+use esp_idf_sys::{
+    esp, esp_vfs_fat_sdmmc_mount_config_t, esp_vfs_fat_sdspi_mount, gpio_num_t, sdmmc_card_t,
+    sdmmc_host_t, sdspi_device_config_t, spi_bus_config_t, spi_bus_initialize, spi_host_device_t,
+    spi_host_device_t_SPI2_HOST,
+};
+use std::fs;
+use std::io::BufReader;
+use std::io::BufWriter;
+use std::io::Read;
+use std::io::Write;
+use std::ptr;
+use std::string::String;
 
-pub fn sd_card(_spi: spi::SPI2) {
-    log::info!("sd_card");
-}
+pub const MOUNT_POINT: &'static str = "/sdcard";
+const C_MOUNT_POINT: &'static [u8] = b"/sdcard\0";
+
+const SPI_HOST_SLOT: spi_host_device_t = spi_host_device_t_SPI2_HOST;
+const SPI_GPIO_MOSI: gpio_num_t = 4;
+const SPI_GPIO_CLK: gpio_num_t = 5;
+const SPI_GPIO_MISO: gpio_num_t = 6;
+const SPI_GPIO_CS: gpio_num_t = 1;
+
+
+bitflags! {
+    struct SDMMCHostFlag: u32 {
+        /// host supports 1-line SD and MMC protocol
+        const BIT1 = 1 << 0;
+        /// host supports 4-line SD and MMC protocol
+        const BIT4 = 1 << 1;
+        /// host supports 8-line MMC protocol
+        const BIT8 = 1 << 2;
+        /// host supports SPI protocol
+        const SPI = 1 << 3;
+        /// host supports DDR mode for SD/MMC
+        const DDR = 1 << 4;
+        /// host `deinit` function called with the slot argument
+        const DEINIT_ARG = 1 << 5;
+    }
+}
+
+enum SDMMCFreq {
+    /// SD/MMC Default speed (limited by clock divider)
+    Default = 20000,
+    /// SD High speed (limited by clock divider)
+    HighSPeed = 40000,
+    /// SD/MMC probing speed
+    Probing = 400,
+    /// MMC 52MHz speed
+    _52M = 52000,
+    /// MMC 26MHz speed
+    _26M = 26000,
+}
+
+
+pub fn sd_card() {
+    let mount_config = esp_vfs_fat_sdmmc_mount_config_t {
+        format_if_mount_failed: false,
+        max_files: 5,
+        allocation_unit_size: 16 * 1024,
+    };
+
+    let mut card: *mut sdmmc_card_t = ptr::null_mut();
+
+    let bus_cfg = spi_bus_config_t {
+        __bindgen_anon_1: esp_idf_sys::spi_bus_config_t__bindgen_ty_1 {
+            mosi_io_num: SPI_GPIO_MOSI,
+        },
+        __bindgen_anon_2: esp_idf_sys::spi_bus_config_t__bindgen_ty_2 {
+            miso_io_num: SPI_GPIO_MISO,
+        },
+        sclk_io_num: SPI_GPIO_CLK,
+        __bindgen_anon_3: esp_idf_sys::spi_bus_config_t__bindgen_ty_3 { quadwp_io_num: -1 },
+        __bindgen_anon_4: esp_idf_sys::spi_bus_config_t__bindgen_ty_4 { quadhd_io_num: -1 },
+        data4_io_num: -1,
+        data5_io_num: -1,
+        data6_io_num: -1,
+        data7_io_num: -1,
+        max_transfer_sz: 4000,
+        flags: 0,
+        intr_flags: 0,
+    };
+
+    let res = esp!(unsafe {
+        spi_bus_initialize(
+            SPI_HOST_SLOT as u32,
+            &bus_cfg,
+            esp_idf_sys::spi_common_dma_t_SPI_DMA_CH_AUTO,
+        )
+    });
+
+    match res {
+        Ok(_) => (),
+        Err(e) => {
+            println!("Failed to initialize SPI Bus: {}", e);
+        }
+    }
+
+    println!("Initialized SPI BUS!");
+
+    let slot_config = sdspi_device_config_t {
+        host_id: SPI_HOST_SLOT,
+        gpio_cs: SPI_GPIO_CS,
+        gpio_cd: -1,
+        gpio_wp: -1,
+        gpio_int: -1,
+    };
+
+    let host = sdmmc_host_t {
+        flags: (SDMMCHostFlag::SPI | SDMMCHostFlag::DEINIT_ARG).bits, //SDMMC_HOST_FLAG_SPI | SDMMC_HOST_FLAG_DEINIT_ARG,
+        slot: SPI_HOST_SLOT as i32,
+        max_freq_khz: SDMMCFreq::Default as i32, //SDMMC_FREQ_DEFAULT,
+        io_voltage: 3.3f32,
+        init: Some(esp_idf_sys::sdspi_host_init),
+        set_bus_width: None,
+        get_bus_width: None,
+        set_bus_ddr_mode: None,
+        set_card_clk: Some(esp_idf_sys::sdspi_host_set_card_clk),
+        do_transaction: Some(esp_idf_sys::sdspi_host_do_transaction),
+        __bindgen_anon_1: esp_idf_sys::sdmmc_host_t__bindgen_ty_1 {
+            deinit_p: Some(esp_idf_sys::sdspi_host_remove_device),
+        },
+        io_int_enable: Some(esp_idf_sys::sdspi_host_io_int_enable),
+        io_int_wait: Some(esp_idf_sys::sdspi_host_io_int_wait),
+        command_timeout_ms: 0,
+    };
+
+    let res = esp!(unsafe {
+        esp_vfs_fat_sdspi_mount(
+            C_MOUNT_POINT.as_ptr() as *const c_char,
+            &host,
+            &slot_config,
+            &mount_config,
+            &mut card as *mut *mut sdmmc_card_t,
+        )
+    });
+
+    match res {
+        Ok(_) => (),
+        Err(e) => {
+            println!("Failed to mount filesystem: {}", e);
+        }
+    }
+}
+
+
+pub fn simple_fs_test() {
+    // Create and write a file, append a file, read a file, delete a file, create a directory, write a file
+
+    let file = fs::File::create("/sdcard/sphinx.txt").expect("create failed");
+    let mut writer = BufWriter::new(file);
+    writer
+        .write_all("Hello World".as_bytes())
+        .expect("write failed");
+    writer
+        .write_all("\nWelcome to your own network".as_bytes())
+        .expect("write failed");
+    writer.flush().expect("Problem flushing");
+    drop(writer);
+    println!("Data written to file");
+
+    let file = fs::OpenOptions::new()
+        .append(true)
+        .open("/sdcard/sphinx.txt")
+        .expect("cannot open file");
+    let mut writer = BufWriter::new(file);
+    writer
+        .write_all("\nDepeche Mode".as_bytes())
+        .expect("write failed");
+    writer
+        .write_all("\nWelcome to my World".as_bytes())
+        .expect("write failed");
+    writer.flush().expect("Problem flushing");
+    drop(writer);
+    println!("File append success");
+
+    let file = fs::File::open("/sdcard/sphinx.txt").expect("Cannot open file");
+    let mut reader = BufReader::new(file);
+    let mut contents = String::new();
+    reader
+        .read_to_string(&mut contents)
+        .expect("could not read file");
+    drop(reader);
+
+    fs::remove_file("/sdcard/sphinx.txt").expect("could not remove file");
+    println!("File is removed");
+
+    let _ = fs::create_dir("/sdcard/data");
+
+    let file = fs::File::create("/sdcard/data/sphinx.txt").expect("Cannot open file");
+    let mut writer = BufWriter::new(file);
+    writer
+        .write_all(contents.as_bytes())
+        .expect("Could not write");
+    writer.flush().expect("Problem flushing");
+    drop(writer);
+    println!("Wrote file in data directory");
+    println!("Simple fs test complete!");
+}