From 52ea111ea66fbeef43da691df0a084f13dec807a Mon Sep 17 00:00:00 2001 From: irriden Date: Thu, 19 Oct 2023 23:12:14 +0000 Subject: [PATCH] sphinx-key: tune configs to solve OOM on mainnet addblock msgs also pulls vls memory optimizations on these addblock msgs --- Cargo.lock | 27 +++++++++++++------------ broker/Cargo.lock | 37 ++++++++++++++++++----------------- broker/Cargo.toml | 12 ++++++------ sphinx-key/Cargo.toml | 6 +++--- sphinx-key/sdkconfig.defaults | 4 ++-- sphinx-key/src/conn/mqtt.rs | 11 ++++++----- 6 files changed, 50 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7a5466c..4ff5081 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -243,7 +243,7 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bolt-derive" version = "0.2.0" -source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5" +source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208" dependencies = [ "proc-macro2", "quote", @@ -1213,7 +1213,7 @@ checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" [[package]] name = "lss-connector" version = "0.1.0" -source = "git+https://github.com/stakwork/sphinx-rs.git?rev=9709ba23b49aca8d8ea09197c901913f44d5fa00#9709ba23b49aca8d8ea09197c901913f44d5fa00" +source = "git+https://github.com/stakwork/sphinx-rs.git?rev=1b6b89123171e621617125e8ab59faa246aa2ee3#1b6b89123171e621617125e8ab59faa246aa2ee3" dependencies = [ "anyhow", "log", @@ -1585,7 +1585,7 @@ dependencies = [ [[package]] name = "rmp-utils" version = "0.1.0" -source = "git+https://github.com/stakwork/sphinx-rs.git?rev=9709ba23b49aca8d8ea09197c901913f44d5fa00#9709ba23b49aca8d8ea09197c901913f44d5fa00" +source = "git+https://github.com/stakwork/sphinx-rs.git?rev=1b6b89123171e621617125e8ab59faa246aa2ee3#1b6b89123171e621617125e8ab59faa246aa2ee3" dependencies = [ "anyhow", "log", @@ -1792,7 +1792,7 @@ dependencies = [ [[package]] name = "sphinx-auther" version = "0.1.12" -source = "git+https://github.com/stakwork/sphinx-rs.git?rev=9709ba23b49aca8d8ea09197c901913f44d5fa00#9709ba23b49aca8d8ea09197c901913f44d5fa00" +source = "git+https://github.com/stakwork/sphinx-rs.git?rev=1b6b89123171e621617125e8ab59faa246aa2ee3#1b6b89123171e621617125e8ab59faa246aa2ee3" dependencies = [ "anyhow", "base64", @@ -1804,7 +1804,7 @@ dependencies = [ [[package]] name = "sphinx-crypter" version = "0.1.0" -source = "git+https://github.com/stakwork/sphinx-rs.git?rev=9709ba23b49aca8d8ea09197c901913f44d5fa00#9709ba23b49aca8d8ea09197c901913f44d5fa00" +source = "git+https://github.com/stakwork/sphinx-rs.git?rev=1b6b89123171e621617125e8ab59faa246aa2ee3#1b6b89123171e621617125e8ab59faa246aa2ee3" dependencies = [ "anyhow", "chacha20poly1305", @@ -1815,7 +1815,7 @@ dependencies = [ [[package]] name = "sphinx-glyph" version = "0.1.2" -source = "git+https://github.com/stakwork/sphinx-rs.git?rev=9709ba23b49aca8d8ea09197c901913f44d5fa00#9709ba23b49aca8d8ea09197c901913f44d5fa00" +source = "git+https://github.com/stakwork/sphinx-rs.git?rev=1b6b89123171e621617125e8ab59faa246aa2ee3#1b6b89123171e621617125e8ab59faa246aa2ee3" dependencies = [ "anyhow", "hex", @@ -1866,7 +1866,7 @@ dependencies = [ [[package]] name = "sphinx-signer" version = "0.1.0" -source = "git+https://github.com/stakwork/sphinx-rs.git?rev=9709ba23b49aca8d8ea09197c901913f44d5fa00#9709ba23b49aca8d8ea09197c901913f44d5fa00" +source = "git+https://github.com/stakwork/sphinx-rs.git?rev=1b6b89123171e621617125e8ab59faa246aa2ee3#1b6b89123171e621617125e8ab59faa246aa2ee3" dependencies = [ "anyhow", "bip39", @@ -2061,9 +2061,9 @@ dependencies = [ [[package]] name = "txoo" -version = "0.6.1" +version = "0.6.2-beta.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b35482e5bf458fa43996535afbca884b2562ab6419e20686340bb19f5305b30" +checksum = "15baf29308bcea1ef58d50bac31680bfde24f37116f82d853c3628ee33a6dc9d" dependencies = [ "bitcoin", "log", @@ -2148,7 +2148,7 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "vls-core" version = "0.10.0" -source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5" +source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208" dependencies = [ "anyhow", "bitcoin", @@ -2172,7 +2172,7 @@ dependencies = [ [[package]] name = "vls-persist" version = "0.10.0" -source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5" +source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208" dependencies = [ "hex", "log", @@ -2185,7 +2185,7 @@ dependencies = [ [[package]] name = "vls-protocol" version = "0.10.0" -source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5" +source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208" dependencies = [ "as-any", "bitcoin-consensus-derive", @@ -2193,12 +2193,13 @@ dependencies = [ "hex", "log", "serde_bolt", + "txoo", ] [[package]] name = "vls-protocol-signer" version = "0.10.0" -source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5" +source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208" dependencies = [ "bit-vec", "log", diff --git a/broker/Cargo.lock b/broker/Cargo.lock index f3de789..42c0b4f 100644 --- a/broker/Cargo.lock +++ b/broker/Cargo.lock @@ -442,7 +442,7 @@ dependencies = [ [[package]] name = "bolt-derive" version = "0.2.0" -source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5" +source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208" dependencies = [ "proc-macro2", "quote", @@ -1606,7 +1606,7 @@ dependencies = [ [[package]] name = "lightning-storage-server" version = "0.3.0" -source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5" +source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208" dependencies = [ "anyhow", "async-trait", @@ -1691,7 +1691,7 @@ dependencies = [ [[package]] name = "lss-connector" version = "0.1.0" -source = "git+https://github.com/stakwork/sphinx-rs?rev=9709ba23b49aca8d8ea09197c901913f44d5fa00#9709ba23b49aca8d8ea09197c901913f44d5fa00" +source = "git+https://github.com/stakwork/sphinx-rs?rev=1b6b89123171e621617125e8ab59faa246aa2ee3#1b6b89123171e621617125e8ab59faa246aa2ee3" dependencies = [ "anyhow", "lightning-storage-server", @@ -2693,7 +2693,7 @@ dependencies = [ [[package]] name = "rmp-utils" version = "0.1.0" -source = "git+https://github.com/stakwork/sphinx-rs?rev=9709ba23b49aca8d8ea09197c901913f44d5fa00#9709ba23b49aca8d8ea09197c901913f44d5fa00" +source = "git+https://github.com/stakwork/sphinx-rs?rev=1b6b89123171e621617125e8ab59faa246aa2ee3#1b6b89123171e621617125e8ab59faa246aa2ee3" dependencies = [ "anyhow", "log", @@ -3268,7 +3268,7 @@ dependencies = [ [[package]] name = "sphinx-auther" version = "0.1.12" -source = "git+https://github.com/stakwork/sphinx-rs?rev=9709ba23b49aca8d8ea09197c901913f44d5fa00#9709ba23b49aca8d8ea09197c901913f44d5fa00" +source = "git+https://github.com/stakwork/sphinx-rs?rev=1b6b89123171e621617125e8ab59faa246aa2ee3#1b6b89123171e621617125e8ab59faa246aa2ee3" dependencies = [ "anyhow", "base64 0.21.2", @@ -3280,7 +3280,7 @@ dependencies = [ [[package]] name = "sphinx-glyph" version = "0.1.2" -source = "git+https://github.com/stakwork/sphinx-rs?rev=9709ba23b49aca8d8ea09197c901913f44d5fa00#9709ba23b49aca8d8ea09197c901913f44d5fa00" +source = "git+https://github.com/stakwork/sphinx-rs?rev=1b6b89123171e621617125e8ab59faa246aa2ee3#1b6b89123171e621617125e8ab59faa246aa2ee3" dependencies = [ "anyhow", "hex", @@ -3326,7 +3326,7 @@ dependencies = [ [[package]] name = "sphinx-signer" version = "0.1.0" -source = "git+https://github.com/stakwork/sphinx-rs?rev=9709ba23b49aca8d8ea09197c901913f44d5fa00#9709ba23b49aca8d8ea09197c901913f44d5fa00" +source = "git+https://github.com/stakwork/sphinx-rs?rev=1b6b89123171e621617125e8ab59faa246aa2ee3#1b6b89123171e621617125e8ab59faa246aa2ee3" dependencies = [ "anyhow", "bip39", @@ -3881,9 +3881,9 @@ dependencies = [ [[package]] name = "txoo" -version = "0.6.1" +version = "0.6.2-beta.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b35482e5bf458fa43996535afbca884b2562ab6419e20686340bb19f5305b30" +checksum = "15baf29308bcea1ef58d50bac31680bfde24f37116f82d853c3628ee33a6dc9d" dependencies = [ "async-trait", "bitcoin", @@ -3897,9 +3897,9 @@ dependencies = [ [[package]] name = "txoo-bitcoind-client" -version = "0.6.1" +version = "0.6.2-beta.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92532d7c19761706c83ea91c66c14e650903379f14b8a8256d8e07975443a3a7" +checksum = "5d94932dc400898dd624373497d47e2140bd5b95be9f4b1d8aa4a9a30b2d6038" dependencies = [ "async-trait", "bitcoin", @@ -4030,7 +4030,7 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "vls-core" version = "0.10.0" -source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5" +source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208" dependencies = [ "anyhow", "backtrace", @@ -4056,7 +4056,7 @@ dependencies = [ [[package]] name = "vls-frontend" version = "0.10.0" -source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5" +source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208" dependencies = [ "async-trait", "lightning-storage-server", @@ -4070,7 +4070,7 @@ dependencies = [ [[package]] name = "vls-persist" version = "0.10.0" -source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5" +source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208" dependencies = [ "hex", "log", @@ -4085,7 +4085,7 @@ dependencies = [ [[package]] name = "vls-protocol" version = "0.10.0" -source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5" +source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208" dependencies = [ "as-any", "bitcoin-consensus-derive", @@ -4093,12 +4093,13 @@ dependencies = [ "hex", "log", "serde_bolt", + "txoo", ] [[package]] name = "vls-protocol-client" version = "0.10.0" -source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5" +source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208" dependencies = [ "anyhow", "async-trait", @@ -4113,7 +4114,7 @@ dependencies = [ [[package]] name = "vls-protocol-signer" version = "0.10.0" -source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5" +source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208" dependencies = [ "bit-vec", "log", @@ -4124,7 +4125,7 @@ dependencies = [ [[package]] name = "vls-proxy" version = "0.10.0" -source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5" +source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208" dependencies = [ "anyhow", "as-any", diff --git a/broker/Cargo.toml b/broker/Cargo.toml index 8b451f8..e25acac 100644 --- a/broker/Cargo.toml +++ b/broker/Cargo.toml @@ -30,17 +30,17 @@ thiserror = "1.0.31" toml = "0.5.9" url = { version = "2.2" } -vls-frontend = { git = "https://gitlab.com/lightning-signer/validating-lightning-signer.git", rev = "186e4b93161134fbdd7ec7fbd034a754e380ffa5" } -vls-protocol = { git = "https://gitlab.com/lightning-signer/validating-lightning-signer.git", rev = "186e4b93161134fbdd7ec7fbd034a754e380ffa5" } -vls-protocol-client = { git = "https://gitlab.com/lightning-signer/validating-lightning-signer.git", rev = "186e4b93161134fbdd7ec7fbd034a754e380ffa5" } -vls-proxy = { git = "https://gitlab.com/lightning-signer/validating-lightning-signer.git", rev = "186e4b93161134fbdd7ec7fbd034a754e380ffa5" } +vls-frontend = { git = "https://gitlab.com/lightning-signer/validating-lightning-signer.git", rev = "19447b1e5d2956aa72aeca4229da7a96c5eaf208" } +vls-protocol = { git = "https://gitlab.com/lightning-signer/validating-lightning-signer.git", rev = "19447b1e5d2956aa72aeca4229da7a96c5eaf208" } +vls-protocol-client = { git = "https://gitlab.com/lightning-signer/validating-lightning-signer.git", rev = "19447b1e5d2956aa72aeca4229da7a96c5eaf208" } +vls-proxy = { git = "https://gitlab.com/lightning-signer/validating-lightning-signer.git", rev = "19447b1e5d2956aa72aeca4229da7a96c5eaf208" } # vls-frontend = { path = "../../vls/vls-frontend" } # vls-protocol = { path = "../../vls/vls-protocol" } # vls-protocol-client = { path = "../../vls/vls-protocol-client" } # vls-proxy = { path = "../../vls/vls-proxy" } -lss-connector = { git = "https://github.com/stakwork/sphinx-rs", rev = "9709ba23b49aca8d8ea09197c901913f44d5fa00" } -sphinx-signer = { git = "https://github.com/stakwork/sphinx-rs", rev = "9709ba23b49aca8d8ea09197c901913f44d5fa00" } +lss-connector = { git = "https://github.com/stakwork/sphinx-rs", rev = "1b6b89123171e621617125e8ab59faa246aa2ee3" } +sphinx-signer = { git = "https://github.com/stakwork/sphinx-rs", rev = "1b6b89123171e621617125e8ab59faa246aa2ee3" } # lss-connector = { path = "../../sphinx-rs/lss-connector" } # sphinx-signer = { path = "../../sphinx-rs/signer" } diff --git a/sphinx-key/Cargo.toml b/sphinx-key/Cargo.toml index 7dda5c3..533d45f 100644 --- a/sphinx-key/Cargo.toml +++ b/sphinx-key/Cargo.toml @@ -20,9 +20,9 @@ serde_urlencoded = "0.7.1" url = "2" # sphinx-rs -lss-connector = { git = "https://github.com/stakwork/sphinx-rs.git", default-features = false, rev = "9709ba23b49aca8d8ea09197c901913f44d5fa00" } -sphinx-crypter = { git = "https://github.com/stakwork/sphinx-rs.git", rev = "9709ba23b49aca8d8ea09197c901913f44d5fa00" } -sphinx-signer = { git = "https://github.com/stakwork/sphinx-rs.git", optional = true, rev = "9709ba23b49aca8d8ea09197c901913f44d5fa00" } +lss-connector = { git = "https://github.com/stakwork/sphinx-rs.git", default-features = false, rev = "1b6b89123171e621617125e8ab59faa246aa2ee3" } +sphinx-crypter = { git = "https://github.com/stakwork/sphinx-rs.git", rev = "1b6b89123171e621617125e8ab59faa246aa2ee3" } +sphinx-signer = { git = "https://github.com/stakwork/sphinx-rs.git", optional = true, rev = "1b6b89123171e621617125e8ab59faa246aa2ee3" } # local # lss-connector = { path = "../../sphinx-rs/lss-connector", default-features = false } # sphinx-crypter = { path = "../../sphinx-rs/crypter" } diff --git a/sphinx-key/sdkconfig.defaults b/sphinx-key/sdkconfig.defaults index 604fef2..3f7bc5c 100644 --- a/sphinx-key/sdkconfig.defaults +++ b/sphinx-key/sdkconfig.defaults @@ -1,6 +1,6 @@ # Rust often needs a bit of an extra main task stack size compared to C (the default is 3K) -CONFIG_ESP_MAIN_TASK_STACK_SIZE=64000 -CONFIG_PTHREAD_TASK_STACK_SIZE_DEFAULT=10000 +CONFIG_ESP_MAIN_TASK_STACK_SIZE=32000 +CONFIG_PTHREAD_TASK_STACK_SIZE_DEFAULT=2500 # CONFIG_LOG_DEFAULT_LEVEL_DEBUG=y CONFIG_FATFS_LFN_STACK=y diff --git a/sphinx-key/src/conn/mqtt.rs b/sphinx-key/src/conn/mqtt.rs index 9d075f7..823152b 100644 --- a/sphinx-key/src/conn/mqtt.rs +++ b/sphinx-key/src/conn/mqtt.rs @@ -28,8 +28,8 @@ pub fn make_client( let mut conf = MqttClientConfiguration { client_id: Some(&client_id), - buffer_size: 4096, - task_stack: 12288, + //buffer_size: 1024, + //task_stack: 12288, username: Some(username), password: Some(password), ..Default::default() @@ -54,7 +54,8 @@ pub fn make_client( info!("MQTT client started"); - thread::spawn(move || { + let builder = thread::Builder::new().stack_size(1524); + builder.spawn(move || { info!("MQTT Listening for messages"); let mut inflight = Vec::new(); let mut inflight_topic = "".to_string(); @@ -113,7 +114,7 @@ pub fn make_client( } } }; - + drop(msg); if let Some((topic, data)) = incoming_message { if topic.ends_with(topics::VLS) { tx.send(CoreEvent::VlsMessage(data)) @@ -141,7 +142,7 @@ pub fn make_client( } } //info!("MQTT connection loop exit"); - }); + })?; Ok(client) }