sphinx-key: tune configs to solve OOM on mainnet addblock msgs

also pulls vls memory optimizations on these addblock msgs
2025-12-17 07:14:23 +01:00 · 2023-10-19 23:12:14 +00:00
parent 66045569ac
commit 52ea111ea6
6 changed files with 50 additions and 47 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -243,7 +243,7 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 [[package]]
 name = "bolt-derive"
 version = "0.2.0"
-source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5"
+source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -1213,7 +1213,7 @@ checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4"
 [[package]]
 name = "lss-connector"
 version = "0.1.0"
-source = "git+https://github.com/stakwork/sphinx-rs.git?rev=9709ba23b49aca8d8ea09197c901913f44d5fa00#9709ba23b49aca8d8ea09197c901913f44d5fa00"
+source = "git+https://github.com/stakwork/sphinx-rs.git?rev=1b6b89123171e621617125e8ab59faa246aa2ee3#1b6b89123171e621617125e8ab59faa246aa2ee3"
 dependencies = [
 "anyhow",
 "log",
@@ -1585,7 +1585,7 @@ dependencies = [
 [[package]]
 name = "rmp-utils"
 version = "0.1.0"
-source = "git+https://github.com/stakwork/sphinx-rs.git?rev=9709ba23b49aca8d8ea09197c901913f44d5fa00#9709ba23b49aca8d8ea09197c901913f44d5fa00"
+source = "git+https://github.com/stakwork/sphinx-rs.git?rev=1b6b89123171e621617125e8ab59faa246aa2ee3#1b6b89123171e621617125e8ab59faa246aa2ee3"
 dependencies = [
 "anyhow",
 "log",
@@ -1792,7 +1792,7 @@ dependencies = [
 [[package]]
 name = "sphinx-auther"
 version = "0.1.12"
-source = "git+https://github.com/stakwork/sphinx-rs.git?rev=9709ba23b49aca8d8ea09197c901913f44d5fa00#9709ba23b49aca8d8ea09197c901913f44d5fa00"
+source = "git+https://github.com/stakwork/sphinx-rs.git?rev=1b6b89123171e621617125e8ab59faa246aa2ee3#1b6b89123171e621617125e8ab59faa246aa2ee3"
 dependencies = [
 "anyhow",
 "base64",
@@ -1804,7 +1804,7 @@ dependencies = [
 [[package]]
 name = "sphinx-crypter"
 version = "0.1.0"
-source = "git+https://github.com/stakwork/sphinx-rs.git?rev=9709ba23b49aca8d8ea09197c901913f44d5fa00#9709ba23b49aca8d8ea09197c901913f44d5fa00"
+source = "git+https://github.com/stakwork/sphinx-rs.git?rev=1b6b89123171e621617125e8ab59faa246aa2ee3#1b6b89123171e621617125e8ab59faa246aa2ee3"
 dependencies = [
 "anyhow",
 "chacha20poly1305",
@@ -1815,7 +1815,7 @@ dependencies = [
 [[package]]
 name = "sphinx-glyph"
 version = "0.1.2"
-source = "git+https://github.com/stakwork/sphinx-rs.git?rev=9709ba23b49aca8d8ea09197c901913f44d5fa00#9709ba23b49aca8d8ea09197c901913f44d5fa00"
+source = "git+https://github.com/stakwork/sphinx-rs.git?rev=1b6b89123171e621617125e8ab59faa246aa2ee3#1b6b89123171e621617125e8ab59faa246aa2ee3"
 dependencies = [
 "anyhow",
 "hex",
@@ -1866,7 +1866,7 @@ dependencies = [
 [[package]]
 name = "sphinx-signer"
 version = "0.1.0"
-source = "git+https://github.com/stakwork/sphinx-rs.git?rev=9709ba23b49aca8d8ea09197c901913f44d5fa00#9709ba23b49aca8d8ea09197c901913f44d5fa00"
+source = "git+https://github.com/stakwork/sphinx-rs.git?rev=1b6b89123171e621617125e8ab59faa246aa2ee3#1b6b89123171e621617125e8ab59faa246aa2ee3"
 dependencies = [
 "anyhow",
 "bip39",
@@ -2061,9 +2061,9 @@ dependencies = [

 [[package]]
 name = "txoo"
-version = "0.6.1"
+version = "0.6.2-beta.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b35482e5bf458fa43996535afbca884b2562ab6419e20686340bb19f5305b30"
+checksum = "15baf29308bcea1ef58d50bac31680bfde24f37116f82d853c3628ee33a6dc9d"
 dependencies = [
 "bitcoin",
 "log",
@@ -2148,7 +2148,7 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 [[package]]
 name = "vls-core"
 version = "0.10.0"
-source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5"
+source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208"
 dependencies = [
 "anyhow",
 "bitcoin",
@@ -2172,7 +2172,7 @@ dependencies = [
 [[package]]
 name = "vls-persist"
 version = "0.10.0"
-source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5"
+source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208"
 dependencies = [
 "hex",
 "log",
@@ -2185,7 +2185,7 @@ dependencies = [
 [[package]]
 name = "vls-protocol"
 version = "0.10.0"
-source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5"
+source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208"
 dependencies = [
 "as-any",
 "bitcoin-consensus-derive",
@@ -2193,12 +2193,13 @@ dependencies = [
 "hex",
 "log",
 "serde_bolt",
+ "txoo",
 ]

 [[package]]
 name = "vls-protocol-signer"
 version = "0.10.0"
-source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5"
+source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208"
 dependencies = [
 "bit-vec",
 "log",
--- a/broker/Cargo.lock
+++ b/broker/Cargo.lock
@@ -442,7 +442,7 @@ dependencies = [
 [[package]]
 name = "bolt-derive"
 version = "0.2.0"
-source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5"
+source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -1606,7 +1606,7 @@ dependencies = [
 [[package]]
 name = "lightning-storage-server"
 version = "0.3.0"
-source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5"
+source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208"
 dependencies = [
 "anyhow",
 "async-trait",
@@ -1691,7 +1691,7 @@ dependencies = [
 [[package]]
 name = "lss-connector"
 version = "0.1.0"
-source = "git+https://github.com/stakwork/sphinx-rs?rev=9709ba23b49aca8d8ea09197c901913f44d5fa00#9709ba23b49aca8d8ea09197c901913f44d5fa00"
+source = "git+https://github.com/stakwork/sphinx-rs?rev=1b6b89123171e621617125e8ab59faa246aa2ee3#1b6b89123171e621617125e8ab59faa246aa2ee3"
 dependencies = [
 "anyhow",
 "lightning-storage-server",
@@ -2693,7 +2693,7 @@ dependencies = [
 [[package]]
 name = "rmp-utils"
 version = "0.1.0"
-source = "git+https://github.com/stakwork/sphinx-rs?rev=9709ba23b49aca8d8ea09197c901913f44d5fa00#9709ba23b49aca8d8ea09197c901913f44d5fa00"
+source = "git+https://github.com/stakwork/sphinx-rs?rev=1b6b89123171e621617125e8ab59faa246aa2ee3#1b6b89123171e621617125e8ab59faa246aa2ee3"
 dependencies = [
 "anyhow",
 "log",
@@ -3268,7 +3268,7 @@ dependencies = [
 [[package]]
 name = "sphinx-auther"
 version = "0.1.12"
-source = "git+https://github.com/stakwork/sphinx-rs?rev=9709ba23b49aca8d8ea09197c901913f44d5fa00#9709ba23b49aca8d8ea09197c901913f44d5fa00"
+source = "git+https://github.com/stakwork/sphinx-rs?rev=1b6b89123171e621617125e8ab59faa246aa2ee3#1b6b89123171e621617125e8ab59faa246aa2ee3"
 dependencies = [
 "anyhow",
 "base64 0.21.2",
@@ -3280,7 +3280,7 @@ dependencies = [
 [[package]]
 name = "sphinx-glyph"
 version = "0.1.2"
-source = "git+https://github.com/stakwork/sphinx-rs?rev=9709ba23b49aca8d8ea09197c901913f44d5fa00#9709ba23b49aca8d8ea09197c901913f44d5fa00"
+source = "git+https://github.com/stakwork/sphinx-rs?rev=1b6b89123171e621617125e8ab59faa246aa2ee3#1b6b89123171e621617125e8ab59faa246aa2ee3"
 dependencies = [
 "anyhow",
 "hex",
@@ -3326,7 +3326,7 @@ dependencies = [
 [[package]]
 name = "sphinx-signer"
 version = "0.1.0"
-source = "git+https://github.com/stakwork/sphinx-rs?rev=9709ba23b49aca8d8ea09197c901913f44d5fa00#9709ba23b49aca8d8ea09197c901913f44d5fa00"
+source = "git+https://github.com/stakwork/sphinx-rs?rev=1b6b89123171e621617125e8ab59faa246aa2ee3#1b6b89123171e621617125e8ab59faa246aa2ee3"
 dependencies = [
 "anyhow",
 "bip39",
@@ -3881,9 +3881,9 @@ dependencies = [

 [[package]]
 name = "txoo"
-version = "0.6.1"
+version = "0.6.2-beta.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b35482e5bf458fa43996535afbca884b2562ab6419e20686340bb19f5305b30"
+checksum = "15baf29308bcea1ef58d50bac31680bfde24f37116f82d853c3628ee33a6dc9d"
 dependencies = [
 "async-trait",
 "bitcoin",
@@ -3897,9 +3897,9 @@ dependencies = [

 [[package]]
 name = "txoo-bitcoind-client"
-version = "0.6.1"
+version = "0.6.2-beta.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92532d7c19761706c83ea91c66c14e650903379f14b8a8256d8e07975443a3a7"
+checksum = "5d94932dc400898dd624373497d47e2140bd5b95be9f4b1d8aa4a9a30b2d6038"
 dependencies = [
 "async-trait",
 "bitcoin",
@@ -4030,7 +4030,7 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 [[package]]
 name = "vls-core"
 version = "0.10.0"
-source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5"
+source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208"
 dependencies = [
 "anyhow",
 "backtrace",
@@ -4056,7 +4056,7 @@ dependencies = [
 [[package]]
 name = "vls-frontend"
 version = "0.10.0"
-source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5"
+source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208"
 dependencies = [
 "async-trait",
 "lightning-storage-server",
@@ -4070,7 +4070,7 @@ dependencies = [
 [[package]]
 name = "vls-persist"
 version = "0.10.0"
-source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5"
+source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208"
 dependencies = [
 "hex",
 "log",
@@ -4085,7 +4085,7 @@ dependencies = [
 [[package]]
 name = "vls-protocol"
 version = "0.10.0"
-source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5"
+source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208"
 dependencies = [
 "as-any",
 "bitcoin-consensus-derive",
@@ -4093,12 +4093,13 @@ dependencies = [
 "hex",
 "log",
 "serde_bolt",
+ "txoo",
 ]

 [[package]]
 name = "vls-protocol-client"
 version = "0.10.0"
-source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5"
+source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208"
 dependencies = [
 "anyhow",
 "async-trait",
@@ -4113,7 +4114,7 @@ dependencies = [
 [[package]]
 name = "vls-protocol-signer"
 version = "0.10.0"
-source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5"
+source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208"
 dependencies = [
 "bit-vec",
 "log",
@@ -4124,7 +4125,7 @@ dependencies = [
 [[package]]
 name = "vls-proxy"
 version = "0.10.0"
-source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=186e4b93161134fbdd7ec7fbd034a754e380ffa5#186e4b93161134fbdd7ec7fbd034a754e380ffa5"
+source = "git+https://gitlab.com/lightning-signer/validating-lightning-signer.git?rev=19447b1e5d2956aa72aeca4229da7a96c5eaf208#19447b1e5d2956aa72aeca4229da7a96c5eaf208"
 dependencies = [
 "anyhow",
 "as-any",
--- a/broker/Cargo.toml
+++ b/broker/Cargo.toml
@@ -30,17 +30,17 @@ thiserror         = "1.0.31"
 toml              = "0.5.9"
 url               = { version = "2.2" }

-vls-frontend        = { git = "https://gitlab.com/lightning-signer/validating-lightning-signer.git", rev = "186e4b93161134fbdd7ec7fbd034a754e380ffa5" }
-vls-protocol        = { git = "https://gitlab.com/lightning-signer/validating-lightning-signer.git", rev = "186e4b93161134fbdd7ec7fbd034a754e380ffa5" }
-vls-protocol-client = { git = "https://gitlab.com/lightning-signer/validating-lightning-signer.git", rev = "186e4b93161134fbdd7ec7fbd034a754e380ffa5" }
-vls-proxy           = { git = "https://gitlab.com/lightning-signer/validating-lightning-signer.git", rev = "186e4b93161134fbdd7ec7fbd034a754e380ffa5" }
+vls-frontend        = { git = "https://gitlab.com/lightning-signer/validating-lightning-signer.git", rev = "19447b1e5d2956aa72aeca4229da7a96c5eaf208" }
+vls-protocol        = { git = "https://gitlab.com/lightning-signer/validating-lightning-signer.git", rev = "19447b1e5d2956aa72aeca4229da7a96c5eaf208" }
+vls-protocol-client = { git = "https://gitlab.com/lightning-signer/validating-lightning-signer.git", rev = "19447b1e5d2956aa72aeca4229da7a96c5eaf208" }
+vls-proxy           = { git = "https://gitlab.com/lightning-signer/validating-lightning-signer.git", rev = "19447b1e5d2956aa72aeca4229da7a96c5eaf208" }
 # vls-frontend        = { path = "../../vls/vls-frontend" }
 # vls-protocol        = { path = "../../vls/vls-protocol" }
 # vls-protocol-client = { path = "../../vls/vls-protocol-client" }
 # vls-proxy           = { path = "../../vls/vls-proxy" }

-lss-connector = { git = "https://github.com/stakwork/sphinx-rs", rev = "9709ba23b49aca8d8ea09197c901913f44d5fa00" }
-sphinx-signer = { git = "https://github.com/stakwork/sphinx-rs", rev = "9709ba23b49aca8d8ea09197c901913f44d5fa00" }
+lss-connector = { git = "https://github.com/stakwork/sphinx-rs", rev = "1b6b89123171e621617125e8ab59faa246aa2ee3" }
+sphinx-signer = { git = "https://github.com/stakwork/sphinx-rs", rev = "1b6b89123171e621617125e8ab59faa246aa2ee3" }
 # lss-connector = { path = "../../sphinx-rs/lss-connector" }
 # sphinx-signer = { path = "../../sphinx-rs/signer" }

--- a/sphinx-key/Cargo.toml
+++ b/sphinx-key/Cargo.toml
@@ -20,9 +20,9 @@ serde_urlencoded = "0.7.1"
 url              = "2"

 # sphinx-rs
-lss-connector  = { git = "https://github.com/stakwork/sphinx-rs.git", default-features = false, rev = "9709ba23b49aca8d8ea09197c901913f44d5fa00" }
-sphinx-crypter = { git = "https://github.com/stakwork/sphinx-rs.git", rev = "9709ba23b49aca8d8ea09197c901913f44d5fa00" }
-sphinx-signer  = { git = "https://github.com/stakwork/sphinx-rs.git", optional = true, rev = "9709ba23b49aca8d8ea09197c901913f44d5fa00" }
+lss-connector  = { git = "https://github.com/stakwork/sphinx-rs.git", default-features = false, rev = "1b6b89123171e621617125e8ab59faa246aa2ee3" }
+sphinx-crypter = { git = "https://github.com/stakwork/sphinx-rs.git", rev = "1b6b89123171e621617125e8ab59faa246aa2ee3" }
+sphinx-signer  = { git = "https://github.com/stakwork/sphinx-rs.git", optional = true, rev = "1b6b89123171e621617125e8ab59faa246aa2ee3" }
 # local
 # lss-connector  = { path = "../../sphinx-rs/lss-connector", default-features = false }
 # sphinx-crypter  = { path = "../../sphinx-rs/crypter" }
--- a/sphinx-key/sdkconfig.defaults
+++ b/sphinx-key/sdkconfig.defaults
@@ -1,6 +1,6 @@
 # Rust often needs a bit of an extra main task stack size compared to C (the default is 3K)
-CONFIG_ESP_MAIN_TASK_STACK_SIZE=64000
-CONFIG_PTHREAD_TASK_STACK_SIZE_DEFAULT=10000
+CONFIG_ESP_MAIN_TASK_STACK_SIZE=32000
+CONFIG_PTHREAD_TASK_STACK_SIZE_DEFAULT=2500
 # CONFIG_LOG_DEFAULT_LEVEL_DEBUG=y
 CONFIG_FATFS_LFN_STACK=y

--- a/sphinx-key/src/conn/mqtt.rs
+++ b/sphinx-key/src/conn/mqtt.rs
@@ -28,8 +28,8 @@ pub fn make_client(

    let mut conf = MqttClientConfiguration {
        client_id: Some(&client_id),
-        buffer_size: 4096,
-        task_stack: 12288,
+        //buffer_size: 1024,
+        //task_stack: 12288,
        username: Some(username),
        password: Some(password),
        ..Default::default()
@@ -54,7 +54,8 @@ pub fn make_client(

    info!("MQTT client started");

-    thread::spawn(move || {
+    let builder = thread::Builder::new().stack_size(1524);
+    builder.spawn(move || {
        info!("MQTT Listening for messages");
        let mut inflight = Vec::new();
        let mut inflight_topic = "".to_string();
@@ -113,7 +114,7 @@ pub fn make_client(
                                    }
                                }
                            };
-
+                            drop(msg);
                            if let Some((topic, data)) = incoming_message {
                                if topic.ends_with(topics::VLS) {
                                    tx.send(CoreEvent::VlsMessage(data))
@@ -141,7 +142,7 @@ pub fn make_client(
            }
        }
        //info!("MQTT connection loop exit");
-    });
+    })?;

    Ok(client)
 }