multi-signer broker refactor

This commit is contained in:
Evan Feenstra
2023-03-21 13:21:39 -07:00
parent 700e8f4ef3
commit 167d2d3c09
7 changed files with 297 additions and 123 deletions

4
broker/Cargo.lock generated
View File

@@ -2666,7 +2666,7 @@ dependencies = [
[[package]]
name = "rumqttd"
version = "0.12.6"
source = "git+https://github.com/Evanfeenstra/rumqtt?branch=sphinx-2#8811257886364ce0d1f09773c3aa5596e457d8c0"
source = "git+https://github.com/Evanfeenstra/rumqtt?branch=sphinx-2#96af7b82e5d7e97f52547c5c8a868fd55e4661fe"
dependencies = [
"axum",
"bytes",
@@ -3062,6 +3062,8 @@ dependencies = [
"rumqttc",
"rumqttd",
"secp256k1",
"serde",
"serde_json",
"sphinx-signer",
"thiserror",
"toml",

View File

@@ -33,6 +33,8 @@ toml = "0.5.9"
rocket = {version = "0.5.0-rc.2", features = ["json"]}
thiserror = "1.0.31"
hex = "0.4.3"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
[features]
default = ["std"]

View File

@@ -10,15 +10,16 @@ mod util;
use crate::chain_tracker::MqttSignerPort;
use crate::mqtt::{check_auth, start_broker};
use crate::unix_fd::SignerLoop;
use crate::util::read_broker_config;
use crate::util::{read_broker_config, Settings};
use clap::{arg, App};
use rocket::tokio::{
self,
sync::{broadcast, mpsc, oneshot},
};
use rumqttd::AuthMsg;
use rumqttd::{oneshot as std_oneshot, AuthMsg};
use serde::{Deserialize, Serialize};
use std::env;
use std::sync::Arc;
use std::sync::{Arc, Mutex};
use url::Url;
use vls_frontend::{frontend::SourceFactory, Frontend};
use vls_proxy::client::UnixClient;
@@ -26,6 +27,7 @@ use vls_proxy::connection::{open_parent_fd, UnixConnection};
use vls_proxy::portfront::SignerPortFront;
use vls_proxy::util::{add_hsmd_args, handle_hsmd_version};
#[derive(Debug, Serialize, Deserialize)]
pub struct Connections {
pub pubkey: Option<String>,
pub clients: Vec<String>,
@@ -41,6 +43,25 @@ impl Connections {
pub fn set_pubkey(&mut self, pk: &str) {
self.pubkey = Some(pk.to_string())
}
pub fn add_client(&mut self, cid: &str) {
let cids = cid.to_string();
if !self.clients.contains(&cids) {
self.clients.push(cids)
}
}
pub fn remove_client(&mut self, cid: &str) {
let cids = cid.to_string();
if self.clients.contains(&cids) {
self.clients.retain(|x| x != cid)
}
}
pub fn client_action(&mut self, cid: &str, connected: bool) {
if connected {
self.add_client(cid);
} else {
self.remove_client(cid);
}
}
}
pub struct Channel {
@@ -55,6 +76,7 @@ pub struct ChannelRequest {
pub topic: String,
pub message: Vec<u8>,
pub reply_tx: oneshot::Sender<ChannelReply>,
pub cid: Option<String>, // if it exists, only try the one client
}
impl ChannelRequest {
pub fn new(topic: &str, message: Vec<u8>) -> (Self, oneshot::Receiver<ChannelReply>) {
@@ -63,9 +85,22 @@ impl ChannelRequest {
topic: topic.to_string(),
message,
reply_tx,
cid: None,
};
(cr, reply_rx)
}
pub fn for_cid(&mut self, cid: &str) {
self.cid = Some(cid.to_string())
}
pub fn new_for(
cid: &str,
topic: &str,
message: Vec<u8>,
) -> (Self, oneshot::Receiver<ChannelReply>) {
let (mut cr, reply_rx) = ChannelRequest::new(topic, message);
cr.for_cid(cid);
(cr, reply_rx)
}
}
// mpsc reply
@@ -100,9 +135,9 @@ async fn rocket() -> _ {
panic!("end")
} else {
if matches.is_present("test") {
run_test::run_test().await
run_test::run_test()
} else {
run_main(parent_fd).await
run_main(parent_fd)
}
}
}
@@ -114,32 +149,71 @@ fn make_clap_app() -> App<'static> {
add_hsmd_args(app)
}
async fn run_main(parent_fd: i32) -> rocket::Rocket<rocket::Build> {
let settings = read_broker_config(BROKER_CONFIG_PATH);
let (mqtt_tx, mqtt_rx) = mpsc::channel(10000);
// blocks until a connection received
pub fn main_setup(
settings: Settings,
mqtt_rx: mpsc::Receiver<ChannelRequest>,
error_tx: broadcast::Sender<Vec<u8>>,
) -> Arc<Mutex<Connections>> {
let (auth_tx, auth_rx) = std::sync::mpsc::channel::<AuthMsg>();
// let (unix_tx, mut unix_rx) = mpsc::channel(10000);
let (status_tx, mut status_rx) = mpsc::channel(10000);
let (error_tx, error_rx) = broadcast::channel(10000);
error_log::log_errors(error_rx);
let (status_tx, status_rx) = std::sync::mpsc::channel();
let mut conns = Connections::new();
let conns1 = Connections::new();
let conns = Arc::new(Mutex::new(conns1));
// authenticator
let conns_ = conns.clone();
std::thread::spawn(move || {
while let Ok(am) = auth_rx.recv() {
let ok = check_auth(&am.username, &am.password, &mut conns);
let mut cs = conns_.lock().unwrap();
let ok = check_auth(&am.username, &am.password, &mut cs);
let _ = am.reply.send(ok);
}
});
// broker
log::info!("=> start broker on network: {}", settings.network);
start_broker(mqtt_rx, status_tx, error_tx.clone(), settings, auth_tx)
start_broker(
settings,
mqtt_rx,
status_tx,
error_tx.clone(),
auth_tx,
conns.clone(),
)
.expect("BROKER FAILED TO START");
// client connections state
let (startup_tx, startup_rx) = std_oneshot::channel();
let conns_ = conns.clone();
std::thread::spawn(move || {
log::info!("=> wait for connected status");
// wait for connection = true
let status = status_rx.recv().await.expect("couldnt receive");
log::info!("=> connected: {}: {}", status.0, status.1);
let (cid, connected) = status_rx.recv().expect("couldnt receive");
let mut cs = conns_.lock().unwrap();
cs.client_action(&cid, connected);
drop(cs);
log::info!("=> connected: {}: {}", cid, connected);
let _ = startup_tx.send(true);
while let Ok((cid, connected)) = status_rx.recv() {
let mut cs = conns_.lock().unwrap();
cs.client_action(&cid, connected);
drop(cs)
}
});
let _ = startup_rx.recv();
conns
}
fn run_main(parent_fd: i32) -> rocket::Rocket<rocket::Build> {
let settings = read_broker_config(BROKER_CONFIG_PATH);
let (mqtt_tx, mqtt_rx) = mpsc::channel(10000);
let (error_tx, error_rx) = broadcast::channel(10000);
error_log::log_errors(error_rx);
let conns = main_setup(settings, mqtt_rx, error_tx.clone());
// let mqtt_tx_ = mqtt_tx.clone();
// tokio::spawn(async move {
@@ -163,15 +237,16 @@ async fn run_main(parent_fd: i32) -> rocket::Rocket<rocket::Build> {
tokio::spawn(async move {
frontend.start();
});
} else {
log::warn!("Running without a frontend")
}
let conn = UnixConnection::new(parent_fd);
let client = UnixClient::new(conn);
// TODO pass status_rx into SignerLoop
let mut signer_loop = SignerLoop::new(client, mqtt_tx.clone());
// spawn CLN listener on a std thread
let cln_client = UnixClient::new(UnixConnection::new(parent_fd));
// TODO pass status_rx into SignerLoop?
let mut signer_loop = SignerLoop::new(cln_client, mqtt_tx.clone());
// spawn CLN listener
std::thread::spawn(move || {
signer_loop.start(Some(settings));
});
routes::launch_rocket(mqtt_tx, error_tx, settings)
routes::launch_rocket(mqtt_tx, error_tx, settings, conns)
}

View File

@@ -1,20 +1,23 @@
use crate::util::Settings;
use crate::Connections;
use crate::{ChannelReply, ChannelRequest};
use rocket::tokio::{sync::broadcast, sync::mpsc};
use rumqttd::{Alert, AlertEvent, AuthMsg, Broker, Config, Notification};
use rumqttd::{local::LinkTx, Alert, AlertEvent, AuthMsg, Broker, Config, Notification};
use sphinx_signer::sphinx_glyph::sphinx_auther::token::Token;
use sphinx_signer::sphinx_glyph::topics;
use std::sync::{Arc, Mutex};
use std::time::Duration;
// must get a reply within this time, or disconnects
// const REPLY_TIMEOUT_MS: u64 = 10000;
pub fn start_broker(
mut receiver: mpsc::Receiver<ChannelRequest>,
status_sender: mpsc::Sender<(String, bool)>,
error_sender: broadcast::Sender<Vec<u8>>,
settings: Settings,
mut receiver: mpsc::Receiver<ChannelRequest>,
status_sender: std::sync::mpsc::Sender<(String, bool)>,
error_sender: broadcast::Sender<Vec<u8>>,
auth_sender: std::sync::mpsc::Sender<AuthMsg>,
connections: Arc<Mutex<Connections>>,
) -> anyhow::Result<()> {
let conf = config(settings);
// let client_id = expected_client_id.to_string();
@@ -35,7 +38,7 @@ pub fn start_broker(
});
// connected/disconnected status alerts
let status_sender_ = status_sender.clone();
let (internal_status_tx, internal_status_rx) = std::sync::mpsc::channel();
let _alerts_handle = std::thread::spawn(move || loop {
let alert = alerts.poll();
log::info!("Alert: {:?}", alert);
@@ -49,7 +52,7 @@ pub fn start_broker(
AlertEvent::Disconnect => Some(false),
_ => None,
} {
let _ = status_sender_.blocking_send((cid, status));
let _ = internal_status_tx.send((cid, status));
}
}
}
@@ -57,21 +60,96 @@ pub fn start_broker(
}
});
let (msg_tx, mut msg_rx): (mpsc::Sender<Vec<u8>>, mpsc::Receiver<Vec<u8>>) =
mpsc::channel(10000);
link_tx.subscribe(topics::VLS_RETURN).unwrap();
link_tx.subscribe(topics::CONTROL_RETURN).unwrap();
link_tx.subscribe(topics::ERROR).unwrap();
// track connections
let status_sender_ = status_sender.clone();
let link_tx_ = link_tx.clone();
let _conns_task = std::thread::spawn(move || {
while let Ok((cid, is)) = internal_status_rx.recv() {
if is {
subs(&cid, link_tx_.clone());
} else {
unsubs(&cid, link_tx_.clone());
}
let _ = status_sender_.send((cid, is));
}
});
// String is the client id
let (msg_tx, msg_rx) = std::sync::mpsc::channel::<(String, Vec<u8>)>();
// receive from CLN, Frontend, or Controller
let conns_ = connections.clone();
let _relay_task = std::thread::spawn(move || {
while let Some(msg) = receiver.blocking_recv() {
if let Some(cid) = msg.cid {
// for a specific client
let pub_topic = format!("{}/{}", cid, msg.topic);
if let Err(e) = link_tx.publish(pub_topic, msg.message.clone()) {
log::error!("failed to pub to link_tx! {} {:?}", cid, e);
}
let rep = msg_rx.recv();
if let Ok((cid, reply)) = rep {
if let Err(_) = msg.reply_tx.send(ChannelReply { reply }) {
log::warn!("could not send on reply_tx {}", cid);
}
}
} else {
// send to each client in turn
'retry_loop: loop {
// get the current list of connected clients
let cs = conns_.lock().unwrap();
let client_list = cs.clients.clone();
drop(cs);
// wait a second if there are no clients
if client_list.len() == 0 {
std::thread::sleep(Duration::from_secs(1));
}
for client in client_list.iter() {
let pub_topic = format!("{}/{}", client, msg.topic);
if let Err(e) = link_tx.publish(pub_topic, msg.message.clone()) {
log::error!("failed to pub to link_tx! {:?}", e);
}
// and receive from the correct client (or timeout to next)
let dur = Duration::from_secs(9);
let rep = msg_rx.recv_timeout(dur);
if let Ok((cid, reply)) = rep {
if &cid == client {
if let Err(_) = msg.reply_tx.send(ChannelReply { reply }) {
log::warn!("could not send on reply_tx");
}
break 'retry_loop;
} else {
log::warn!("Mismatched client id!");
// wait a second before trying again
std::thread::sleep(Duration::from_secs(1));
}
}
}
}
}
}
});
// receive replies back from glyph
let _sub_task = std::thread::spawn(move || {
while let Ok(message) = link_rx.recv() {
if let Some(n) = message {
match n {
Notification::Forward(f) => {
if f.publish.topic == topics::ERROR {
let _ = error_sender.send(f.publish.topic.to_vec());
let topic_res = std::str::from_utf8(&f.publish.topic);
if let Err(_) = topic_res {
continue;
}
let topic = topic_res.unwrap();
if topic.ends_with(topics::ERROR) {
let _ = error_sender.send(f.publish.payload.to_vec());
} else {
if let Err(e) = msg_tx.blocking_send(f.publish.payload.to_vec()) {
let ts: Vec<&str> = topic.split("/").collect();
if ts.len() != 2 {
continue;
}
let cid = ts[0].to_string();
if let Err(e) = msg_tx.send((cid, f.publish.payload.to_vec())) {
log::error!("failed to pub to msg_tx! {:?}", e);
}
}
@@ -82,22 +160,8 @@ pub fn start_broker(
}
});
let _relay_task = std::thread::spawn(move || {
while let Some(msg) = receiver.blocking_recv() {
if let Err(e) = link_tx.publish(msg.topic, msg.message) {
log::error!("failed to pub to link_tx! {:?}", e);
}
let rep = msg_rx.blocking_recv();
if let Some(reply) = rep {
if let Err(_) = msg.reply_tx.send(ChannelReply { reply }) {
log::warn!("could not send on reply_tx");
}
}
}
});
// _sub_task.await.unwrap();
// _relay_task.await.unwrap();
// _sub_task.await.unwrap();
// _alerts_handle.await?;
std::thread::sleep(Duration::from_secs(1));
@@ -105,15 +169,34 @@ pub fn start_broker(
Ok(())
}
fn subs(cid: &str, mut ltx: LinkTx) {
ltx.subscribe(format!("{}/{}", cid, topics::VLS_RETURN))
.unwrap();
ltx.subscribe(format!("{}/{}", cid, topics::CONTROL_RETURN))
.unwrap();
ltx.subscribe(format!("{}/{}", cid, topics::ERROR)).unwrap();
}
fn unsubs(cid: &str, mut ltx: LinkTx) {
// ltx.unsubscribe(format!("{}/{}", cid, topics::VLS_RETURN))
// .unwrap();
// ltx.unsubscribe(format!("{}/{}", cid, topics::CONTROL_RETURN))
// .unwrap();
// ltx.unsubscribe(format!("{}/{}", cid, topics::ERROR))
// .unwrap();
}
pub fn check_auth(username: &str, password: &str, conns: &mut crate::Connections) -> bool {
match Token::from_base64(password) {
Ok(t) => match t.recover() {
Ok(pubkey) => {
// pubkey must match signature
if &pubkey.to_string() == username {
if let Some(pk) = &conns.pubkey {
// if there is an existing then it must match it
// if there is an existing pubkey then new client must match
pk == username
} else {
// set the Connections pubkey
conns.set_pubkey(username);
true
}

View File

@@ -1,5 +1,6 @@
use crate::util::Settings;
use crate::ChannelRequest;
use crate::Connections;
use rocket::fairing::{Fairing, Info, Kind};
use rocket::http::Header;
use rocket::response::stream::{Event, EventStream};
@@ -12,17 +13,29 @@ use rocket::*;
use sphinx_signer::sphinx_glyph::{error::Error as GlyphError, topics};
use std::net::IpAddr::V4;
use std::net::Ipv4Addr;
use std::sync::{Arc, Mutex};
pub type Result<T> = std::result::Result<T, Error>;
#[post("/control?<msg>")]
pub async fn control(sender: &State<Sender<ChannelRequest>>, msg: &str) -> Result<String> {
#[get("/clients")]
pub async fn get_clients(conns: &State<Arc<Mutex<Connections>>>) -> Result<String> {
use std::ops::Deref;
let cs = conns.lock().unwrap();
Ok(serde_json::to_string(&cs.deref())?)
}
#[post("/control?<msg>&<cid>")]
pub async fn control(
sender: &State<Sender<ChannelRequest>>,
msg: &str,
cid: &str,
) -> Result<String> {
let message = hex::decode(msg)?;
// FIXME validate?
if message.len() < 65 {
return Err(Error::Fail);
}
let (request, reply_rx) = ChannelRequest::new(topics::CONTROL, message);
let (request, reply_rx) = ChannelRequest::new_for(cid, topics::CONTROL, message);
// send to ESP
let _ = sender.send(request).await.map_err(|_| Error::Fail)?;
// wait for reply
@@ -54,6 +67,7 @@ pub fn launch_rocket(
tx: Sender<ChannelRequest>,
error_tx: broadcast::Sender<Vec<u8>>,
settings: Settings,
conns: Arc<Mutex<Connections>>,
) -> Rocket<Build> {
let config = Config {
address: V4(Ipv4Addr::UNSPECIFIED),
@@ -62,10 +76,11 @@ pub fn launch_rocket(
};
rocket::build()
.configure(config)
.mount("/api/", routes![control, errors])
.mount("/api/", routes![control, errors, get_clients])
.attach(CORS)
.manage(tx)
.manage(error_tx)
.manage(conns)
}
#[derive(Debug, thiserror::Error)]
@@ -76,6 +91,8 @@ pub enum Error {
Io(#[from] std::io::Error),
#[error("hex error: {0}")]
Hex(#[from] hex::FromHexError),
#[error("serde error: {0}")]
Serde(#[from] serde_json::Error),
}
use rocket::http::Status;

View File

@@ -1,53 +1,34 @@
use crate::mqtt::{check_auth, start_broker};
use crate::routes::launch_rocket;
use crate::util::Settings;
use crate::ChannelRequest;
use crate::Connections;
use rocket::tokio::{self, sync::broadcast, sync::mpsc};
use rumqttd::AuthMsg;
use sphinx_signer::{parser, sphinx_glyph::topics};
use vls_protocol::serde_bolt::WireString;
use vls_protocol::{msgs, msgs::Message};
// const CLIENT_ID: &str = "test-1";
pub async fn run_test() -> rocket::Rocket<rocket::Build> {
pub fn run_test() -> rocket::Rocket<rocket::Build> {
log::info!("TEST...");
// let mut id = 0u16;
// let mut sequence = 1;
let settings = Settings::default();
let (tx, rx) = mpsc::channel(1000);
let (auth_tx, auth_rx) = std::sync::mpsc::channel::<AuthMsg>();
let (status_tx, mut status_rx) = mpsc::channel(1000);
let (error_tx, error_rx) = broadcast::channel(1000);
let (mqtt_tx, mqtt_rx) = mpsc::channel(10000);
let (error_tx, error_rx) = broadcast::channel(10000);
crate::error_log::log_errors(error_rx);
let mut conns = Connections::new();
// block until connection
let conns = crate::main_setup(settings, mqtt_rx, error_tx.clone());
log::info!("=> off to the races!");
std::thread::spawn(move || {
while let Ok(am) = auth_rx.recv() {
let ok = check_auth(&am.username, &am.password, &mut conns);
let _ = am.reply.send(ok);
}
});
start_broker(rx, status_tx, error_tx.clone(), settings, auth_tx)
.expect("FAILED TO START BROKER");
log::info!("BROKER started!");
// let mut connected = false;
// let tx_ = tx.clone();
let tx_ = mqtt_tx.clone();
tokio::spawn(async move {
while let Some(status) = status_rx.recv().await {
log::info!("========> CONNECTED! {} {}", status.0, status.1);
}
});
// tokio::spawn(async move {
// loop {
// tokio::select! {
let mut id = 0;
let mut sequence = 0;
loop {
// select! (
// status = status_rx.recv() => {
// if let Some(connection_status) = status {
// connected = connection_status;
@@ -66,12 +47,21 @@ pub async fn run_test() -> rocket::Rocket<rocket::Build> {
// sequence = sequence.wrapping_add(1);
// id += 1;
// }
// tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
// std::thread::sleep(std::time::Duration::from_secs(1)).await;
// }
// };
// }
// });
launch_rocket(tx, error_tx, settings)
// )
let res = iteration(id, sequence, tx_.clone()).await;
if let Err(e) = res {
log::warn!("===> iteration failed {:?}", e);
} else {
sequence = sequence.wrapping_add(1);
id += 1;
}
tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
}
});
launch_rocket(mqtt_tx, error_tx, settings, conns)
}
#[allow(dead_code)]
@@ -80,12 +70,12 @@ pub async fn iteration(
id: u16,
sequence: u16,
tx: mpsc::Sender<ChannelRequest>,
connected: bool,
// connected: bool,
) -> anyhow::Result<()> {
return Ok(());
if !connected {
return Ok(());
}
// return Ok(());
// if !connected {
// return Ok(());
// }
log::info!("do a ping!");
let ping = msgs::Ping {
id,

View File

@@ -141,11 +141,16 @@ impl<C: 'static + Client> SignerLoop<C> {
fn set_channel_pubkey(&mut self, raw_msg: Vec<u8>) -> Result<()> {
let msg = msgs::from_vec(raw_msg.clone())?;
match msg {
Message::HsmdInitReplyV2(r) => self.chan.pubkey = r.node_id.0,
Message::HsmdInit2Reply(r) => self.chan.pubkey = r.node_id.0,
_ => (),
let pk = match msg {
Message::HsmdInitReplyV2(r) => Some(r.node_id.0),
Message::HsmdInit2Reply(r) => Some(r.node_id.0),
_ => None,
};
if let Some(pubkey) = pk {
let pks = hex::encode(pubkey);
log::info!("PUBKEY received from CLN: {}", pks);
self.chan.pubkey = pubkey;
}
Ok(())
}