From 75a3e6d2c7f3366dcadb92083a2b59a6d905756e Mon Sep 17 00:00:00 2001 From: asmo Date: Tue, 9 Sep 2025 14:26:03 +0200 Subject: [PATCH] Prometheus crate (#883) * feat: introduce `cdk-prometheus` crate with Prometheus server and CDK-specific metrics support --- .github/workflows/ci.yml | 28 +- Cargo.toml | 3 + Dockerfile | 2 +- crates/cdk-axum/Cargo.toml | 3 +- crates/cdk-axum/src/lib.rs | 7 + crates/cdk-axum/src/metrics.rs | 41 + crates/cdk-common/Cargo.toml | 2 + crates/cdk-common/src/payment.rs | 188 ++ crates/cdk-integration-tests/Cargo.toml | 2 +- .../src/bin/start_regtest_mints.rs | 1 + crates/cdk-integration-tests/src/shared.rs | 3 + crates/cdk-mintd/Cargo.toml | 11 +- crates/cdk-mintd/example.config.toml | 19 +- crates/cdk-mintd/src/config.rs | 10 + crates/cdk-mintd/src/env_vars/mod.rs | 9 + crates/cdk-mintd/src/env_vars/prometheus.rs | 31 + crates/cdk-mintd/src/lib.rs | 97 +- crates/cdk-prometheus/Cargo.toml | 47 + crates/cdk-prometheus/README.md | 189 ++ crates/cdk-prometheus/src/error.rs | 32 + crates/cdk-prometheus/src/lib.rs | 84 + crates/cdk-prometheus/src/metrics.rs | 427 ++++ crates/cdk-prometheus/src/process.rs | 107 + crates/cdk-prometheus/src/server.rs | 317 +++ crates/cdk-sql-common/Cargo.toml | 3 +- crates/cdk-sql-common/src/mint/mod.rs | 235 +- crates/cdk-sql-common/src/pool.rs | 38 +- crates/cdk-sqlite/Cargo.toml | 3 +- crates/cdk/Cargo.toml | 4 +- crates/cdk/src/mint/builder.rs | 1 - crates/cdk/src/mint/issue/mod.rs | 436 ++-- crates/cdk/src/mint/melt.rs | 187 +- crates/cdk/src/mint/mod.rs | 175 +- crates/cdk/src/mint/swap.rs | 63 +- docker-compose.yaml | 53 +- misc/provisioning/dashboards/dashboard.json | 1983 +++++++++++++++++ misc/provisioning/dashboards/dashboard.yaml | 8 + misc/provisioning/datasources/datasource.yml | 8 + misc/provisioning/prometheus.yml | 8 + 39 files changed, 4504 insertions(+), 361 deletions(-) create mode 100644 crates/cdk-axum/src/metrics.rs create mode 100644 crates/cdk-mintd/src/env_vars/prometheus.rs create mode 100644 crates/cdk-prometheus/Cargo.toml create mode 100644 crates/cdk-prometheus/README.md create mode 100644 crates/cdk-prometheus/src/error.rs create mode 100644 crates/cdk-prometheus/src/lib.rs create mode 100644 crates/cdk-prometheus/src/metrics.rs create mode 100644 crates/cdk-prometheus/src/process.rs create mode 100644 crates/cdk-prometheus/src/server.rs create mode 100644 misc/provisioning/dashboards/dashboard.json create mode 100644 misc/provisioning/dashboards/dashboard.yaml create mode 100644 misc/provisioning/datasources/datasource.yml create mode 100644 misc/provisioning/prometheus.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ddba0daf..19d1ae2c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: ' - name: typos run: nix develop -i -L .#nightly --command typos - + examples: name: "Run examples" runs-on: ubuntu-latest @@ -167,7 +167,7 @@ jobs: [ -p cdk-integration-tests, ] - database: + database: [ SQLITE, POSTGRES @@ -195,7 +195,7 @@ jobs: shared-key: "stable" - name: Test run: nix develop -i -L .#stable --command just itest ${{ matrix.database }} - + fake-mint-itest: name: "Integration fake mint tests" runs-on: ubuntu-latest @@ -207,7 +207,7 @@ jobs: [ -p cdk-integration-tests, ] - database: + database: [ SQLITE, ] @@ -236,7 +236,7 @@ jobs: run: nix develop -i -L .#stable --command cargo clippy -- -D warnings - name: Test fake auth mint run: nix develop -i -L .#stable --command just fake-mint-itest ${{ matrix.database }} - + pure-itest: name: "Integration fake wallet tests" runs-on: ubuntu-latest @@ -244,7 +244,7 @@ jobs: needs: [pre-commit-checks, clippy] strategy: matrix: - database: + database: [ memory, sqlite, @@ -256,7 +256,7 @@ jobs: - name: Free Disk Space (Ubuntu) uses: jlumbroso/free-disk-space@main with: - tool-cache: true + tool-cache: true android: true dotnet: true haskell: true @@ -286,7 +286,7 @@ jobs: needs: [pre-commit-checks, clippy, pure-itest, fake-mint-itest, regtest-itest] strategy: matrix: - ln: + ln: [ FAKEWALLET, CLN, @@ -298,7 +298,7 @@ jobs: - name: Free Disk Space (Ubuntu) uses: jlumbroso/free-disk-space@main with: - tool-cache: true + tool-cache: true android: true dotnet: true haskell: true @@ -357,7 +357,7 @@ jobs: - name: Build run: nix develop -i -L .#msrv --command cargo build ${{ matrix.build-args }} - + check-wasm: name: Check WASM runs-on: ubuntu-latest @@ -389,7 +389,7 @@ jobs: - name: Build cdk and binding run: nix develop -i -L ".#${{ matrix.rust }}" --command cargo build ${{ matrix.build-args }} --target ${{ matrix.target }} - + check-wasm-msrv: name: Check WASM runs-on: ubuntu-latest @@ -428,7 +428,7 @@ jobs: needs: [pre-commit-checks, clippy, pure-itest, fake-mint-itest] strategy: matrix: - database: + database: [ SQLITE, ] @@ -456,7 +456,7 @@ jobs: - name: Stop and clean up Docker Compose run: | docker compose -f misc/keycloak/docker-compose-recover.yml down - + doc-tests: name: "Documentation Tests" runs-on: ubuntu-latest @@ -475,7 +475,7 @@ jobs: shared-key: "stable" - name: Run doc tests run: nix develop -i -L .#stable --command cargo test --doc - + strict-docs: name: "Strict Documentation Check" runs-on: ubuntu-latest diff --git a/Cargo.toml b/Cargo.toml index e2142eb0..452bac84 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,6 +60,7 @@ cdk-sqlite = { path = "./crates/cdk-sqlite", default-features = true, version = cdk-postgres = { path = "./crates/cdk-postgres", default-features = true, version = "=0.12.0" } cdk-signatory = { path = "./crates/cdk-signatory", version = "=0.12.0", default-features = false } cdk-mintd = { path = "./crates/cdk-mintd", version = "=0.12.0", default-features = false } +cdk-prometheus = { path = "./crates/cdk-prometheus", version = "=0.12.0", default-features = false } clap = { version = "4.5.31", features = ["derive"] } ciborium = { version = "0.2.2", default-features = false, features = ["std"] } cbor-diag = "0.1.12" @@ -108,6 +109,8 @@ tonic-build = "0.13.1" strum = "0.27.1" strum_macros = "0.27.1" rustls = { version = "0.23.27", default-features = false, features = ["ring"] } +prometheus = { version = "0.13.4", features = ["process"], default-features = false } + diff --git a/Dockerfile b/Dockerfile index 9d37caf1..b4359169 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ COPY Cargo.toml ./Cargo.toml COPY crates ./crates # Start the Nix daemon and develop the environment -RUN nix develop --extra-experimental-features nix-command --extra-experimental-features flakes --command cargo build --release --bin cdk-mintd --features redis +RUN nix develop --extra-experimental-features nix-command --extra-experimental-features flakes --command cargo build --release --bin cdk-mintd --features redis --features prometheus # Create a runtime stage FROM debian:trixie-slim diff --git a/crates/cdk-axum/Cargo.toml b/crates/cdk-axum/Cargo.toml index 3bebf0bd..7ef7e145 100644 --- a/crates/cdk-axum/Cargo.toml +++ b/crates/cdk-axum/Cargo.toml @@ -15,7 +15,7 @@ default = ["auth"] redis = ["dep:redis"] swagger = ["cdk/swagger", "dep:utoipa"] auth = ["cdk/auth"] - +prometheus = ["dep:cdk-prometheus"] [dependencies] anyhow.workspace = true async-trait.workspace = true @@ -27,6 +27,7 @@ tokio.workspace = true tracing.workspace = true utoipa = { workspace = true, optional = true } futures.workspace = true +cdk-prometheus = { workspace = true , optional = true} moka = { version = "0.12.10", features = ["future"] } serde_json.workspace = true paste = "1.0.15" diff --git a/crates/cdk-axum/src/lib.rs b/crates/cdk-axum/src/lib.rs index 182f4f78..3d72986d 100644 --- a/crates/cdk-axum/src/lib.rs +++ b/crates/cdk-axum/src/lib.rs @@ -17,6 +17,8 @@ use cache::HttpCache; use cdk::mint::Mint; use router_handlers::*; +mod metrics; + #[cfg(feature = "auth")] mod auth; mod bolt12_router; @@ -322,6 +324,11 @@ pub async fn create_mint_router_with_custom_cache( mint_router }; + #[cfg(feature = "prometheus")] + let mint_router = mint_router.layer(axum::middleware::from_fn_with_state( + state.clone(), + metrics::global_metrics_middleware, + )); let mint_router = mint_router .layer(from_fn(cors_middleware)) .with_state(state); diff --git a/crates/cdk-axum/src/metrics.rs b/crates/cdk-axum/src/metrics.rs new file mode 100644 index 00000000..81156d3b --- /dev/null +++ b/crates/cdk-axum/src/metrics.rs @@ -0,0 +1,41 @@ +#[cfg(feature = "prometheus")] +use std::time::Instant; + +#[cfg(feature = "prometheus")] +use axum::body::Body; +#[cfg(feature = "prometheus")] +use axum::extract::MatchedPath; +#[cfg(feature = "prometheus")] +use axum::http::Request; +#[cfg(feature = "prometheus")] +use axum::middleware::Next; +#[cfg(feature = "prometheus")] +use axum::response::Response; +#[cfg(feature = "prometheus")] +use cdk_prometheus::global; + +/// Global metrics middleware that uses the singleton instance. +/// This version doesn't require access to MintState and can be used in any Axum application. +#[cfg(feature = "prometheus")] +pub async fn global_metrics_middleware( + matched_path: Option, + req: Request, + next: Next, +) -> Response { + let start_time = Instant::now(); + + let response = next.run(req).await; + + let endpoint_path = matched_path + .map(|mp| mp.as_str().to_string()) + .unwrap_or_default(); + + let status_code = response.status().as_u16().to_string(); + let request_duration = start_time.elapsed().as_secs_f64(); + + // Always use global metrics + global::record_http_request(&endpoint_path, &status_code); + global::record_http_request_duration(request_duration, &endpoint_path); + + response +} diff --git a/crates/cdk-common/Cargo.toml b/crates/cdk-common/Cargo.toml index 5afe1060..d19e851a 100644 --- a/crates/cdk-common/Cargo.toml +++ b/crates/cdk-common/Cargo.toml @@ -18,6 +18,7 @@ bench = [] wallet = ["cashu/wallet"] mint = ["cashu/mint", "dep:uuid"] auth = ["cashu/auth"] +prometheus = ["cdk-prometheus/default"] [dependencies] async-trait.workspace = true @@ -30,6 +31,7 @@ lightning-invoice.workspace = true lightning.workspace = true thiserror.workspace = true tracing.workspace = true +cdk-prometheus = { workspace = true, optional = true} url.workspace = true uuid = { workspace = true, optional = true } utoipa = { workspace = true, optional = true } diff --git a/crates/cdk-common/src/payment.rs b/crates/cdk-common/src/payment.rs index 1bcabfcd..15f18427 100644 --- a/crates/cdk-common/src/payment.rs +++ b/crates/cdk-common/src/payment.rs @@ -6,6 +6,8 @@ use std::pin::Pin; use async_trait::async_trait; use cashu::util::hex; use cashu::{Bolt11Invoice, MeltOptions}; +#[cfg(feature = "prometheus")] +use cdk_prometheus::METRICS; use futures::Stream; use lightning::offers::offer::Offer; use lightning_invoice::ParseOrSemanticError; @@ -411,3 +413,189 @@ impl TryFrom for Bolt11Settings { serde_json::from_value(value).map_err(|err| err.into()) } } + +/// Metrics wrapper for MintPayment implementations +/// +/// This wrapper implements the Decorator pattern to collect metrics on all +/// MintPayment trait methods. It wraps any existing MintPayment implementation +/// and automatically records timing and operation metrics. +#[derive(Clone)] +#[cfg(feature = "prometheus")] +pub struct MetricsMintPayment { + inner: T, +} +#[cfg(feature = "prometheus")] +impl MetricsMintPayment +where + T: MintPayment, +{ + /// Create a new metrics wrapper around a MintPayment implementation + pub fn new(inner: T) -> Self { + Self { inner } + } + + /// Get reference to the underlying implementation + pub fn inner(&self) -> &T { + &self.inner + } + + /// Consume the wrapper and return the inner implementation + pub fn into_inner(self) -> T { + self.inner + } +} + +#[async_trait] +#[cfg(feature = "prometheus")] +impl MintPayment for MetricsMintPayment +where + T: MintPayment + Send + Sync, +{ + type Err = T::Err; + + async fn get_settings(&self) -> Result { + let start = std::time::Instant::now(); + METRICS.inc_in_flight_requests("get_settings"); + + let result = self.inner.get_settings().await; + + let duration = start.elapsed().as_secs_f64(); + METRICS.record_mint_operation_histogram("get_settings", result.is_ok(), duration); + METRICS.dec_in_flight_requests("get_settings"); + + result + } + + async fn create_incoming_payment_request( + &self, + unit: &CurrencyUnit, + options: IncomingPaymentOptions, + ) -> Result { + let start = std::time::Instant::now(); + METRICS.inc_in_flight_requests("create_incoming_payment_request"); + + let result = self + .inner + .create_incoming_payment_request(unit, options) + .await; + + let duration = start.elapsed().as_secs_f64(); + METRICS.record_mint_operation_histogram( + "create_incoming_payment_request", + result.is_ok(), + duration, + ); + METRICS.dec_in_flight_requests("create_incoming_payment_request"); + + result + } + + async fn get_payment_quote( + &self, + unit: &CurrencyUnit, + options: OutgoingPaymentOptions, + ) -> Result { + let start = std::time::Instant::now(); + METRICS.inc_in_flight_requests("get_payment_quote"); + + let result = self.inner.get_payment_quote(unit, options).await; + + let duration = start.elapsed().as_secs_f64(); + let success = result.is_ok(); + + if let Ok(ref quote) = result { + let amount: f64 = u64::from(quote.amount) as f64; + let fee: f64 = u64::from(quote.fee) as f64; + METRICS.record_lightning_payment(amount, fee); + } + + METRICS.record_mint_operation_histogram("get_payment_quote", success, duration); + METRICS.dec_in_flight_requests("get_payment_quote"); + + result + } + async fn wait_payment_event( + &self, + ) -> Result + Send>>, Self::Err> { + let start = std::time::Instant::now(); + METRICS.inc_in_flight_requests("wait_payment_event"); + + let result = self.inner.wait_payment_event().await; + + let duration = start.elapsed().as_secs_f64(); + let success = result.is_ok(); + + METRICS.record_mint_operation_histogram("wait_payment_event", success, duration); + METRICS.dec_in_flight_requests("wait_payment_event"); + + result + } + + async fn make_payment( + &self, + unit: &CurrencyUnit, + options: OutgoingPaymentOptions, + ) -> Result { + let start = std::time::Instant::now(); + METRICS.inc_in_flight_requests("make_payment"); + + let result = self.inner.make_payment(unit, options).await; + + let duration = start.elapsed().as_secs_f64(); + let success = result.is_ok(); + + METRICS.record_mint_operation_histogram("make_payment", success, duration); + METRICS.dec_in_flight_requests("make_payment"); + + result + } + + fn is_wait_invoice_active(&self) -> bool { + self.inner.is_wait_invoice_active() + } + + fn cancel_wait_invoice(&self) { + self.inner.cancel_wait_invoice() + } + + async fn check_incoming_payment_status( + &self, + payment_identifier: &PaymentIdentifier, + ) -> Result, Self::Err> { + let start = std::time::Instant::now(); + METRICS.inc_in_flight_requests("check_incoming_payment_status"); + + let result = self + .inner + .check_incoming_payment_status(payment_identifier) + .await; + + let duration = start.elapsed().as_secs_f64(); + METRICS.record_mint_operation_histogram( + "check_incoming_payment_status", + result.is_ok(), + duration, + ); + METRICS.dec_in_flight_requests("check_incoming_payment_status"); + + result + } + + async fn check_outgoing_payment( + &self, + payment_identifier: &PaymentIdentifier, + ) -> Result { + let start = std::time::Instant::now(); + METRICS.inc_in_flight_requests("check_outgoing_payment"); + + let result = self.inner.check_outgoing_payment(payment_identifier).await; + + let duration = start.elapsed().as_secs_f64(); + let success = result.is_ok(); + + METRICS.record_mint_operation_histogram("check_outgoing_payment", success, duration); + METRICS.dec_in_flight_requests("check_outgoing_payment"); + + result + } +} diff --git a/crates/cdk-integration-tests/Cargo.toml b/crates/cdk-integration-tests/Cargo.toml index 7a257603..8e5dce7f 100644 --- a/crates/cdk-integration-tests/Cargo.toml +++ b/crates/cdk-integration-tests/Cargo.toml @@ -29,7 +29,7 @@ cdk-sqlite = { workspace = true } cdk-redb = { workspace = true } cdk-fake-wallet = { workspace = true } cdk-common = { workspace = true, features = ["mint", "wallet", "auth"] } -cdk-mintd = { workspace = true, features = ["cln", "lnd", "fakewallet", "grpc-processor", "auth", "lnbits", "management-rpc", "sqlite", "postgres", "ldk-node"] } +cdk-mintd = { workspace = true, features = ["cln", "lnd", "fakewallet", "grpc-processor", "auth", "lnbits", "management-rpc", "sqlite", "postgres", "ldk-node", "prometheus"] } futures = { workspace = true, default-features = false, features = [ "executor", ] } diff --git a/crates/cdk-integration-tests/src/bin/start_regtest_mints.rs b/crates/cdk-integration-tests/src/bin/start_regtest_mints.rs index 4c361fb5..3ebcf261 100644 --- a/crates/cdk-integration-tests/src/bin/start_regtest_mints.rs +++ b/crates/cdk-integration-tests/src/bin/start_regtest_mints.rs @@ -286,6 +286,7 @@ fn create_ldk_settings( grpc_processor: None, database: cdk_mintd::config::Database::default(), mint_management_rpc: None, + prometheus: None, auth: None, } } diff --git a/crates/cdk-integration-tests/src/shared.rs b/crates/cdk-integration-tests/src/shared.rs index d854441e..c052ca9c 100644 --- a/crates/cdk-integration-tests/src/shared.rs +++ b/crates/cdk-integration-tests/src/shared.rs @@ -219,6 +219,7 @@ pub fn create_fake_wallet_settings( }, mint_management_rpc: None, auth: None, + prometheus: Some(Default::default()), } } @@ -265,6 +266,7 @@ pub fn create_cln_settings( database: cdk_mintd::config::Database::default(), mint_management_rpc: None, auth: None, + prometheus: Some(Default::default()), } } @@ -310,5 +312,6 @@ pub fn create_lnd_settings( database: cdk_mintd::config::Database::default(), mint_management_rpc: None, auth: None, + prometheus: Some(Default::default()), } } diff --git a/crates/cdk-mintd/Cargo.toml b/crates/cdk-mintd/Cargo.toml index 240e791c..ae83b751 100644 --- a/crates/cdk-mintd/Cargo.toml +++ b/crates/cdk-mintd/Cargo.toml @@ -28,6 +28,7 @@ sqlcipher = ["sqlite", "cdk-sqlite/sqlcipher"] swagger = ["cdk-axum/swagger", "dep:utoipa", "dep:utoipa-swagger-ui"] redis = ["cdk-axum/redis"] auth = ["cdk/auth", "cdk-axum/auth", "cdk-sqlite?/auth", "cdk-postgres?/auth"] +prometheus = ["cdk/prometheus", "dep:cdk-prometheus", "cdk-sqlite?/prometheus", "cdk-axum/prometheus"] [dependencies] anyhow.workspace = true @@ -38,8 +39,9 @@ cdk = { workspace = true, features = [ ] } cdk-sqlite = { workspace = true, features = [ "mint" -], optional = true } -cdk-postgres = { workspace = true, features = ["mint"], optional = true } +], optional = true } +cdk-common = {workspace = true, features = ["prometheus"]} +cdk-postgres = { workspace = true, features = ["mint"], optional = true} cdk-cln = { workspace = true, optional = true } cdk-lnbits = { workspace = true, optional = true } cdk-lnd = { workspace = true, optional = true } @@ -50,6 +52,7 @@ cdk-signatory.workspace = true cdk-mint-rpc = { workspace = true, optional = true } cdk-payment-processor = { workspace = true, optional = true } config.workspace = true +cdk-prometheus = { workspace = true, optional = true , features = ["system-metrics"]} clap.workspace = true bitcoin.workspace = true tokio = { workspace = true, default-features = false, features = ["signal"] } @@ -63,11 +66,7 @@ tower-http = { workspace = true, features = ["compression-full", "decompression- tower.workspace = true lightning-invoice.workspace = true home.workspace = true -url.workspace = true utoipa = { workspace = true, optional = true } utoipa-swagger-ui = { version = "9.0.0", features = ["axum"], optional = true } [build-dependencies] -# Dep of utopia 2.5.0 breaks so keeping here for now -zip = "=2.4.2" -time = "=0.3.39" diff --git a/crates/cdk-mintd/example.config.toml b/crates/cdk-mintd/example.config.toml index 2b103175..6aa7510e 100644 --- a/crates/cdk-mintd/example.config.toml +++ b/crates/cdk-mintd/example.config.toml @@ -1,3 +1,4 @@ + [info] url = "https://mint.thesimplekid.dev/" listen_host = "127.0.0.1" @@ -20,7 +21,11 @@ enabled = false # address = "127.0.0.1" # port = 8086 - +#[prometheus] +#enabled = true +#address = "127.0.0.1" +#port = 9090 +# [info.http_cache] # memory or redis backend = "memory" @@ -130,12 +135,12 @@ reserve_fee_min = 4 # webserver_host = "127.0.0.1" # Default: 127.0.0.1 # webserver_port = 0 # 0 = auto-assign available port -# [fake_wallet] -# supported_units = ["sat"] -# fee_percent = 0.02 -# reserve_fee_min = 1 -# min_delay_time = 1 -# max_delay_time = 3 +[fake_wallet] +supported_units = ["sat"] +fee_percent = 0.02 +reserve_fee_min = 1 +min_delay_time = 1 +max_delay_time = 3 # [grpc_processor] # gRPC Payment Processor configuration diff --git a/crates/cdk-mintd/src/config.rs b/crates/cdk-mintd/src/config.rs index e4877d7c..a04ab7dc 100644 --- a/crates/cdk-mintd/src/config.rs +++ b/crates/cdk-mintd/src/config.rs @@ -452,6 +452,16 @@ pub struct Settings { #[cfg(feature = "management-rpc")] pub mint_management_rpc: Option, pub auth: Option, + #[cfg(feature = "prometheus")] + pub prometheus: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[cfg(feature = "prometheus")] +pub struct Prometheus { + pub enabled: bool, + pub address: Option, + pub port: Option, } #[derive(Debug, Clone, Serialize, Deserialize, Default)] diff --git a/crates/cdk-mintd/src/env_vars/mod.rs b/crates/cdk-mintd/src/env_vars/mod.rs index 8a4edcde..61501aab 100644 --- a/crates/cdk-mintd/src/env_vars/mod.rs +++ b/crates/cdk-mintd/src/env_vars/mod.rs @@ -25,6 +25,8 @@ mod lnbits; mod lnd; #[cfg(feature = "management-rpc")] mod management_rpc; +#[cfg(feature = "prometheus")] +mod prometheus; use std::env; use std::str::FromStr; @@ -50,6 +52,8 @@ pub use lnd::*; #[cfg(feature = "management-rpc")] pub use management_rpc::*; pub use mint_info::*; +#[cfg(feature = "prometheus")] +pub use prometheus::*; use crate::config::{DatabaseEngine, LnBackend, Settings}; @@ -98,6 +102,11 @@ impl Settings { ); } + #[cfg(feature = "prometheus")] + { + self.prometheus = Some(self.prometheus.clone().unwrap_or_default().from_env()); + } + match self.ln.ln_backend { #[cfg(feature = "cln")] LnBackend::Cln => { diff --git a/crates/cdk-mintd/src/env_vars/prometheus.rs b/crates/cdk-mintd/src/env_vars/prometheus.rs new file mode 100644 index 00000000..446bc64e --- /dev/null +++ b/crates/cdk-mintd/src/env_vars/prometheus.rs @@ -0,0 +1,31 @@ +//! Prometheus environment variables + +use std::env; + +use crate::config::Prometheus; + +pub const ENV_PROMETHEUS_ENABLED: &str = "CDK_MINTD_PROMETHEUS_ENABLED"; +pub const ENV_PROMETHEUS_ADDRESS: &str = "CDK_MINTD_PROMETHEUS_ADDRESS"; +pub const ENV_PROMETHEUS_PORT: &str = "CDK_MINTD_PROMETHEUS_PORT"; + +impl Prometheus { + pub fn from_env(mut self) -> Self { + if let Ok(enabled_str) = env::var(ENV_PROMETHEUS_ENABLED) { + if let Ok(enabled) = enabled_str.parse() { + self.enabled = enabled; + } + } + + if let Ok(address) = env::var(ENV_PROMETHEUS_ADDRESS) { + self.address = Some(address); + } + + if let Ok(port_str) = env::var(ENV_PROMETHEUS_PORT) { + if let Ok(port) = port_str.parse() { + self.port = Some(port); + } + } + + self + } +} diff --git a/crates/cdk-mintd/src/lib.rs b/crates/cdk-mintd/src/lib.rs index ff13f63b..9071c73c 100644 --- a/crates/cdk-mintd/src/lib.rs +++ b/crates/cdk-mintd/src/lib.rs @@ -13,10 +13,7 @@ use std::sync::Arc; use anyhow::{anyhow, bail, Result}; use axum::Router; use bip39::Mnemonic; -// internal crate modules use cdk::cdk_database::{self, MintDatabase, MintKVStore, MintKeysDatabase}; -use cdk::cdk_payment; -use cdk::cdk_payment::MintPayment; use cdk::mint::{Mint, MintBuilder, MintMeltLimits}; #[cfg(any( feature = "cln", @@ -41,14 +38,22 @@ use cdk::nuts::{AuthRequired, Method, ProtectedEndpoint, RoutePath}; use cdk::nuts::{ContactInfo, MintVersion, PaymentMethod}; use cdk::types::QuoteTTL; use cdk_axum::cache::HttpCache; +// internal crate modules +#[cfg(feature = "prometheus")] +use cdk_common::payment::MetricsMintPayment; +use cdk_common::payment::MintPayment; +#[cfg(feature = "auth")] +use cdk_postgres::MintPgAuthDatabase; #[cfg(feature = "postgres")] -use cdk_postgres::{MintPgAuthDatabase, MintPgDatabase}; +use cdk_postgres::MintPgDatabase; #[cfg(all(feature = "auth", feature = "sqlite"))] use cdk_sqlite::mint::MintSqliteAuthDatabase; #[cfg(feature = "sqlite")] use cdk_sqlite::MintSqliteDatabase; use cli::CLIArgs; -use config::{AuthType, DatabaseEngine, LnBackend}; +#[cfg(feature = "auth")] +use config::AuthType; +use config::{DatabaseEngine, LnBackend}; use env_vars::ENV_WORK_DIR; use setup::LnBackendSetup; use tower::ServiceBuilder; @@ -440,6 +445,8 @@ async fn configure_lightning_backend( None, ) .await?; + #[cfg(feature = "prometheus")] + let cln = MetricsMintPayment::new(cln); mint_builder = configure_backend_for_unit( settings, @@ -463,6 +470,8 @@ async fn configure_lightning_backend( None, ) .await?; + #[cfg(feature = "prometheus")] + let lnbits = MetricsMintPayment::new(lnbits); mint_builder = configure_backend_for_unit( settings, @@ -486,6 +495,8 @@ async fn configure_lightning_backend( None, ) .await?; + #[cfg(feature = "prometheus")] + let lnd = MetricsMintPayment::new(lnd); mint_builder = configure_backend_for_unit( settings, @@ -512,6 +523,8 @@ async fn configure_lightning_backend( _kv_store.clone(), ) .await?; + #[cfg(feature = "prometheus")] + let fake = MetricsMintPayment::new(fake); mint_builder = configure_backend_for_unit( settings, @@ -541,6 +554,8 @@ async fn configure_lightning_backend( let processor = grpc_processor .setup(ln_routers, settings, unit.clone(), None, work_dir, None) .await?; + #[cfg(feature = "prometheus")] + let processor = MetricsMintPayment::new(processor); mint_builder = configure_backend_for_unit( settings, @@ -595,7 +610,7 @@ async fn configure_backend_for_unit( mut mint_builder: MintBuilder, unit: cdk::nuts::CurrencyUnit, mint_melt_limits: MintMeltLimits, - backend: Arc + Send + Sync>, + backend: Arc + Send + Sync>, ) -> Result { let payment_settings = backend.get_settings().await?; @@ -974,7 +989,48 @@ async fn start_services_with_shutdown( ); } } + // Create a broadcast channel to share shutdown signal between services + let (shutdown_tx, _) = tokio::sync::broadcast::channel::<()>(1); + // Start Prometheus server if enabled + #[cfg(feature = "prometheus")] + let prometheus_handle = { + if let Some(prometheus_settings) = &settings.prometheus { + if prometheus_settings.enabled { + let addr = prometheus_settings + .address + .clone() + .unwrap_or("127.0.0.1".to_string()); + let port = prometheus_settings.port.unwrap_or(9000); + + let address = format!("{}:{}", addr, port) + .parse() + .expect("Invalid prometheus address"); + + let server = cdk_prometheus::PrometheusBuilder::new() + .bind_address(address) + .build_with_cdk_metrics()?; + + let mut shutdown_rx = shutdown_tx.subscribe(); + let prometheus_shutdown = async move { + let _ = shutdown_rx.recv().await; + }; + + Some(tokio::spawn(async move { + if let Err(e) = server.start(prometheus_shutdown).await { + tracing::error!("Failed to start prometheus server: {}", e); + } + })) + } else { + None + } + } else { + None + } + }; + + #[cfg(not(feature = "prometheus"))] + let prometheus_handle: Option> = None; for router in ln_routers { mint_service = mint_service.merge(router); } @@ -987,8 +1043,24 @@ async fn start_services_with_shutdown( tracing::info!("listening on {}", listener.local_addr().unwrap()); + // Create a task to wait for the shutdown signal and broadcast it + let shutdown_broadcast_task = { + let shutdown_tx = shutdown_tx.clone(); + tokio::spawn(async move { + shutdown_signal.await; + tracing::info!("Shutdown signal received, broadcasting to all services"); + let _ = shutdown_tx.send(()); + }) + }; + + // Create shutdown future for axum server + let mut axum_shutdown_rx = shutdown_tx.subscribe(); + let axum_shutdown = async move { + let _ = axum_shutdown_rx.recv().await; + }; + // Wait for axum server to complete with custom shutdown signal - let axum_result = axum::serve(listener, mint_service).with_graceful_shutdown(shutdown_signal); + let axum_result = axum::serve(listener, mint_service).with_graceful_shutdown(axum_shutdown); match axum_result.await { Ok(_) => { @@ -1001,6 +1073,17 @@ async fn start_services_with_shutdown( } } + // Wait for the shutdown broadcast task to complete + let _ = shutdown_broadcast_task.await; + + // Wait for prometheus server to shutdown if it was started + #[cfg(feature = "prometheus")] + if let Some(handle) = prometheus_handle { + if let Err(e) = handle.await { + tracing::warn!("Prometheus server task failed: {}", e); + } + } + mint.stop().await?; #[cfg(feature = "management-rpc")] diff --git a/crates/cdk-prometheus/Cargo.toml b/crates/cdk-prometheus/Cargo.toml new file mode 100644 index 00000000..13c8a7de --- /dev/null +++ b/crates/cdk-prometheus/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "cdk-prometheus" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +readme = "README.md" +description = "Prometheus metrics export server for CDK applications" + +[features] +default = ["system-metrics"] +system-metrics = ["sysinfo"] + +[dependencies] +# Prometheus +prometheus = "0.13" + +# Async runtime +tokio.workspace = true +futures.workspace = true + +# Error handling +anyhow.workspace = true +thiserror.workspace = true + +# Serialization +serde.workspace = true +serde_json.workspace = true + +# System metrics (optional) +sysinfo = { version = "0.32", optional = true } + +# Tracing +tracing.workspace = true + +# Utility +once_cell.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["full"] } +reqwest.workspace = true +tracing-subscriber.workspace = true + +[lints] +workspace = true diff --git a/crates/cdk-prometheus/README.md b/crates/cdk-prometheus/README.md new file mode 100644 index 00000000..00ecdaf8 --- /dev/null +++ b/crates/cdk-prometheus/README.md @@ -0,0 +1,189 @@ +# CDK Prometheus + +A small, focused crate that provides Prometheus metrics for CDK-based services. It bundles a ready-to-use metrics registry, a background HTTP server to expose metrics, helper functions for common CDK domains (HTTP, auth, Lightning, DB, mint operations), and an ergonomic macro for conditional metrics recording. + +- Out-of-the-box metrics for HTTP, auth, Lightning payments, database, and mint operations +- Global, lazily-initialized metrics instance you can use anywhere +- Optional background server to expose metrics on /metrics +- Re-exports the prometheus crate for custom instrumentation +- Optional system metrics (feature-gated) + +## Installation + +Add the crate to your Cargo.toml (replace the version as needed): + +```toml +[dependencies] cdk-prometheus = { version = "0.1", features = ["system-metrics"] } +``` + +- Feature flags: + - system-metrics: include basic process/system metrics collected periodically. + +Note for downstream crates: the provided record_metrics! macro is gated at call-site by a feature named prometheus. If you use that macro, declare a prometheus feature in your application crate and enable it to compile the macro calls into real metrics (otherwise they no-op). + +## Quick start +### Docker +Start Prometheus and Grafana with docker-compose: +``` +docker compose up -d prometheus grafana +``` +Start your mintd +``` +./mintd -w ~/.cdk-mintd +``` +Check Prometheus and Grafana +* `curl localhost:9000/metrics` for checking CDK metrics +* `http://localhost:9090/targets?search=` checking the prometheus collector (you should see http://host.docker.internal:9000/metrics) +* `http://localhost:3011/d/cdk-mint-dashboard/cdk-mint-dashboard` Grafana dashboard (default login: admin/admin) + +### Rust +Expose a Prometheus endpoint with a default registry and CDK metrics: +```rust +use cdk_prometheus::start_default_server_with_metrics; +#[tokio::main] async fn main() -> anyhow::Result<()> { // Starts an HTTP server (default bind and path) and registers CDK metrics into its registry start_default_server_with_metrics().await?; Ok(()) } +``` + +Or start it in the background (e.g., from your application bootstrap): +```rust +use cdk_prometheus::start_background_server_with_metrics; +fn main() -> anyhow::Result<()> { let _handle = start_background_server_with_metrics()?; // Continue bootstrapping your application... Ok(()) } +``` + +## Recording metrics + +You can record metrics using: +- The global helpers (simple functions) +- The global singleton METRICS (direct methods) +- The record_metrics! macro (conditional recording with an optional instance) + +### Global helpers +```rust +use cdk_prometheus::global; +fn handle_request() { + global::record_http_request("/health", "200"); global::record_http_request_duration(0.003, "/health"); + global::record_auth_attempt(); + global::record_auth_success(); + + // Lightning and DB + global::record_lightning_payment(1500.0, 2.0); // amount, fee (both in base units you track) + global::record_db_operation(0.015, "select_user"); + global::set_db_connections_active(8); + + // Mint operations + global::inc_in_flight_requests("get_payment_quote"); + // ... do work ... + global::record_mint_operation("get_payment_quote", true); + global::record_mint_operation_histogram("get_payment_quote", true, 0.021); + global::dec_in_flight_requests("get_payment_quote"); + + // Errors + global::record_error(); +} +``` + +### Using the global METRICS instance directly +```rust +use cdk_prometheus::METRICS; +fn do_db_work() { METRICS.record_db_operation(0.005, "update_user"); } +``` + +### Using the record_metrics! macro + +The macro lets you write grouped calls concisely and optionally pass an instance to use; if no instance is present, it automatically falls back to the global helpers. At call-site, wrap your invocations with a prometheus feature so they can be disabled in minimal builds. +```rust +use cdk_prometheus::record_metrics; +fn run_operation(metrics_opt: Option) { // Use instance if present, otherwise fallback to global record_metrics!(metrics_opt => { inc_in_flight_requests("make_payment"); record_mint_operation("make_payment", true); record_mint_operation_histogram("make_payment", true, 0.123); dec_in_flight_requests("make_payment"); }); + // Or call directly on the global helpers + record_metrics!({ + record_error(); + }); +} +``` + +## Exposing the /metrics endpoint + +If you just need sane defaults, use the convenience starters shown above. If you want finer control (bind address, path, system metrics), build the server explicitly: +```rust +use cdk_prometheus::{PrometheusBuilder, PrometheusServer, CdkMetrics, prometheus::Registry}; +fn build_and_run() -> anyhow::Result>> { // Build a server wired up with the default CDK metrics let server = PrometheusBuilder::new().build_with_cdk_metrics()?; let handle = server.start_background(); Ok(handle) } +``` + +Notes: +- Default bind address and metrics path are set by the server configuration (commonly 127.0.0.1:9090 and /metrics). +- With system-metrics enabled, the server periodically updates process/system gauges. + +## What’s included + +The default CDK metrics instance (CdkMetrics) registers and maintains counters, histograms, and gauges for common areas: +- HTTP: request totals, durations +- Auth: attempts and successes +- Lightning: payment totals, amounts, fees +- Database: operation totals, latencies, active connections +- Mint: operation totals, in-flight gauges, per-operation latencies +- Errors: a general counter + +You can use these immediately through the global helpers or the METRICS instance. + +## Adding custom metrics + +This crate re-exports the prometheus crate and exposes the underlying Registry so you can define and register your own metrics: +```rust +use cdk_prometheus::{prometheus, global}; +fn register_custom_metric() -> Result<(), prometheus::Error> { let my_counter = prometheus::IntCounter::new("my_counter", "A custom counter")?; let registry = global::registry(); // Arcregistry.register(Box::new(my_counter.clone()))?; + my_counter.inc(); + Ok(()) +} +``` + +If you prefer instance-level control: +```rust +use std::sync::Arc; use cdk_prometheus::{create_cdk_metrics, prometheus}; +fn with_instance() -> anyhow::Result<()> { let metrics = create_cdk_metrics()?; let registry: Arc[prometheus::Registry]() = metrics.registry(); + let hist = prometheus::Histogram::with_opts( + prometheus::HistogramOpts::new("my_latency_seconds", "My op latency") +)?; +registry.register(Box::new(hist))?; +Ok(()) +} +``` + +## Scraping with Prometheus + +Example scrape_config: +```yaml +scrape_configs: +- job_name: 'cdk' + scrape_interval: 15s + static_configs: + - targets: ['127.0.0.1:9090'] +``` + +If you changed the bind address or path, make sure to update targets or the metrics_path in your Prometheus configuration accordingly. + +## System metrics (optional) + +Enable the system-metrics feature to export basic process/system metrics. The server updates these at a configurable interval. +```toml +cdk-prometheus = { version = "0.1", features = ["system-metrics"] } +``` + +## Error handling + +Common error types surfaced by this crate include: +- Server bind failures +- Metrics collection/registry errors +- System metrics collection errors (when enabled) + +Handle these at startup and monitor logs during runtime. + +## Best practices + +- Run the metrics server on localhost or a private interface and use a Prometheus agent/sidecar if needed. +- Register application-specific metrics early in your bootstrap so they are visible from the first scrape. +- Use histograms for latencies and size distributions; use counters for event totals; use gauges for in-flight or current-state values. +- Keep label cardinality bounded. + +## License + +MIT +``` diff --git a/crates/cdk-prometheus/src/error.rs b/crates/cdk-prometheus/src/error.rs new file mode 100644 index 00000000..9d8746f9 --- /dev/null +++ b/crates/cdk-prometheus/src/error.rs @@ -0,0 +1,32 @@ +use thiserror::Error; + +/// Errors that can occur in the Prometheus crate +#[derive(Error, Debug)] +pub enum PrometheusError { + /// Server binding error + #[error("Failed to bind to address {address}: {source}")] + ServerBind { + address: String, + #[source] + source: std::io::Error, + }, + + /// Metrics collection error + #[error("Failed to collect metrics: {0}")] + MetricsCollection(String), + + /// Registry error + #[error("Registry error: {source}")] + Registry { + #[from] + source: prometheus::Error, + }, + + /// System metrics error + #[cfg(feature = "system-metrics")] + #[error("System metrics error: {0}")] + SystemMetrics(String), +} + +/// Result type for Prometheus operations +pub type Result = std::result::Result; diff --git a/crates/cdk-prometheus/src/lib.rs b/crates/cdk-prometheus/src/lib.rs new file mode 100644 index 00000000..913b2645 --- /dev/null +++ b/crates/cdk-prometheus/src/lib.rs @@ -0,0 +1,84 @@ +//! # CDK Prometheus + +pub mod error; +pub mod metrics; +pub mod server; + +#[cfg(feature = "system-metrics")] +pub mod process; + +// Re-exports for convenience +pub use error::{PrometheusError, Result}; +pub use metrics::{global, CdkMetrics, METRICS}; +#[cfg(feature = "system-metrics")] +pub use process::SystemMetrics; +// Re-export prometheus crate for custom metrics +pub use prometheus; +pub use server::{PrometheusBuilder, PrometheusConfig, PrometheusServer}; + +/// Macro for recording metrics with optional fallback to global instance +/// +/// Usage: +/// ```rust +/// use cdk_prometheus::record_metrics; +/// +/// // With optional metrics instance +/// record_metrics!(metrics_option => { +/// dec_in_flight_requests("operation"); +/// record_mint_operation("operation", true); +/// }); +/// +/// // Direct global calls +/// record_metrics!({ +/// dec_in_flight_requests("operation"); +/// record_mint_operation("operation", true); +/// }); +/// ``` +#[macro_export] +macro_rules! record_metrics { + // Pattern for using optional metrics with fallback to global + ($metrics_opt:expr => { $($method:ident($($arg:expr),*));* $(;)? }) => { + #[cfg(feature = "prometheus")] + { + if let Some(metrics) = $metrics_opt.as_ref() { + $( + metrics.$method($($arg),*); + )* + } else { + $( + $crate::global::$method($($arg),*); + )* + } + } + }; + + // Pattern for using global metrics directly + ({ $($method:ident($($arg:expr),*));* $(;)? }) => { + #[cfg(feature = "prometheus")] + { + $( + $crate::global::$method($($arg),*); + )* + } + }; +} + +/// Convenience function to create a new CDK metrics instance +/// +/// # Errors +/// Returns an error if any of the metrics cannot be created or registered +pub fn create_cdk_metrics() -> Result { + CdkMetrics::new() +} + +/// Convenience function to start a Prometheus server with specific metrics +/// +/// # Errors +/// Returns an error if the server cannot be created or started +pub async fn start_default_server_with_metrics( + shutdown_signal: impl std::future::Future + Send + 'static, +) -> Result<()> { + let server = PrometheusBuilder::new().build_with_cdk_metrics()?; + + server.start(shutdown_signal).await +} diff --git a/crates/cdk-prometheus/src/metrics.rs b/crates/cdk-prometheus/src/metrics.rs new file mode 100644 index 00000000..e761c5a3 --- /dev/null +++ b/crates/cdk-prometheus/src/metrics.rs @@ -0,0 +1,427 @@ +use std::sync::Arc; + +use prometheus::{ + Histogram, HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, Registry, +}; + +/// Global metrics instance +pub static METRICS: std::sync::LazyLock = std::sync::LazyLock::new(CdkMetrics::default); + +/// Custom metrics for CDK applications +#[derive(Clone, Debug)] +pub struct CdkMetrics { + registry: Arc, + + // HTTP metrics + http_requests_total: IntCounterVec, + http_request_duration: HistogramVec, + + // Authentication metrics + auth_attempts_total: IntCounter, + auth_successes_total: IntCounter, + + // Lightning metrics + lightning_payments_total: IntCounter, + lightning_payment_amount: Histogram, + lightning_payment_fees: Histogram, + + // Database metrics + db_operations_total: IntCounter, + db_operation_duration: HistogramVec, + db_connections_active: IntGauge, + + // Error metrics + errors_total: IntCounter, + + // Mint metrics + mint_operations_total: IntCounterVec, + mint_in_flight_requests: IntGaugeVec, + mint_operation_duration: HistogramVec, +} + +impl CdkMetrics { + /// Create a new instance with default metrics + /// + /// # Errors + /// Returns an error if any of the metrics cannot be created or registered + pub fn new() -> crate::Result { + let registry = Arc::new(Registry::new()); + + // Create and register HTTP metrics + let (http_requests_total, http_request_duration) = Self::create_http_metrics(®istry)?; + + // Create and register authentication metrics + let (auth_attempts_total, auth_successes_total) = Self::create_auth_metrics(®istry)?; + + // Create and register Lightning metrics + let (lightning_payments_total, lightning_payment_amount, lightning_payment_fees) = + Self::create_lightning_metrics(®istry)?; + + // Create and register database metrics + let (db_operations_total, db_operation_duration, db_connections_active) = + Self::create_db_metrics(®istry)?; + + // Create and register error metrics + let errors_total = Self::create_error_metrics(®istry)?; + + // Create and register mint metrics + let (mint_operations_total, mint_operation_duration, mint_in_flight_requests) = + Self::create_mint_metrics(®istry)?; + + Ok(Self { + registry, + http_requests_total, + http_request_duration, + auth_attempts_total, + auth_successes_total, + lightning_payments_total, + lightning_payment_amount, + lightning_payment_fees, + db_operations_total, + db_operation_duration, + db_connections_active, + errors_total, + mint_operations_total, + mint_in_flight_requests, + mint_operation_duration, + }) + } + + /// Create and register HTTP metrics + /// + /// # Errors + /// Returns an error if any of the metrics cannot be created or registered + fn create_http_metrics(registry: &Registry) -> crate::Result<(IntCounterVec, HistogramVec)> { + let http_requests_total = IntCounterVec::new( + prometheus::Opts::new("cdk_http_requests_total", "Total number of HTTP requests"), + &["endpoint", "status"], + )?; + registry.register(Box::new(http_requests_total.clone()))?; + + let http_request_duration = HistogramVec::new( + prometheus::HistogramOpts::new( + "cdk_http_request_duration_seconds", + "HTTP request duration in seconds", + ) + .buckets(vec![ + 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, + ]), + &["endpoint"], + )?; + registry.register(Box::new(http_request_duration.clone()))?; + + Ok((http_requests_total, http_request_duration)) + } + + /// Create and register authentication metrics + /// + /// # Errors + /// Returns an error if any of the metrics cannot be created or registered + fn create_auth_metrics(registry: &Registry) -> crate::Result<(IntCounter, IntCounter)> { + let auth_attempts_total = + IntCounter::new("cdk_auth_attempts_total", "Total authentication attempts")?; + registry.register(Box::new(auth_attempts_total.clone()))?; + + let auth_successes_total = IntCounter::new( + "cdk_auth_successes_total", + "Total successful authentications", + )?; + registry.register(Box::new(auth_successes_total.clone()))?; + + Ok((auth_attempts_total, auth_successes_total)) + } + + /// Create and register Lightning metrics + /// + /// # Errors + /// Returns an error if any of the metrics cannot be created or registered + fn create_lightning_metrics( + registry: &Registry, + ) -> crate::Result<(IntCounter, Histogram, Histogram)> { + let wallet_operations_total = + IntCounter::new("cdk_wallet_operations_total", "Total wallet operations")?; + registry.register(Box::new(wallet_operations_total))?; + + let lightning_payments_total = + IntCounter::new("cdk_lightning_payments_total", "Total Lightning payments")?; + registry.register(Box::new(lightning_payments_total.clone()))?; + + let lightning_payment_amount = Histogram::with_opts( + prometheus::HistogramOpts::new( + "cdk_lightning_payment_amount_sats", + "Lightning payment amounts in satoshis", + ) + .buckets(vec![ + 1.0, + 10.0, + 100.0, + 1000.0, + 10_000.0, + 100_000.0, + 1_000_000.0, + ]), + )?; + registry.register(Box::new(lightning_payment_amount.clone()))?; + + let lightning_payment_fees = Histogram::with_opts( + prometheus::HistogramOpts::new( + "cdk_lightning_payment_fees_sats", + "Lightning payment fees in satoshis", + ) + .buckets(vec![0.0, 1.0, 5.0, 10.0, 50.0, 100.0, 500.0, 1000.0]), + )?; + registry.register(Box::new(lightning_payment_fees.clone()))?; + + Ok(( + lightning_payments_total, + lightning_payment_amount, + lightning_payment_fees, + )) + } + + /// Create and register database metrics + /// + /// # Errors + /// Returns an error if any of the metrics cannot be created or registered + fn create_db_metrics( + registry: &Registry, + ) -> crate::Result<(IntCounter, HistogramVec, IntGauge)> { + let db_operations_total = + IntCounter::new("cdk_db_operations_total", "Total database operations")?; + registry.register(Box::new(db_operations_total.clone()))?; + let db_operation_duration = HistogramVec::new( + prometheus::HistogramOpts::new( + "cdk_db_operation_duration_seconds", + "Database operation duration in seconds", + ) + .buckets(vec![0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0]), + &["operation"], + )?; + registry.register(Box::new(db_operation_duration.clone()))?; + + let db_connections_active = IntGauge::new( + "cdk_db_connections_active", + "Number of active database connections", + )?; + registry.register(Box::new(db_connections_active.clone()))?; + + Ok(( + db_operations_total, + db_operation_duration, + db_connections_active, + )) + } + + /// Create and register error metrics + /// + /// # Errors + /// Returns an error if any of the metrics cannot be created or registered + fn create_error_metrics(registry: &Registry) -> crate::Result { + let errors_total = IntCounter::new("cdk_errors_total", "Total errors")?; + registry.register(Box::new(errors_total.clone()))?; + + Ok(errors_total) + } + + /// Create and register mint metrics + /// + /// # Errors + /// Returns an error if any of the metrics cannot be created or registered + fn create_mint_metrics( + registry: &Registry, + ) -> crate::Result<(IntCounterVec, HistogramVec, IntGaugeVec)> { + let mint_operations_total = IntCounterVec::new( + prometheus::Opts::new( + "cdk_mint_operations_total", + "Total number of mint operations", + ), + &["operation", "status"], + )?; + registry.register(Box::new(mint_operations_total.clone()))?; + + let mint_operation_duration = HistogramVec::new( + prometheus::HistogramOpts::new( + "cdk_mint_operation_duration_seconds", + "Duration of mint operations in seconds", + ) + .buckets(vec![ + 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, + ]), + &["operation", "status"], + )?; + registry.register(Box::new(mint_operation_duration.clone()))?; + + let mint_in_flight_requests = IntGaugeVec::new( + prometheus::Opts::new( + "cdk_mint_in_flight_requests", + "Number of in-flight mint requests", + ), + &["operation"], + )?; + registry.register(Box::new(mint_in_flight_requests.clone()))?; + + Ok(( + mint_operations_total, + mint_operation_duration, + mint_in_flight_requests, + )) + } + + /// Get the metrics registry + #[must_use] + pub fn registry(&self) -> Arc { + Arc::::clone(&self.registry) + } + + // HTTP metrics methods + pub fn record_http_request(&self, endpoint: &str, status: &str) { + self.http_requests_total + .with_label_values(&[endpoint, status]) + .inc(); + } + + pub fn record_http_request_duration(&self, duration_seconds: f64, endpoint: &str) { + self.http_request_duration + .with_label_values(&[endpoint]) + .observe(duration_seconds); + } + + // Authentication metrics methods + pub fn record_auth_attempt(&self) { + self.auth_attempts_total.inc(); + } + + pub fn record_auth_success(&self) { + self.auth_successes_total.inc(); + } + + // Lightning metrics methods + pub fn record_lightning_payment(&self, amount: f64, fee: f64) { + self.lightning_payments_total.inc(); + self.lightning_payment_amount.observe(amount); + self.lightning_payment_fees.observe(fee); + } + + // Database metrics methods + pub fn record_db_operation(&self, duration_seconds: f64, op: &str) { + self.db_operations_total.inc(); + self.db_operation_duration + .with_label_values(&[op]) + .observe(duration_seconds); + } + + pub fn set_db_connections_active(&self, count: i64) { + self.db_connections_active.set(count); + } + + // Error metrics methods + pub fn record_error(&self) { + self.errors_total.inc(); + } + + // Mint metrics methods + pub fn record_mint_operation(&self, operation: &str, success: bool) { + let status = if success { "success" } else { "error" }; + self.mint_operations_total + .with_label_values(&[operation, status]) + .inc(); + } + pub fn record_mint_operation_histogram( + &self, + operation: &str, + success: bool, + duration_seconds: f64, + ) { + let status = if success { "success" } else { "error" }; + self.mint_operation_duration + .with_label_values(&[operation, status]) + .observe(duration_seconds); + } + pub fn inc_in_flight_requests(&self, operation: &str) { + self.mint_in_flight_requests + .with_label_values(&[operation]) + .inc(); + } + + pub fn dec_in_flight_requests(&self, operation: &str) { + self.mint_in_flight_requests + .with_label_values(&[operation]) + .dec(); + } +} + +impl Default for CdkMetrics { + fn default() -> Self { + Self::new().expect("Failed to create default CdkMetrics") + } +} + +/// Helper functions for recording metrics using the global instance +pub mod global { + use super::METRICS; + + /// Record an HTTP request using the global metrics instance + pub fn record_http_request(endpoint: &str, status: &str) { + METRICS.record_http_request(endpoint, status); + } + + /// Record HTTP request duration using the global metrics instance + pub fn record_http_request_duration(duration_seconds: f64, endpoint: &str) { + METRICS.record_http_request_duration(duration_seconds, endpoint); + } + + /// Record authentication attempt using the global metrics instance + pub fn record_auth_attempt() { + METRICS.record_auth_attempt(); + } + + /// Record authentication success using the global metrics instance + pub fn record_auth_success() { + METRICS.record_auth_success(); + } + + /// Record Lightning payment using the global metrics instance + pub fn record_lightning_payment(amount: f64, fee: f64) { + METRICS.record_lightning_payment(amount, fee); + } + + /// Record database operation using the global metrics instance + pub fn record_db_operation(duration_seconds: f64, op: &str) { + METRICS.record_db_operation(duration_seconds, op); + } + + /// Set database connections active using the global metrics instance + pub fn set_db_connections_active(count: i64) { + METRICS.set_db_connections_active(count); + } + + /// Record error using the global metrics instance + pub fn record_error() { + METRICS.record_error(); + } + + /// Record mint operation using the global metrics instance + pub fn record_mint_operation(operation: &str, success: bool) { + METRICS.record_mint_operation(operation, success); + } + + /// Record mint operation with histogram using the global metrics instance + pub fn record_mint_operation_histogram(operation: &str, success: bool, duration_seconds: f64) { + METRICS.record_mint_operation_histogram(operation, success, duration_seconds); + } + + /// Increment in-flight requests using the global metrics instance + pub fn inc_in_flight_requests(operation: &str) { + METRICS.inc_in_flight_requests(operation); + } + + /// Decrement in-flight requests using the global metrics instance + pub fn dec_in_flight_requests(operation: &str) { + METRICS.dec_in_flight_requests(operation); + } + + /// Get the metrics registry from the global instance + pub fn registry() -> std::sync::Arc { + METRICS.registry() + } +} diff --git a/crates/cdk-prometheus/src/process.rs b/crates/cdk-prometheus/src/process.rs new file mode 100644 index 00000000..7ccc11f0 --- /dev/null +++ b/crates/cdk-prometheus/src/process.rs @@ -0,0 +1,107 @@ +use std::sync::Arc; + +#[cfg(feature = "system-metrics")] +use prometheus::{Gauge, IntGauge, Registry}; +#[cfg(feature = "system-metrics")] +use sysinfo::{Pid, System}; + +/// System metrics collector that provides CPU, memory, disk, network, and process metrics +#[cfg(feature = "system-metrics")] +#[derive(Clone, Debug)] +pub struct SystemMetrics { + registry: Arc, + system: Arc>, + + // Process metrics (for the CDK process) + process_cpu_usage_percent: Gauge, + process_memory_bytes: IntGauge, + process_memory_percent: Gauge, +} + +#[cfg(feature = "system-metrics")] +impl SystemMetrics { + /// Create a new `SystemMetrics` instance + /// + /// # Errors + /// Returns an error if any of the metrics cannot be created or registered + pub fn new() -> crate::Result { + let registry = Arc::new(Registry::new()); + // Process metrics + let process_cpu_usage_percent = Gauge::new( + "process_cpu_usage_percent", + "CPU usage percentage of the CDK process (0-100)", + )?; + registry.register(Box::new(process_cpu_usage_percent.clone()))?; + + let process_memory_bytes = IntGauge::new( + "process_memory_bytes", + "Memory usage of the CDK process in bytes", + )?; + registry.register(Box::new(process_memory_bytes.clone()))?; + + let process_memory_percent = Gauge::new( + "process_memory_percent", + "Memory usage percentage of the CDK process (0-100)", + )?; + registry.register(Box::new(process_memory_percent.clone()))?; + + // Initialize system with all needed refresh kinds + let system = Arc::new(std::sync::Mutex::new(System::new())); + + let result = Self { + registry, + system, + process_cpu_usage_percent, + process_memory_bytes, + process_memory_percent, + }; + + Ok(result) + } + + /// Get the metrics registry + #[must_use] + pub fn registry(&self) -> Arc { + Arc::::clone(&self.registry) + } + + /// Update all system metrics + /// + /// # Errors + /// Returns an error if the system mutex cannot be locked + pub fn update_metrics(&self) -> crate::Result<()> { + let mut system = self.system.lock().map_err(|e| { + crate::error::PrometheusError::SystemMetrics(format!("Failed to lock system: {e}")) + })?; + // Refresh system information + system.refresh_all(); + + // Update memory metrics + let total_memory = i64::try_from(system.total_memory()).unwrap_or(i64::MAX); + + // Update process metrics for the current process + // This is a simplified approach that may not work perfectly in all cases + if let Some(process) = system.process(Pid::from(std::process::id() as usize)) { + // Get CPU usage if available + let process_cpu = process.cpu_usage(); + self.process_cpu_usage_percent.set(f64::from(process_cpu)); + + // Get memory usage if available + let process_memory = i64::try_from(process.memory()).unwrap_or(i64::MAX); + self.process_memory_bytes.set(process_memory); + + // Calculate memory percentage + if total_memory > 0 { + // Precision loss is acceptable for percentage calculation + #[allow(clippy::cast_precision_loss)] + let process_memory_percent = (process_memory as f64 / total_memory as f64) * 100.0; + self.process_memory_percent.set(process_memory_percent); + } + } + + // Drop the system lock early to avoid resource contention + drop(system); + + Ok(()) + } +} diff --git a/crates/cdk-prometheus/src/server.rs b/crates/cdk-prometheus/src/server.rs new file mode 100644 index 00000000..68e495ab --- /dev/null +++ b/crates/cdk-prometheus/src/server.rs @@ -0,0 +1,317 @@ +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; + +use prometheus::{Registry, TextEncoder}; + +use crate::metrics::METRICS; +#[cfg(feature = "system-metrics")] +use crate::process::SystemMetrics; + +/// Configuration for the Prometheus server +#[derive(Debug, Clone)] +pub struct PrometheusConfig { + /// Address to bind the server to (default: "127.0.0.1:9090") + pub bind_address: SocketAddr, + /// Path to serve metrics on (default: "/metrics") + pub metrics_path: String, + /// Whether to include system metrics (default: true if feature enabled) + #[cfg(feature = "system-metrics")] + pub include_system_metrics: bool, + /// How often to update system metrics in seconds (default: 15) + #[cfg(feature = "system-metrics")] + pub system_metrics_interval: u64, +} + +impl Default for PrometheusConfig { + fn default() -> Self { + Self { + bind_address: "127.0.0.1:9090".parse().expect("Invalid default address"), + metrics_path: "/metrics".to_string(), + #[cfg(feature = "system-metrics")] + include_system_metrics: true, + #[cfg(feature = "system-metrics")] + system_metrics_interval: 15, + } + } +} + +/// Prometheus metrics server +#[derive(Debug)] +pub struct PrometheusServer { + config: PrometheusConfig, + registry: Arc, + #[cfg(feature = "system-metrics")] + system_metrics: Option, +} + +impl PrometheusServer { + /// Create a new Prometheus server with CDK metrics + /// + /// # Errors + /// Returns an error if system metrics cannot be created (when enabled) + pub fn new(config: PrometheusConfig) -> crate::Result { + let registry = METRICS.registry(); + + #[cfg(feature = "system-metrics")] + let system_metrics = if config.include_system_metrics { + let sys_metrics = SystemMetrics::new()?; + Some(sys_metrics) + } else { + None + }; + + Ok(Self { + config, + registry, + #[cfg(feature = "system-metrics")] + system_metrics, + }) + } + + /// Create a new Prometheus server with custom registry + #[must_use] + pub const fn with_registry(config: PrometheusConfig, registry: Arc) -> Self { + Self { + config, + registry, + #[cfg(feature = "system-metrics")] + system_metrics: None, + } + } + + /// Create a metrics handler function that gathers and encodes metrics + fn create_metrics_handler( + registry: Arc, + #[cfg(feature = "system-metrics")] system_metrics: Option, + ) -> impl Fn() -> String { + move || { + let encoder = TextEncoder::new(); + + // Collect metrics from our registry + #[cfg(feature = "system-metrics")] + let mut metric_families = registry.gather(); + #[cfg(not(feature = "system-metrics"))] + let metric_families = registry.gather(); + + // Add system metrics if available + #[cfg(feature = "system-metrics")] + if let Some(ref sys_metrics) = system_metrics { + // Update system metrics before collection + if let Err(e) = sys_metrics.update_metrics() { + tracing::warn!("Failed to update system metrics: {e}"); + } + + let sys_registry = sys_metrics.registry(); + let mut sys_families = sys_registry.gather(); + metric_families.append(&mut sys_families); + } + + // Encode metrics to string + encoder + .encode_to_string(&metric_families) + .unwrap_or_else(|e| { + tracing::error!("Failed to encode metrics: {e}"); + format!("Failed to encode metrics: {e}") + }) + } + } + + /// Start the Prometheus HTTP server + /// + /// # Errors + /// This function always returns Ok as errors are handled internally + pub async fn start( + self, + shutdown_signal: impl std::future::Future + Send + 'static, + ) -> crate::Result<()> { + // Create and start the exporter + let binding = self.config.bind_address; + let registry_clone = Arc::::clone(&self.registry); + + // Create a handler that exposes our registry + #[cfg(feature = "system-metrics")] + let metrics_handler = + Self::create_metrics_handler(registry_clone, self.system_metrics.clone()); + + #[cfg(not(feature = "system-metrics"))] + let metrics_handler = Self::create_metrics_handler(registry_clone); + + // Start the exporter in a background task + let path = self.config.metrics_path.clone(); + + // Create a channel for signaling the server task to shutdown + let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel::<()>(); + + // Spawn the server task + let server_handle = tokio::spawn(async move { + // We're using a simple HTTP server to expose our metrics + use std::io::{Read, Write}; + use std::net::TcpListener; + + // Create a TCP listener + let listener = match TcpListener::bind(binding) { + Ok(listener) => { + // Set non-blocking mode to allow for shutdown checking + if let Err(e) = listener.set_nonblocking(true) { + tracing::error!("Failed to set non-blocking mode: {e}"); + return; + } + listener + } + Err(e) => { + tracing::error!("Failed to bind TCP listener: {e}"); + return; + } + }; + tracing::info!("Started Prometheus server on {} at path {}", binding, path); + + // Accept connections with shutdown signal handling + loop { + // Check for shutdown signal + if shutdown_rx.try_recv().is_ok() { + tracing::info!("Shutdown signal received, stopping Prometheus server"); + break; + } + + // Try to accept a connection (non-blocking) + match listener.accept() { + Ok((mut stream, _)) => { + // Handle the connection + let mut buffer = [0; 1024]; + match stream.read(&mut buffer) { + Ok(0) => {} + Ok(bytes_read) => { + // Convert the buffer to a string + let request = String::from_utf8_lossy(&buffer[..bytes_read]); + + // Check if the request is for our metrics path + if request.contains(&format!("GET {path} HTTP")) { + // Get the metrics + let metrics = metrics_handler(); + + // Write the response + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\n\r\n{}", + metrics.len(), + metrics + ); + + if let Err(e) = stream.write_all(response.as_bytes()) { + tracing::error!("Failed to write response: {e}"); + } + } else { + // Write a 404 response + let response = "HTTP/1.1 404 Not Found\r\nContent-Type: text/plain\r\nContent-Length: 9\r\n\r\nNot Found"; + if let Err(e) = stream.write_all(response.as_bytes()) { + tracing::error!("Failed to write response: {e}"); + } + } + } + Err(e) => { + tracing::error!("Failed to read from stream: {e}"); + } + } + } + Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock => { + // No connection available, continue the loop + tokio::time::sleep(Duration::from_millis(10)).await; + } + Err(e) => { + tracing::error!("Failed to accept connection: {e}"); + // Add a small delay to prevent busy looping on persistent errors + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + } + + tracing::info!("Prometheus server stopped"); + }); + + // Wait for the shutdown signal + shutdown_signal.await; + + // Signal the server to shutdown + let _ = shutdown_tx.send(()); + + // Wait for the server task to complete (with a timeout) + match tokio::time::timeout(Duration::from_secs(5), server_handle).await { + Ok(result) => { + if let Err(e) = result { + tracing::error!("Server task failed: {e}"); + } + } + Err(_) => { + tracing::warn!("Server shutdown timed out after 5 seconds"); + } + } + + Ok(()) + } +} + +/// Builder for easy Prometheus server setup +#[derive(Debug)] +pub struct PrometheusBuilder { + config: PrometheusConfig, +} + +impl PrometheusBuilder { + /// Create a new builder with default configuration + #[must_use] + pub fn new() -> Self { + Self { + config: PrometheusConfig::default(), + } + } + + /// Set the bind address + #[must_use] + pub const fn bind_address(mut self, addr: SocketAddr) -> Self { + self.config.bind_address = addr; + self + } + + /// Set the metrics path + #[must_use] + pub fn metrics_path>(mut self, path: S) -> Self { + self.config.metrics_path = path.into(); + self + } + + /// Enable or disable system metrics + #[cfg(feature = "system-metrics")] + #[must_use] + pub const fn system_metrics(mut self, enabled: bool) -> Self { + self.config.include_system_metrics = enabled; + self + } + + /// Set system metrics update interval + #[cfg(feature = "system-metrics")] + #[must_use] + pub const fn system_metrics_interval(mut self, seconds: u64) -> Self { + self.config.system_metrics_interval = seconds; + self + } + + /// Build the server with specific CDK metrics instance + /// + /// # Errors + /// Returns an error if system metrics cannot be created (when enabled) + pub fn build_with_cdk_metrics(self) -> crate::Result { + PrometheusServer::new(self.config) + } + + /// Build the server with custom registry + #[must_use] + pub fn build_with_registry(self, registry: Arc) -> PrometheusServer { + PrometheusServer::with_registry(self.config, registry) + } +} + +impl Default for PrometheusBuilder { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/cdk-sql-common/Cargo.toml b/crates/cdk-sql-common/Cargo.toml index 7785ff4d..c5b512cc 100644 --- a/crates/cdk-sql-common/Cargo.toml +++ b/crates/cdk-sql-common/Cargo.toml @@ -16,10 +16,11 @@ default = ["mint", "wallet", "auth"] mint = ["cdk-common/mint"] wallet = ["cdk-common/wallet"] auth = ["cdk-common/auth"] - +prometheus = ["cdk-prometheus"] [dependencies] async-trait.workspace = true cdk-common = { workspace = true, features = ["test"] } +cdk-prometheus = { workspace = true, optional = true } bitcoin.workspace = true thiserror.workspace = true tracing.workspace = true diff --git a/crates/cdk-sql-common/src/mint/mod.rs b/crates/cdk-sql-common/src/mint/mod.rs index ca750e6f..935e825d 100644 --- a/crates/cdk-sql-common/src/mint/mod.rs +++ b/crates/cdk-sql-common/src/mint/mod.rs @@ -57,6 +57,8 @@ mod migrations; #[cfg(feature = "auth")] pub use auth::SQLMintAuthDatabase; +#[cfg(feature = "prometheus")] +use cdk_prometheus::METRICS; /// Mint SQL Database #[derive(Debug, Clone)] @@ -299,11 +301,27 @@ where type Err = Error; async fn commit(self: Box) -> Result<(), Error> { - self.inner.commit().await + let result = self.inner.commit().await; + #[cfg(feature = "prometheus")] + { + let success = result.is_ok(); + METRICS.record_mint_operation("transaction_commit", success); + METRICS.record_mint_operation_histogram("transaction_commit", success, 1.0); + } + + Ok(result?) } async fn rollback(self: Box) -> Result<(), Error> { - self.inner.rollback().await + let result = self.inner.rollback().await; + + #[cfg(feature = "prometheus")] + { + let success = result.is_ok(); + METRICS.record_mint_operation("transaction_rollback", success); + METRICS.record_mint_operation_histogram("transaction_rollback", success, 1.0); + } + Ok(result?) } } @@ -443,12 +461,14 @@ where async fn begin_transaction<'a>( &'a self, ) -> Result + Send + Sync + 'a>, Error> { - Ok(Box::new(SQLTransaction { + let tx = SQLTransaction { inner: ConnectionWithTransaction::new( self.pool.get().map_err(|e| Error::Database(Box::new(e)))?, ) .await?, - })) + }; + + Ok(Box::new(tx)) } async fn get_active_keyset_id(&self, unit: &CurrencyUnit) -> Result, Self::Err> { @@ -1072,35 +1092,58 @@ where type Err = Error; async fn get_mint_quote(&self, quote_id: &QuoteId) -> Result, Self::Err> { + #[cfg(feature = "prometheus")] + METRICS.inc_in_flight_requests("get_mint_quote"); + + #[cfg(feature = "prometheus")] + let start_time = std::time::Instant::now(); let conn = self.pool.get().map_err(|e| Error::Database(Box::new(e)))?; - let payments = get_mint_quote_payments(&*conn, quote_id).await?; - let issuance = get_mint_quote_issuance(&*conn, quote_id).await?; + let result = async { + let payments = get_mint_quote_payments(&*conn, quote_id).await?; + let issuance = get_mint_quote_issuance(&*conn, quote_id).await?; - Ok(query( - r#" - SELECT - id, - amount, - unit, - request, - expiry, - request_lookup_id, - pubkey, - created_time, - amount_paid, - amount_issued, - payment_method, - request_lookup_id_kind - FROM - mint_quote - WHERE id = :id"#, - )? - .bind("id", quote_id.to_string()) - .fetch_one(&*conn) - .await? - .map(|row| sql_row_to_mint_quote(row, payments, issuance)) - .transpose()?) + query( + r#" + SELECT + id, + amount, + unit, + request, + expiry, + request_lookup_id, + pubkey, + created_time, + amount_paid, + amount_issued, + payment_method, + request_lookup_id_kind + FROM + mint_quote + WHERE id = :id"#, + )? + .bind("id", quote_id.to_string()) + .fetch_one(&*conn) + .await? + .map(|row| sql_row_to_mint_quote(row, payments, issuance)) + .transpose() + } + .await; + + #[cfg(feature = "prometheus")] + { + let success = result.is_ok(); + + METRICS.record_mint_operation("get_mint_quote", success); + METRICS.record_mint_operation_histogram( + "get_mint_quote", + success, + start_time.elapsed().as_secs_f64(), + ); + METRICS.dec_in_flight_requests("get_mint_quote"); + } + + result } async fn get_mint_quote_by_request( @@ -1228,35 +1271,59 @@ where &self, quote_id: &QuoteId, ) -> Result, Self::Err> { + #[cfg(feature = "prometheus")] + METRICS.inc_in_flight_requests("get_melt_quote"); + + #[cfg(feature = "prometheus")] + let start_time = std::time::Instant::now(); let conn = self.pool.get().map_err(|e| Error::Database(Box::new(e)))?; - Ok(query( - r#" - SELECT - id, - unit, - amount, - request, - fee_reserve, - expiry, - state, - payment_preimage, - request_lookup_id, - created_time, - paid_time, - payment_method, - options, - request_lookup_id_kind - FROM - melt_quote - WHERE - id=:id - "#, - )? - .bind("id", quote_id.to_string()) - .fetch_one(&*conn) - .await? - .map(sql_row_to_melt_quote) - .transpose()?) + + let result = async { + query( + r#" + SELECT + id, + unit, + amount, + request, + fee_reserve, + expiry, + state, + payment_preimage, + request_lookup_id, + created_time, + paid_time, + payment_method, + options, + request_lookup_id_kind + FROM + melt_quote + WHERE + id=:id + "#, + )? + .bind("id", quote_id.to_string()) + .fetch_one(&*conn) + .await? + .map(sql_row_to_melt_quote) + .transpose() + } + .await; + + #[cfg(feature = "prometheus")] + { + let success = result.is_ok(); + + METRICS.record_mint_operation("get_melt_quote", success); + METRICS.record_mint_operation_histogram( + "get_melt_quote", + success, + start_time.elapsed().as_secs_f64(), + ); + METRICS.dec_in_flight_requests("get_melt_quote"); + } + + result } async fn get_melt_quotes(&self) -> Result, Self::Err> { @@ -1826,20 +1893,64 @@ where async fn begin_transaction<'a>( &'a self, ) -> Result + Send + Sync + 'a>, Error> { - Ok(Box::new(SQLTransaction { + let tx = SQLTransaction { inner: ConnectionWithTransaction::new( self.pool.get().map_err(|e| Error::Database(Box::new(e)))?, ) .await?, - })) + }; + + Ok(Box::new(tx)) } async fn get_mint_info(&self) -> Result { - Ok(self.fetch_from_config("mint_info").await?) + #[cfg(feature = "prometheus")] + METRICS.inc_in_flight_requests("get_mint_info"); + + #[cfg(feature = "prometheus")] + let start_time = std::time::Instant::now(); + + let result = self.fetch_from_config("mint_info").await; + + #[cfg(feature = "prometheus")] + { + let success = result.is_ok(); + + METRICS.record_mint_operation("get_mint_info", success); + METRICS.record_mint_operation_histogram( + "get_mint_info", + success, + start_time.elapsed().as_secs_f64(), + ); + METRICS.dec_in_flight_requests("get_mint_info"); + } + + Ok(result?) } async fn get_quote_ttl(&self) -> Result { - Ok(self.fetch_from_config("quote_ttl").await?) + #[cfg(feature = "prometheus")] + METRICS.inc_in_flight_requests("get_quote_ttl"); + + #[cfg(feature = "prometheus")] + let start_time = std::time::Instant::now(); + + let result = self.fetch_from_config("quote_ttl").await; + + #[cfg(feature = "prometheus")] + { + let success = result.is_ok(); + + METRICS.record_mint_operation("get_quote_ttl", success); + METRICS.record_mint_operation_histogram( + "get_quote_ttl", + success, + start_time.elapsed().as_secs_f64(), + ); + METRICS.dec_in_flight_requests("get_quote_ttl"); + } + + Ok(result?) } } diff --git a/crates/cdk-sql-common/src/pool.rs b/crates/cdk-sql-common/src/pool.rs index ebcac59f..9315a081 100644 --- a/crates/cdk-sql-common/src/pool.rs +++ b/crates/cdk-sql-common/src/pool.rs @@ -6,9 +6,10 @@ use std::fmt::Debug; use std::ops::{Deref, DerefMut}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Condvar, Mutex}; -use std::time::Duration; +use std::time::{Duration, Instant}; -use tokio::time::Instant; +#[cfg(feature = "prometheus")] +use cdk_prometheus::metrics::METRICS; use crate::database::DatabaseConnector; @@ -86,6 +87,8 @@ where { resource: Option<(Arc, RM::Connection)>, pool: Arc>, + #[cfg(feature = "prometheus")] + start_time: Instant, } impl Debug for PooledResource @@ -105,7 +108,16 @@ where if let Some(resource) = self.resource.take() { let mut active_resource = self.pool.queue.lock().expect("active_resource"); active_resource.push(resource); - self.pool.in_use.fetch_sub(1, Ordering::AcqRel); + let _in_use = self.pool.in_use.fetch_sub(1, Ordering::AcqRel); + + #[cfg(feature = "prometheus")] + { + METRICS.set_db_connections_active(_in_use as i64); + + let duration = self.start_time.elapsed().as_secs_f64(); + + METRICS.record_db_operation(duration, "drop"); + } // Notify a waiting thread self.pool.waiter.notify_one(); @@ -155,6 +167,18 @@ where self.get_timeout(self.default_timeout) } + /// Increments the in_use connection counter and updates the metric + fn increment_connection_counter(&self) -> usize { + let in_use = self.in_use.fetch_add(1, Ordering::AcqRel); + + #[cfg(feature = "prometheus")] + { + METRICS.set_db_connections_active(in_use as i64); + } + + in_use + } + /// Get a new resource or fail after timeout is reached. /// /// This function will return a free resource or create a new one if there is still room for it; @@ -171,18 +195,20 @@ where if let Some((stale, resource)) = resources.pop() { if !stale.load(Ordering::SeqCst) { drop(resources); - self.in_use.fetch_add(1, Ordering::AcqRel); + self.increment_connection_counter(); return Ok(PooledResource { resource: Some((stale, resource)), pool: self.clone(), + #[cfg(feature = "prometheus")] + start_time: Instant::now(), }); } } if self.in_use.load(Ordering::Relaxed) < self.max_size { drop(resources); - self.in_use.fetch_add(1, Ordering::AcqRel); + self.increment_connection_counter(); let stale: Arc = Arc::new(false.into()); return Ok(PooledResource { @@ -191,6 +217,8 @@ where RM::new_resource(&self.config, stale, timeout)?, )), pool: self.clone(), + #[cfg(feature = "prometheus")] + start_time: Instant::now(), }); } diff --git a/crates/cdk-sqlite/Cargo.toml b/crates/cdk-sqlite/Cargo.toml index 094005f7..5ffd314d 100644 --- a/crates/cdk-sqlite/Cargo.toml +++ b/crates/cdk-sqlite/Cargo.toml @@ -17,10 +17,11 @@ mint = ["cdk-common/mint", "cdk-sql-common/mint"] wallet = ["cdk-common/wallet", "cdk-sql-common/wallet"] auth = ["cdk-common/auth", "cdk-sql-common/auth"] sqlcipher = ["rusqlite/bundled-sqlcipher"] - +prometheus = ["cdk-sql-common/prometheus", "cdk-prometheus"] [dependencies] async-trait.workspace = true cdk-common = { workspace = true, features = ["test"] } +cdk-prometheus = { workspace = true, optional = true } bitcoin.workspace = true cdk-sql-common = { workspace = true } rusqlite = { version = "0.31", features = ["bundled"]} diff --git a/crates/cdk/Cargo.toml b/crates/cdk/Cargo.toml index dec29d19..529dfebb 100644 --- a/crates/cdk/Cargo.toml +++ b/crates/cdk/Cargo.toml @@ -20,7 +20,7 @@ bip353 = ["dep:trust-dns-resolver"] swagger = ["mint", "dep:utoipa", "cdk-common/swagger"] bench = [] http_subscription = [] - +prometheus = ["dep:cdk-prometheus"] [dependencies] cdk-common.workspace = true @@ -44,7 +44,7 @@ utoipa = { workspace = true, optional = true } uuid.workspace = true jsonwebtoken = { workspace = true, optional = true } trust-dns-resolver = { version = "0.23.2", optional = true } - +cdk-prometheus = {workspace = true, optional = true} # -Z minimal-versions sync_wrapper = "0.1.2" bech32 = "0.9.1" diff --git a/crates/cdk/src/mint/builder.rs b/crates/cdk/src/mint/builder.rs index b0cc1903..765ecc9f 100644 --- a/crates/cdk/src/mint/builder.rs +++ b/crates/cdk/src/mint/builder.rs @@ -268,7 +268,6 @@ impl MintBuilder { self.payment_processors.insert(key, payment_processor); Ok(()) } - /// Sets the input fee ppk for a given unit /// /// The unit **MUST** already have been added with a ln backend diff --git a/crates/cdk/src/mint/issue/mod.rs b/crates/cdk/src/mint/issue/mod.rs index 1e5767db..f3965ee3 100644 --- a/crates/cdk/src/mint/issue/mod.rs +++ b/crates/cdk/src/mint/issue/mod.rs @@ -10,6 +10,8 @@ use cdk_common::{ MintQuoteBolt11Response, MintQuoteBolt12Request, MintQuoteBolt12Response, MintQuoteState, MintRequest, MintResponse, NotificationPayload, PaymentMethod, PublicKey, }; +#[cfg(feature = "prometheus")] +use cdk_prometheus::METRICS; use tracing::instrument; use crate::mint::Verification; @@ -187,130 +189,147 @@ impl Mint { &self, mint_quote_request: MintQuoteRequest, ) -> Result { - let unit: CurrencyUnit; - let amount; - let pubkey; - let payment_method; + #[cfg(feature = "prometheus")] + METRICS.inc_in_flight_requests("get_mint_quote"); - let create_invoice_response = match mint_quote_request { - MintQuoteRequest::Bolt11(bolt11_request) => { - unit = bolt11_request.unit; - amount = Some(bolt11_request.amount); - pubkey = bolt11_request.pubkey; - payment_method = PaymentMethod::Bolt11; + let result = async { + let unit: CurrencyUnit; + let amount; + let pubkey; + let payment_method; - self.check_mint_request_acceptable( - Some(bolt11_request.amount), - &unit, - &payment_method, - ) - .await?; + let create_invoice_response = match mint_quote_request { + MintQuoteRequest::Bolt11(bolt11_request) => { + unit = bolt11_request.unit; + amount = Some(bolt11_request.amount); + pubkey = bolt11_request.pubkey; + payment_method = PaymentMethod::Bolt11; - let ln = self.get_payment_processor(unit.clone(), PaymentMethod::Bolt11)?; - - let mint_ttl = self.localstore.get_quote_ttl().await?.mint_ttl; - - let quote_expiry = unix_time() + mint_ttl; - - let settings = ln.get_settings().await?; - let settings: Bolt11Settings = serde_json::from_value(settings)?; - - let description = bolt11_request.description; - - if description.is_some() && !settings.invoice_description { - tracing::error!("Backend does not support invoice description"); - return Err(Error::InvoiceDescriptionUnsupported); - } - - let bolt11_options = Bolt11IncomingPaymentOptions { - description, - amount: bolt11_request.amount, - unix_expiry: Some(quote_expiry), - }; - - let incoming_options = IncomingPaymentOptions::Bolt11(bolt11_options); - - ln.create_incoming_payment_request(&unit, incoming_options) - .await - .map_err(|err| { - tracing::error!("Could not create invoice: {}", err); - Error::InvalidPaymentRequest - })? - } - MintQuoteRequest::Bolt12(bolt12_request) => { - unit = bolt12_request.unit; - amount = bolt12_request.amount; - pubkey = Some(bolt12_request.pubkey); - payment_method = PaymentMethod::Bolt12; - - self.check_mint_request_acceptable(amount, &unit, &payment_method) + self.check_mint_request_acceptable( + Some(bolt11_request.amount), + &unit, + &payment_method, + ) .await?; - let ln = self.get_payment_processor(unit.clone(), payment_method.clone())?; + let ln = self.get_payment_processor(unit.clone(), PaymentMethod::Bolt11)?; - let description = bolt12_request.description; + let mint_ttl = self.localstore.get_quote_ttl().await?.mint_ttl; - let bolt12_options = Bolt12IncomingPaymentOptions { - description, - amount, - unix_expiry: None, - }; + let quote_expiry = unix_time() + mint_ttl; - let incoming_options = IncomingPaymentOptions::Bolt12(Box::new(bolt12_options)); + let settings = ln.get_settings().await?; + let settings: Bolt11Settings = serde_json::from_value(settings)?; - ln.create_incoming_payment_request(&unit, incoming_options) - .await - .map_err(|err| { - tracing::error!("Could not create invoice: {}", err); - Error::InvalidPaymentRequest - })? + let description = bolt11_request.description; + + if description.is_some() && !settings.invoice_description { + tracing::error!("Backend does not support invoice description"); + return Err(Error::InvoiceDescriptionUnsupported); + } + + let bolt11_options = Bolt11IncomingPaymentOptions { + description, + amount: bolt11_request.amount, + unix_expiry: Some(quote_expiry), + }; + + let incoming_options = IncomingPaymentOptions::Bolt11(bolt11_options); + + ln.create_incoming_payment_request(&unit, incoming_options) + .await + .map_err(|err| { + tracing::error!("Could not create invoice: {}", err); + Error::InvalidPaymentRequest + })? + } + MintQuoteRequest::Bolt12(bolt12_request) => { + unit = bolt12_request.unit; + amount = bolt12_request.amount; + pubkey = Some(bolt12_request.pubkey); + payment_method = PaymentMethod::Bolt12; + + self.check_mint_request_acceptable(amount, &unit, &payment_method) + .await?; + + let ln = self.get_payment_processor(unit.clone(), payment_method.clone())?; + + let description = bolt12_request.description; + + let bolt12_options = Bolt12IncomingPaymentOptions { + description, + amount, + unix_expiry: None, + }; + + let incoming_options = IncomingPaymentOptions::Bolt12(Box::new(bolt12_options)); + + ln.create_incoming_payment_request(&unit, incoming_options) + .await + .map_err(|err| { + tracing::error!("Could not create invoice: {}", err); + Error::InvalidPaymentRequest + })? + } + }; + + let quote = MintQuote::new( + None, + create_invoice_response.request.to_string(), + unit.clone(), + amount, + create_invoice_response.expiry.unwrap_or(0), + create_invoice_response.request_lookup_id.clone(), + pubkey, + Amount::ZERO, + Amount::ZERO, + payment_method.clone(), + unix_time(), + vec![], + vec![], + ); + + tracing::debug!( + "New {} mint quote {} for {:?} {} with request id {:?}", + payment_method, + quote.id, + amount, + unit, + create_invoice_response.request_lookup_id.to_string(), + ); + + let mut tx = self.localstore.begin_transaction().await?; + tx.add_mint_quote(quote.clone()).await?; + tx.commit().await?; + + match payment_method { + PaymentMethod::Bolt11 => { + let res: MintQuoteBolt11Response = quote.clone().into(); + self.pubsub_manager + .broadcast(NotificationPayload::MintQuoteBolt11Response(res)); + } + PaymentMethod::Bolt12 => { + let res: MintQuoteBolt12Response = quote.clone().try_into()?; + self.pubsub_manager + .broadcast(NotificationPayload::MintQuoteBolt12Response(res)); + } + PaymentMethod::Custom(_) => {} } - }; - let quote = MintQuote::new( - None, - create_invoice_response.request.to_string(), - unit.clone(), - amount, - create_invoice_response.expiry.unwrap_or(0), - create_invoice_response.request_lookup_id.clone(), - pubkey, - Amount::ZERO, - Amount::ZERO, - payment_method.clone(), - unix_time(), - vec![], - vec![], - ); + quote.try_into() + } + .await; - tracing::debug!( - "New {} mint quote {} for {:?} {} with request id {:?}", - payment_method, - quote.id, - amount, - unit, - create_invoice_response.request_lookup_id.to_string(), - ); - - let mut tx = self.localstore.begin_transaction().await?; - tx.add_mint_quote(quote.clone()).await?; - tx.commit().await?; - - match payment_method { - PaymentMethod::Bolt11 => { - let res: MintQuoteBolt11Response = quote.clone().into(); - self.pubsub_manager - .broadcast(NotificationPayload::MintQuoteBolt11Response(res)); + #[cfg(feature = "prometheus")] + { + METRICS.dec_in_flight_requests("get_mint_quote"); + METRICS.record_mint_operation("get_mint_quote", result.is_ok()); + if result.is_err() { + METRICS.record_error(); } - PaymentMethod::Bolt12 => { - let res: MintQuoteBolt12Response = quote.clone().try_into()?; - self.pubsub_manager - .broadcast(NotificationPayload::MintQuoteBolt12Response(res)); - } - PaymentMethod::Custom(_) => {} } - quote.try_into() + result } /// Retrieves all mint quotes from the database @@ -320,8 +339,25 @@ impl Mint { /// * `Error` if database access fails #[instrument(skip_all)] pub async fn mint_quotes(&self) -> Result, Error> { - let quotes = self.localstore.get_mint_quotes().await?; - Ok(quotes) + #[cfg(feature = "prometheus")] + METRICS.inc_in_flight_requests("mint_quotes"); + + let result = async { + let quotes = self.localstore.get_mint_quotes().await?; + Ok(quotes) + } + .await; + + #[cfg(feature = "prometheus")] + { + METRICS.dec_in_flight_requests("mint_quotes"); + METRICS.record_mint_operation("mint_quotes", result.is_ok()); + if result.is_err() { + METRICS.record_error(); + } + } + + result } /// Removes a mint quote from the database @@ -334,11 +370,27 @@ impl Mint { /// * `Error` if the quote doesn't exist or removal fails #[instrument(skip_all)] pub async fn remove_mint_quote(&self, quote_id: &QuoteId) -> Result<(), Error> { - let mut tx = self.localstore.begin_transaction().await?; - tx.remove_mint_quote(quote_id).await?; - tx.commit().await?; + #[cfg(feature = "prometheus")] + METRICS.inc_in_flight_requests("remove_mint_quote"); - Ok(()) + let result = async { + let mut tx = self.localstore.begin_transaction().await?; + tx.remove_mint_quote(quote_id).await?; + tx.commit().await?; + Ok(()) + } + .await; + + #[cfg(feature = "prometheus")] + { + METRICS.dec_in_flight_requests("remove_mint_quote"); + METRICS.record_mint_operation("remove_mint_quote", result.is_ok()); + if result.is_err() { + METRICS.record_error(); + } + } + + result } /// Marks a mint quote as paid based on the payment request ID @@ -357,33 +409,48 @@ impl Mint { &self, wait_payment_response: WaitPaymentResponse, ) -> Result<(), Error> { - if wait_payment_response.payment_amount == Amount::ZERO { - tracing::warn!( - "Received payment response with 0 amount with payment id {}.", - wait_payment_response.payment_id.to_string() - ); + #[cfg(feature = "prometheus")] + METRICS.inc_in_flight_requests("pay_mint_quote_for_request_id"); + let result = async { + if wait_payment_response.payment_amount == Amount::ZERO { + tracing::warn!( + "Received payment response with 0 amount with payment id {}.", + wait_payment_response.payment_id.to_string() + ); + return Err(Error::AmountUndefined); + } - return Err(Error::AmountUndefined); + let mut tx = self.localstore.begin_transaction().await?; + + if let Ok(Some(mint_quote)) = tx + .get_mint_quote_by_request_lookup_id(&wait_payment_response.payment_identifier) + .await + { + self.pay_mint_quote(&mut tx, &mint_quote, wait_payment_response) + .await?; + } else { + tracing::warn!( + "Could not get request for request lookup id {:?}.", + wait_payment_response.payment_identifier + ); + } + + tx.commit().await?; + + Ok(()) } + .await; - let mut tx = self.localstore.begin_transaction().await?; - - if let Ok(Some(mint_quote)) = tx - .get_mint_quote_by_request_lookup_id(&wait_payment_response.payment_identifier) - .await + #[cfg(feature = "prometheus")] { - self.pay_mint_quote(&mut tx, &mint_quote, wait_payment_response) - .await?; - } else { - tracing::warn!( - "Could not get request for request lookup id {:?}.", - wait_payment_response.payment_identifier - ); + METRICS.dec_in_flight_requests("pay_mint_quote_for_request_id"); + METRICS.record_mint_operation("pay_mint_quote_for_request_id", result.is_ok()); + if result.is_err() { + METRICS.record_error(); + } } - tx.commit().await?; - - Ok(()) + result } /// Marks a specific mint quote as paid @@ -405,8 +472,30 @@ impl Mint { mint_quote: &MintQuote, wait_payment_response: WaitPaymentResponse, ) -> Result<(), Error> { - Self::handle_mint_quote_payment(tx, mint_quote, wait_payment_response, &self.pubsub_manager) + #[cfg(feature = "prometheus")] + METRICS.inc_in_flight_requests("pay_mint_quote"); + + let result = async { + Self::handle_mint_quote_payment( + tx, + mint_quote, + wait_payment_response, + &self.pubsub_manager, + ) .await + } + .await; + + #[cfg(feature = "prometheus")] + { + METRICS.dec_in_flight_requests("pay_mint_quote"); + METRICS.record_mint_operation("pay_mint_quote", result.is_ok()); + if result.is_err() { + METRICS.record_error(); + } + } + + result } /// Checks the status of a mint quote and updates it if necessary @@ -422,17 +511,33 @@ impl Mint { /// * `Error` if the quote doesn't exist or checking fails #[instrument(skip(self))] pub async fn check_mint_quote(&self, quote_id: &QuoteId) -> Result { - let mut quote = self - .localstore - .get_mint_quote(quote_id) - .await? - .ok_or(Error::UnknownQuote)?; + #[cfg(feature = "prometheus")] + METRICS.inc_in_flight_requests("check_mint_quote"); + let result = async { + let mut quote = self + .localstore + .get_mint_quote(quote_id) + .await? + .ok_or(Error::UnknownQuote)?; - if quote.payment_method == PaymentMethod::Bolt11 { - self.check_mint_quote_paid(&mut quote).await?; + if quote.payment_method == PaymentMethod::Bolt11 { + self.check_mint_quote_paid(&mut quote).await?; + } + + quote.try_into() + } + .await; + + #[cfg(feature = "prometheus")] + { + METRICS.dec_in_flight_requests("check_mint_quote"); + METRICS.record_mint_operation("check_mint_quote", result.is_ok()); + if result.is_err() { + METRICS.record_error(); + } } - quote.try_into() + result } /// Processes a mint request to issue new tokens @@ -456,15 +561,18 @@ impl Mint { &self, mint_request: MintRequest, ) -> Result { - let mut mint_quote = self - .localstore - .get_mint_quote(&mint_request.quote) - .await? - .ok_or(Error::UnknownQuote)?; - if mint_quote.payment_method == PaymentMethod::Bolt11 { - self.check_mint_quote_paid(&mut mint_quote).await?; - } + #[cfg(feature = "prometheus")] + METRICS.inc_in_flight_requests("process_mint_request"); + let result = async { + let mut mint_quote = self + .localstore + .get_mint_quote(&mint_request.quote) + .await? + .ok_or(Error::UnknownQuote)?; + if mint_quote.payment_method == PaymentMethod::Bolt11 { + self.check_mint_quote_paid(&mut mint_quote).await?; + } // get the blind signatures before having starting the db transaction, if there are any // rollbacks this blind_signatures will be lost, and the signature is stateless. It is not a // good idea to call an external service (which is really a trait, it could be anything @@ -504,10 +612,10 @@ impl Mint { PaymentMethod::Bolt12 => { if mint_quote.amount_issued() > mint_quote.amount_paid() { tracing::error!( - "Quote state should not be issued if issued {} is > paid {}.", - mint_quote.amount_issued(), - mint_quote.amount_paid() - ); + "Quote state should not be issued if issued {} is > paid {}.", + mint_quote.amount_issued(), + mint_quote.amount_paid() + ); return Err(Error::UnpaidQuote); } mint_quote.amount_paid() - mint_quote.amount_issued() @@ -565,7 +673,7 @@ impl Mint { &blind_signatures, Some(mint_request.quote.clone()), ) - .await?; + .await?; let amount_issued = mint_request.total_amount()?; @@ -582,4 +690,16 @@ impl Mint { signatures: blind_signatures, }) } + .await; + + #[cfg(feature = "prometheus")] + { + METRICS.dec_in_flight_requests("process_mint_request"); + METRICS.record_mint_operation("process_mint_request", result.is_ok()); + if result.is_err() { + METRICS.record_error(); + } + } + result + } } diff --git a/crates/cdk/src/mint/melt.rs b/crates/cdk/src/mint/melt.rs index 099a6544..aba1ee48 100644 --- a/crates/cdk/src/mint/melt.rs +++ b/crates/cdk/src/mint/melt.rs @@ -13,6 +13,8 @@ use cdk_common::payment::{ }; use cdk_common::quote_id::QuoteId; use cdk_common::{MeltOptions, MeltQuoteBolt12Request}; +#[cfg(feature = "prometheus")] +use cdk_prometheus::METRICS; use lightning::offers::offer::Offer; use tracing::instrument; @@ -131,6 +133,8 @@ impl Mint { &self, melt_request: &MeltQuoteBolt11Request, ) -> Result, Error> { + #[cfg(feature = "prometheus")] + METRICS.inc_in_flight_requests("get_melt_bolt11_quote"); let MeltQuoteBolt11Request { request, unit, @@ -183,6 +187,12 @@ impl Mint { err ); + #[cfg(feature = "prometheus")] + { + METRICS.dec_in_flight_requests("get_melt_bolt11_quote"); + METRICS.record_mint_operation("get_melt_bolt11_quote", false); + METRICS.record_error(); + } Error::UnsupportedUnit })?; @@ -315,6 +325,12 @@ impl Mint { tx.add_melt_quote(quote.clone()).await?; tx.commit().await?; + #[cfg(feature = "prometheus")] + { + METRICS.dec_in_flight_requests("get_melt_bolt11_quote"); + METRICS.record_mint_operation("get_melt_bolt11_quote", true); + } + Ok(quote.into()) } @@ -324,20 +340,50 @@ impl Mint { &self, quote_id: &QuoteId, ) -> Result, Error> { - let quote = self - .localstore - .get_melt_quote(quote_id) - .await? - .ok_or(Error::UnknownQuote)?; + #[cfg(feature = "prometheus")] + METRICS.inc_in_flight_requests("check_melt_quote"); + let quote = match self.localstore.get_melt_quote(quote_id).await { + Ok(Some(quote)) => quote, + Ok(None) => { + #[cfg(feature = "prometheus")] + { + METRICS.dec_in_flight_requests("check_melt_quote"); + METRICS.record_mint_operation("check_melt_quote", false); + METRICS.record_error(); + } + return Err(Error::UnknownQuote); + } + Err(err) => { + #[cfg(feature = "prometheus")] + { + METRICS.dec_in_flight_requests("check_melt_quote"); + METRICS.record_mint_operation("check_melt_quote", false); + METRICS.record_error(); + } + return Err(err.into()); + } + }; - let blind_signatures = self + let blind_signatures = match self .localstore .get_blind_signatures_for_quote(quote_id) - .await?; + .await + { + Ok(signatures) => signatures, + Err(err) => { + #[cfg(feature = "prometheus")] + { + METRICS.dec_in_flight_requests("check_melt_quote"); + METRICS.record_mint_operation("check_melt_quote", false); + METRICS.record_error(); + } + return Err(err.into()); + } + }; let change = (!blind_signatures.is_empty()).then_some(blind_signatures); - Ok(MeltQuoteBolt11Response { + let response = MeltQuoteBolt11Response { quote: quote.id, paid: Some(quote.state == MeltQuoteState::Paid), state: quote.state, @@ -348,7 +394,15 @@ impl Mint { change, request: Some(quote.request.to_string()), unit: Some(quote.unit.clone()), - }) + }; + + #[cfg(feature = "prometheus")] + { + METRICS.dec_in_flight_requests("check_melt_quote"); + METRICS.record_mint_operation("check_melt_quote", true); + } + + Ok(response) } /// Get melt quotes @@ -520,6 +574,9 @@ impl Mint { &self, melt_request: &MeltRequest, ) -> Result, Error> { + #[cfg(feature = "prometheus")] + METRICS.inc_in_flight_requests("melt_bolt11"); + use std::sync::Arc; async fn check_payment_state( ln: Arc + Send + Sync>, @@ -543,21 +600,43 @@ impl Mint { let mut tx = self.localstore.begin_transaction().await?; - let (proof_writer, quote) = self + let (proof_writer, quote) = match self .verify_melt_request(&mut tx, verification, melt_request) .await - .map_err(|err| { + { + Ok(result) => result, + Err(err) => { tracing::debug!("Error attempting to verify melt quote: {}", err); - err - })?; - let settled_internally_amount = self + #[cfg(feature = "prometheus")] + { + METRICS.dec_in_flight_requests("melt_bolt11"); + METRICS.record_mint_operation("melt_bolt11", false); + METRICS.record_error(); + } + + return Err(err); + } + }; + + let settled_internally_amount = match self .handle_internal_melt_mint(&mut tx, "e, melt_request) .await - .map_err(|err| { + { + Ok(amount) => amount, + Err(err) => { tracing::error!("Attempting to settle internally failed: {}", err); - err - })?; + + #[cfg(feature = "prometheus")] + { + METRICS.dec_in_flight_requests("melt_bolt11"); + METRICS.record_mint_operation("melt_bolt11", false); + METRICS.record_error(); + } + + return Err(err); + } + }; let (tx, preimage, amount_spent_quote_unit, quote) = match settled_internally_amount { Some(amount_spent) => (tx, None, amount_spent, quote), @@ -669,6 +748,14 @@ impl Mint { melt_request.quote() ); proof_writer.rollback().await?; + + #[cfg(feature = "prometheus")] + { + METRICS.dec_in_flight_requests("melt_bolt11"); + METRICS.record_mint_operation("melt_bolt11", false); + METRICS.record_error(); + } + return Err(Error::PaymentFailed); } MeltQuoteState::Pending => { @@ -677,6 +764,13 @@ impl Mint { melt_request.quote() ); proof_writer.commit(); + #[cfg(feature = "prometheus")] + { + METRICS.dec_in_flight_requests("melt_bolt11"); + METRICS.record_mint_operation("melt_bolt11", false); + METRICS.record_error(); + } + return Err(Error::PendingQuote); } } @@ -716,7 +810,7 @@ impl Mint { // If we made it here the payment has been made. // We process the melt burning the inputs and returning change - let res = self + let res = match self .process_melt_request( tx, proof_writer, @@ -726,10 +820,27 @@ impl Mint { amount_spent_quote_unit, ) .await - .map_err(|err| { + { + Ok(response) => response, + Err(err) => { tracing::error!("Could not process melt request: {}", err); - err - })?; + + #[cfg(feature = "prometheus")] + { + METRICS.dec_in_flight_requests("melt_bolt11"); + METRICS.record_mint_operation("melt_bolt11", false); + METRICS.record_error(); + } + + return Err(err); + } + }; + + #[cfg(feature = "prometheus")] + { + METRICS.dec_in_flight_requests("melt_bolt11"); + METRICS.record_mint_operation("melt_bolt11", true); + } Ok(res) } @@ -747,11 +858,20 @@ impl Mint { payment_preimage: Option, total_spent: Amount, ) -> Result, Error> { + #[cfg(feature = "prometheus")] + METRICS.inc_in_flight_requests("process_melt_request"); + let input_ys = melt_request.inputs().ys()?; - proof_writer + let update_proof_states_result = proof_writer .update_proofs_states(&mut tx, &input_ys, State::Spent) - .await?; + .await; + + if update_proof_states_result.is_err() { + #[cfg(feature = "prometheus")] + self.record_melt_quote_failure("process_melt_request"); + return Err(update_proof_states_result.err().unwrap()); + } tx.update_melt_quote_state( melt_request.quote(), @@ -853,7 +973,6 @@ impl Mint { change.clone(), MeltQuoteState::Paid, ); - tracing::debug!( "Melt for quote {} completed total spent {}, total inputs: {}, change given: {}", quote.id, @@ -865,8 +984,7 @@ impl Mint { .expect("Change cannot overflow")) .unwrap_or_default() ); - - Ok(MeltQuoteBolt11Response { + let response = MeltQuoteBolt11Response { amount: quote.amount, paid: Some(true), payment_preimage, @@ -877,6 +995,21 @@ impl Mint { expiry: quote.expiry, request: Some(quote.request.to_string()), unit: Some(quote.unit.clone()), - }) + }; + + #[cfg(feature = "prometheus")] + { + METRICS.dec_in_flight_requests("process_melt_request"); + METRICS.record_mint_operation("process_melt_request", true); + } + + Ok(response) + } + + #[cfg(feature = "prometheus")] + fn record_melt_quote_failure(&self, operation: &str) { + METRICS.dec_in_flight_requests(operation); + METRICS.record_mint_operation(operation, false); + METRICS.record_error(); } } diff --git a/crates/cdk/src/mint/mod.rs b/crates/cdk/src/mint/mod.rs index ec89e81b..2918fcbe 100644 --- a/crates/cdk/src/mint/mod.rs +++ b/crates/cdk/src/mint/mod.rs @@ -14,6 +14,8 @@ use cdk_common::nuts::{self, BlindSignature, BlindedMessage, CurrencyUnit, Id, K use cdk_common::payment::WaitPaymentResponse; pub use cdk_common::quote_id::QuoteId; use cdk_common::secret; +#[cfg(feature = "prometheus")] +use cdk_prometheus::global; use cdk_signatory::signatory::{Signatory, SignatoryKeySet}; use futures::StreamExt; #[cfg(feature = "auth")] @@ -724,41 +726,66 @@ impl Mint { #[tracing::instrument(skip_all)] pub async fn blind_sign( &self, - blinded_messages: Vec, + blinded_message: Vec, ) -> Result, Error> { - self.signatory.blind_sign(blinded_messages).await + #[cfg(feature = "prometheus")] + global::inc_in_flight_requests("blind_sign"); + + let result = self.signatory.blind_sign(blinded_message).await; + + #[cfg(feature = "prometheus")] + { + global::dec_in_flight_requests("blind_sign"); + global::record_mint_operation("blind_sign", result.is_ok()); + } + + result } /// Verify [`Proof`] meets conditions and is signed #[tracing::instrument(skip_all)] pub async fn verify_proofs(&self, proofs: Proofs) -> Result<(), Error> { - proofs - .iter() - .map(|proof| { - // Check if secret is a nut10 secret with conditions - if let Ok(secret) = - <&secret::Secret as TryInto>::try_into(&proof.secret) - { - // Checks and verifies known secret kinds. - // If it is an unknown secret kind it will be treated as a normal secret. - // Spending conditions will **not** be check. It is up to the wallet to ensure - // only supported secret kinds are used as there is no way for the mint to - // enforce only signing supported secrets as they are blinded at - // that point. - match secret.kind() { - Kind::P2PK => { - proof.verify_p2pk()?; - } - Kind::HTLC => { - proof.verify_htlc()?; + #[cfg(feature = "prometheus")] + global::inc_in_flight_requests("verify_proofs"); + + let result = async { + proofs + .iter() + .map(|proof| { + // Check if secret is a nut10 secret with conditions + if let Ok(secret) = + <&secret::Secret as TryInto>::try_into(&proof.secret) + { + // Checks and verifies known secret kinds. + // If it is an unknown secret kind it will be treated as a normal secret. + // Spending conditions will **not** be check. It is up to the wallet to ensure + // only supported secret kinds are used as there is no way for the mint to + // enforce only signing supported secrets as they are blinded at + // that point. + match secret.kind() { + Kind::P2PK => { + proof.verify_p2pk()?; + } + Kind::HTLC => { + proof.verify_htlc()?; + } } } - } - Ok(()) - }) - .collect::, Error>>()?; + Ok(()) + }) + .collect::, Error>>()?; - self.signatory.verify_proofs(proofs).await + self.signatory.verify_proofs(proofs).await + } + .await; + + #[cfg(feature = "prometheus")] + { + global::dec_in_flight_requests("verify_proofs"); + global::record_mint_operation("verify_proofs", result.is_ok()); + } + + result } /// Verify melt request is valid @@ -836,61 +863,92 @@ impl Mint { /// Restore #[instrument(skip_all)] pub async fn restore(&self, request: RestoreRequest) -> Result { - let output_len = request.outputs.len(); + #[cfg(feature = "prometheus")] + global::inc_in_flight_requests("restore"); - let mut outputs = Vec::with_capacity(output_len); - let mut signatures = Vec::with_capacity(output_len); + let result = async { + let output_len = request.outputs.len(); - let blinded_message: Vec = - request.outputs.iter().map(|b| b.blinded_secret).collect(); + let mut outputs = Vec::with_capacity(output_len); + let mut signatures = Vec::with_capacity(output_len); - let blinded_signatures = self - .localstore - .get_blind_signatures(&blinded_message) - .await?; + let blinded_message: Vec = + request.outputs.iter().map(|b| b.blinded_secret).collect(); - assert_eq!(blinded_signatures.len(), output_len); + let blinded_signatures = self + .localstore + .get_blind_signatures(&blinded_message) + .await?; - for (blinded_message, blinded_signature) in - request.outputs.into_iter().zip(blinded_signatures) - { - if let Some(blinded_signature) = blinded_signature { - outputs.push(blinded_message); - signatures.push(blinded_signature); + assert_eq!(blinded_signatures.len(), output_len); + + for (blinded_message, blinded_signature) in + request.outputs.into_iter().zip(blinded_signatures) + { + if let Some(blinded_signature) = blinded_signature { + outputs.push(blinded_message); + signatures.push(blinded_signature); + } } + + Ok(RestoreResponse { + outputs, + signatures: signatures.clone(), + promises: Some(signatures), + }) + } + .await; + + #[cfg(feature = "prometheus")] + { + global::dec_in_flight_requests("restore"); + global::record_mint_operation("restore", result.is_ok()); } - Ok(RestoreResponse { - outputs, - signatures: signatures.clone(), - promises: Some(signatures), - }) + result } /// Get the total amount issed by keyset #[instrument(skip_all)] pub async fn total_issued(&self) -> Result, Error> { - let keysets = self.keysets().keysets; + #[cfg(feature = "prometheus")] + global::inc_in_flight_requests("total_issued"); - let mut total_issued = HashMap::new(); + let result = async { + let keysets = self.keysets().keysets; - for keyset in keysets { - let blinded = self - .localstore - .get_blind_signatures_for_keyset(&keyset.id) - .await?; + let mut total_issued = HashMap::new(); - let total = Amount::try_sum(blinded.iter().map(|b| b.amount))?; + for keyset in keysets { + let blinded = self + .localstore + .get_blind_signatures_for_keyset(&keyset.id) + .await?; - total_issued.insert(keyset.id, total); + let total = Amount::try_sum(blinded.iter().map(|b| b.amount))?; + + total_issued.insert(keyset.id, total); + } + + Ok(total_issued) + } + .await; + + #[cfg(feature = "prometheus")] + { + global::dec_in_flight_requests("total_issued"); + global::record_mint_operation("total_issued", result.is_ok()); } - Ok(total_issued) + result } /// Total redeemed for keyset #[instrument(skip_all)] pub async fn total_redeemed(&self) -> Result, Error> { + #[cfg(feature = "prometheus")] + global::inc_in_flight_requests("total_redeemed"); + let keysets = self.signatory.keysets().await?; let mut total_redeemed = HashMap::new(); @@ -909,6 +967,9 @@ impl Mint { total_redeemed.insert(keyset.id, total_spent); } + #[cfg(feature = "prometheus")] + global::dec_in_flight_requests("total_redeemed"); + Ok(total_redeemed) } } diff --git a/crates/cdk/src/mint/swap.rs b/crates/cdk/src/mint/swap.rs index 670fa079..0e14fa32 100644 --- a/crates/cdk/src/mint/swap.rs +++ b/crates/cdk/src/mint/swap.rs @@ -1,3 +1,5 @@ +#[cfg(feature = "prometheus")] +use cdk_prometheus::METRICS; use tracing::instrument; use super::nut11::{enforce_sig_flag, EnforceSigFlag}; @@ -12,6 +14,8 @@ impl Mint { &self, swap_request: SwapRequest, ) -> Result { + #[cfg(feature = "prometheus")] + METRICS.inc_in_flight_requests("process_swap_request"); // Do the external call before beginning the db transaction // Check any overflow before talking to the signatory swap_request.input_amount()?; @@ -25,7 +29,6 @@ impl Mint { tracing::debug!("Input verification failed: {:?}", err); err })?; - let mut tx = self.localstore.begin_transaction().await?; if let Err(err) = self @@ -38,20 +41,50 @@ impl Mint { .await { tracing::debug!("Attempt to swap unbalanced transaction, aborting: {err}"); + + #[cfg(feature = "prometheus")] + { + METRICS.dec_in_flight_requests("process_swap_request"); + METRICS.record_mint_operation("process_swap_request", false); + METRICS.record_error(); + } + return Err(err); }; - self.validate_sig_flag(&swap_request).await?; - + let validate_sig_result = self.validate_sig_flag(&swap_request).await; + if validate_sig_result.is_err() { + #[cfg(feature = "prometheus")] + self.record_swap_failure("process_swap_request"); + return Err(validate_sig_result.err().unwrap()); + } let mut proof_writer = ProofWriter::new(self.localstore.clone(), self.pubsub_manager.clone()); - let input_ys = proof_writer + let input_ys = match proof_writer .add_proofs(&mut tx, swap_request.inputs()) - .await?; + .await + { + Ok(ys) => ys, + Err(err) => { + #[cfg(feature = "prometheus")] + { + METRICS.dec_in_flight_requests("process_swap_request"); + METRICS.record_mint_operation("process_swap_request", false); + METRICS.record_error(); + } + return Err(err); + } + }; - proof_writer + let update_proof_states_result = proof_writer .update_proofs_states(&mut tx, &input_ys, State::Spent) - .await?; + .await; + + if update_proof_states_result.is_err() { + #[cfg(feature = "prometheus")] + self.record_swap_failure("process_swap_request"); + return Err(update_proof_states_result.err().unwrap()); + } tx.add_blind_signatures( &swap_request @@ -67,7 +100,15 @@ impl Mint { proof_writer.commit(); tx.commit().await?; - Ok(SwapResponse::new(promises)) + let response = SwapResponse::new(promises); + + #[cfg(feature = "prometheus")] + { + METRICS.dec_in_flight_requests("process_swap_request"); + METRICS.record_mint_operation("process_swap_request", true); + } + + Ok(response) } async fn validate_sig_flag(&self, swap_request: &SwapRequest) -> Result<(), Error> { @@ -79,4 +120,10 @@ impl Mint { Ok(()) } + #[cfg(feature = "prometheus")] + fn record_swap_failure(&self, operation: &str) { + METRICS.dec_in_flight_requests(operation); + METRICS.record_mint_operation(operation, false); + METRICS.record_error(); + } } diff --git a/docker-compose.yaml b/docker-compose.yaml index 0f8314e9..4a4ba88c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,5 +1,40 @@ +version: '3.8' + services: # CDK Mint service + + prometheus: + image: prom/prometheus:latest + ports: + - "9090:9090" + volumes: + - ./misc/provisioning/prometheus.yml:/etc/prometheus/prometheus.yml:ro + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/etc/prometheus/console_libraries' + - '--web.console.templates=/etc/prometheus/consoles' + - '--web.enable-lifecycle' + - '--enable-feature=otlp-write-receiver' + extra_hosts: + - "host.docker.internal:host-gateway" + networks: + - cdk + + # Grafana for visualization + grafana: + image: grafana/grafana:latest + ports: + - "3011:3000" + volumes: + - ./misc/provisioning/datasources:/etc/grafana/provisioning/datasources + - ./misc/provisioning/dashboards:/etc/grafana/provisioning/dashboards + environment: + - GF_DASHBOARDS_JSON_ENABLED=true + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_PROVISIONING_PATHS=/etc/grafana/provisioning + networks: + - cdk mintd: build: context: . @@ -16,7 +51,7 @@ services: # Database configuration - choose one: # Option 1: SQLite (embedded, no additional setup needed) - CDK_MINTD_DATABASE=sqlite - # Option 2: ReDB (embedded, no additional setup needed) + # Option 2: ReDB (embedded, no additional setup needed) # - CDK_MINTD_DATABASE=redb # Option 3: PostgreSQL (requires postgres service, enable with: docker-compose --profile postgres up) # - CDK_MINTD_DATABASE=postgres @@ -24,9 +59,17 @@ services: # Cache configuration - CDK_MINTD_CACHE_BACKEND=memory # For Redis cache (requires redis service, enable with: docker-compose --profile redis up): - # - CDK_MINTD_CACHE_REDIS_URL=redis://redis:6379 + # - CDK_MINTD_CACHE_REDIS_URL=redis://redis:6379 # - CDK_MINTD_CACHE_REDIS_KEY_PREFIX=cdk-mintd + - CDK_MINTD_PROMETHEUS_ENABLED=true + - CDK_MINTD_PROMETHEUS_ADDRESS=0.0.0.0 + - CDK_MINTD_PROMETHEUS_PORT=9000 command: ["cdk-mintd"] + depends_on: + - prometheus + - grafana + networks: + - cdk # Uncomment when using PostgreSQL: # depends_on: # - postgres @@ -78,3 +121,9 @@ volumes: driver: local # redis_data: # driver: local + + + +networks: + cdk: + driver: bridge diff --git a/misc/provisioning/dashboards/dashboard.json b/misc/provisioning/dashboards/dashboard.json new file mode 100644 index 00000000..1f1d66c4 --- /dev/null +++ b/misc/provisioning/dashboards/dashboard.json @@ -0,0 +1,1983 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 7, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "vis": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "process_cpu_usage_percent", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "CPU Usage", + "refId": "A" + } + ], + "title": "Process CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "vis": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "process_memory_bytes", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Memory (Bytes)", + "refId": "A" + } + ], + "title": "Process Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "vis": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "process_memory_percent", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Memory (%)", + "refId": "A" + } + ], + "title": "Memory Percentage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "vis": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(cdk_http_requests_total[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{endpoint}} - {{status}}", + "range": true, + "refId": "A" + } + ], + "title": "HTTP Request Rate by Endpoint", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 50 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 8 + }, + "id": 2, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(cdk_http_requests_total{status!=\"200\"}[$__range])) / sum(rate(cdk_http_requests_total[$__range])) * 100", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "Error Rate", + "range": true, + "refId": "A" + } + ], + "title": "HTTP Error Rate", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "vis": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "/v1/keys/{keyset_id} (200)", + "/v1/mint/quote/bolt11 (200)", + "/v1/swap (400)", + "/v1/mint/bolt11 (200)" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 8 + }, + "id": 9, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "cdk_http_requests_total", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{endpoint}} ({{status}})", + "refId": "A" + } + ], + "title": "HTTP Requests Distribution", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "vis": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "p50 - /v1/mint/bolt11" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, rate(cdk_http_request_duration_seconds_bucket[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p50 - {{endpoint}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, rate(cdk_http_request_duration_seconds_bucket[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p95 - {{endpoint}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, rate(cdk_http_request_duration_seconds_bucket[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p99 - {{endpoint}}", + "range": true, + "refId": "C" + } + ], + "title": "HTTP Request Duration Percentiles", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "vis": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(cdk_mint_operations_total[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{operation}} - {{status}}", + "range": true, + "refId": "A" + } + ], + "title": "Mint Operations Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "vis": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "cdk_mint_in_flight_requests", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "title": "In-Flight Mint Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "vis": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "sats" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, rate(cdk_lightning_payment_amount_sats_bucket[5m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p50 Payment Amount", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, rate(cdk_lightning_payment_amount_sats_bucket[5m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p95 Payment Amount", + "refId": "B" + } + ], + "title": "Lightning Payment Amounts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "vis": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "cdk_db_connections_active", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Active Connections", + "refId": "A" + } + ], + "title": "Database Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "vis": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "sats" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, rate(cdk_lightning_payment_fees_sats_bucket[5m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p50 Payment Fees", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, rate(cdk_lightning_payment_fees_sats_bucket[5m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p95 Payment Fees", + "refId": "B" + } + ], + "title": "Lightning Payment Fees", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 1000 + }, + { + "color": "red", + "value": 5000 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "cdk_http_requests_total{status=\"500\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{endpoint}}", + "refId": "A" + } + ], + "title": "Failed Requests (500 Status)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "vis": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(cdk_auth_attempts_total[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Auth Attempts", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(cdk_auth_successes_total[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Auth Successes", + "refId": "B" + } + ], + "title": "Authentication Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "cdk_errors_total", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Total Errors", + "refId": "A" + } + ], + "title": "Total Errors", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "vis": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, rate(cdk_db_operation_duration_seconds_bucket[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p50 - {{operation}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, rate(cdk_db_operation_duration_seconds_bucket[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p95 - {{operation}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, rate(cdk_db_operation_duration_seconds_bucket[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p99 - {{operation}}", + "range": true, + "refId": "C" + } + ], + "title": "Database Operation Duration Percentiles", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "vis": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, rate(cdk_mint_operation_duration_seconds_bucket[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p50 - {{operation}} ({{status}})", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, rate(cdk_mint_operation_duration_seconds_bucket[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p95 - {{operation}} ({{status}})", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, rate(cdk_mint_operation_duration_seconds_bucket[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p99 - {{operation}} ({{status}})", + "range": true, + "refId": "C" + } + ], + "title": "Mint Operation Duration Percentiles", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "vis": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 72 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(cdk_db_operations_total[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "DB Operations Rate", + "range": true, + "refId": "A" + } + ], + "title": "Database Operations Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "vis": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 72 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(cdk_wallet_operations_total[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Wallet Operations Rate", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(cdk_lightning_payments_total[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Lightning Payments Rate", + "refId": "B" + } + ], + "title": "Wallet & Lightning Operations Rate", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 41, + "tags": [ + "cashu", + "cdk", + "mint" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "CDK Mint Dashboard", + "uid": "cdk-mint-dashboard", + "version": 8 +} \ No newline at end of file diff --git a/misc/provisioning/dashboards/dashboard.yaml b/misc/provisioning/dashboards/dashboard.yaml new file mode 100644 index 00000000..5cf7ca33 --- /dev/null +++ b/misc/provisioning/dashboards/dashboard.yaml @@ -0,0 +1,8 @@ +apiVersion: 1 +providers: + - name: 'default' + type: file + disableDeletion: false + updateIntervalSeconds: 10 + options: + path: /etc/grafana/provisioning/dashboards \ No newline at end of file diff --git a/misc/provisioning/datasources/datasource.yml b/misc/provisioning/datasources/datasource.yml new file mode 100644 index 00000000..bcbdc3cd --- /dev/null +++ b/misc/provisioning/datasources/datasource.yml @@ -0,0 +1,8 @@ +apiVersion: 1 + +datasources: + - name: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true \ No newline at end of file diff --git a/misc/provisioning/prometheus.yml b/misc/provisioning/prometheus.yml new file mode 100644 index 00000000..6cc83346 --- /dev/null +++ b/misc/provisioning/prometheus.yml @@ -0,0 +1,8 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: ['host.docker.internal:9000','mintd:9000']