From 573168c92335873010f9d4ed6535727f0f8792b8 Mon Sep 17 00:00:00 2001 From: Fernando Ledesma Date: Thu, 22 Oct 2026 10:27:56 -0500 Subject: [PATCH 1/2] Expose sync time and best-block height as metrics. Poll NodeStatus on the existing metrics interval and publish the Lightning wallet's best-block height plus the six sync and broadcast timestamps. Timestamps are absolute Unix seconds, so an alert can use time() - metric. A timestamp that has never been set is omitted. Height is always present. --- ldk-server/src/util/metrics.rs | 145 +++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) diff --git a/ldk-server/src/util/metrics.rs b/ldk-server/src/util/metrics.rs index a6e3d736..20ce7df0 100644 --- a/ldk-server/src/util/metrics.rs +++ b/ldk-server/src/util/metrics.rs @@ -74,6 +74,19 @@ pub struct Metrics { pub spendable_onchain_balance_sats: AtomicU64, pub total_anchor_channels_reserve_sats: AtomicU64, pub total_lightning_balance_sats: AtomicU64, + pub best_block_height: AtomicU64, + pub lightning_wallet_sync_timestamp_seconds: AtomicU64, + pub onchain_wallet_sync_timestamp_seconds: AtomicU64, + pub fee_rate_cache_update_timestamp_seconds: AtomicU64, + pub rgs_snapshot_timestamp_seconds: AtomicU64, + pub pathfinding_scores_sync_timestamp_seconds: AtomicU64, + pub node_announcement_broadcast_timestamp_seconds: AtomicU64, +} + +/// `None` is stored as 0. Zero is not a real sample: ldk-node only records a +/// timestamp after the Unix epoch, and `gather_metrics` omits a 0. +fn store_optional_timestamp(atomic: &AtomicU64, timestamp: Option) { + atomic.store(timestamp.unwrap_or(0), Ordering::Relaxed); } impl Metrics { @@ -91,6 +104,13 @@ impl Metrics { spendable_onchain_balance_sats: AtomicU64::new(0), total_anchor_channels_reserve_sats: AtomicU64::new(0), total_lightning_balance_sats: AtomicU64::new(0), + best_block_height: AtomicU64::new(0), + lightning_wallet_sync_timestamp_seconds: AtomicU64::new(0), + onchain_wallet_sync_timestamp_seconds: AtomicU64::new(0), + fee_rate_cache_update_timestamp_seconds: AtomicU64::new(0), + rgs_snapshot_timestamp_seconds: AtomicU64::new(0), + pathfinding_scores_sync_timestamp_seconds: AtomicU64::new(0), + node_announcement_broadcast_timestamp_seconds: AtomicU64::new(0), } } @@ -99,6 +119,35 @@ impl Metrics { self.total_peers_count.store(total_peers_count, Ordering::Relaxed); } + fn update_node_status_metrics(&self, node: &Node) { + let status = node.status(); + self.best_block_height.store(status.current_best_block.height as u64, Ordering::Relaxed); + store_optional_timestamp( + &self.lightning_wallet_sync_timestamp_seconds, + status.latest_lightning_wallet_sync_timestamp, + ); + store_optional_timestamp( + &self.onchain_wallet_sync_timestamp_seconds, + status.latest_onchain_wallet_sync_timestamp, + ); + store_optional_timestamp( + &self.fee_rate_cache_update_timestamp_seconds, + status.latest_fee_rate_cache_update_timestamp, + ); + store_optional_timestamp( + &self.rgs_snapshot_timestamp_seconds, + status.latest_rgs_snapshot_timestamp, + ); + store_optional_timestamp( + &self.pathfinding_scores_sync_timestamp_seconds, + status.latest_pathfinding_scores_sync_timestamp, + ); + store_optional_timestamp( + &self.node_announcement_broadcast_timestamp_seconds, + status.latest_node_announcement_broadcast_timestamp, + ); + } + fn store_payment_counts(&self, counts: PaymentCounts) { self.total_payments_count.store(counts.total, Ordering::Relaxed); self.total_successful_payments_count.store(counts.successful, Ordering::Relaxed); @@ -127,6 +176,7 @@ impl Metrics { self.update_peer_count(node); self.update_all_balances(node); + self.update_node_status_metrics(node); } fn update_channel_visibility_counts(&self, channels: &[ChannelDetails]) { @@ -163,6 +213,7 @@ impl Metrics { self.update_peer_count(node); self.update_all_balances(node); + self.update_node_status_metrics(node); } /// Gathers all metrics and formats them into the Prometheus text-based format. @@ -184,6 +235,13 @@ impl Metrics { let _ = writeln!(buffer, "{} {}", name, value); } + fn format_timestamp_metric(buffer: &mut String, name: &str, help: &str, value: u64) { + if value == 0 { + return; + } + format_metric(buffer, name, help, "gauge", value); + } + format_metric( &mut buffer, "ldk_server_total_peers_count", @@ -280,6 +338,56 @@ impl Metrics { self.total_lightning_balance_sats.load(Ordering::Relaxed), ); + format_metric( + &mut buffer, + "ldk_server_best_block_height", + "Height of the best block the Lightning wallet is synced to.", + "gauge", + self.best_block_height.load(Ordering::Relaxed), + ); + + format_timestamp_metric( + &mut buffer, + "ldk_server_lightning_wallet_sync_timestamp_seconds", + "Unix time in seconds of the last successful Lightning wallet sync to the chain tip.", + self.lightning_wallet_sync_timestamp_seconds.load(Ordering::Relaxed), + ); + + format_timestamp_metric( + &mut buffer, + "ldk_server_onchain_wallet_sync_timestamp_seconds", + "Unix time in seconds of the last successful on-chain wallet sync to the chain tip.", + self.onchain_wallet_sync_timestamp_seconds.load(Ordering::Relaxed), + ); + + format_timestamp_metric( + &mut buffer, + "ldk_server_fee_rate_cache_update_timestamp_seconds", + "Unix time in seconds of the last successful fee-rate cache update.", + self.fee_rate_cache_update_timestamp_seconds.load(Ordering::Relaxed), + ); + + format_timestamp_metric( + &mut buffer, + "ldk_server_rgs_snapshot_timestamp_seconds", + "Unix time in seconds when the last applied rapid gossip sync snapshot was generated.", + self.rgs_snapshot_timestamp_seconds.load(Ordering::Relaxed), + ); + + format_timestamp_metric( + &mut buffer, + "ldk_server_pathfinding_scores_sync_timestamp_seconds", + "Unix time in seconds of the last successful merge of external pathfinding scores.", + self.pathfinding_scores_sync_timestamp_seconds.load(Ordering::Relaxed), + ); + + format_timestamp_metric( + &mut buffer, + "ldk_server_node_announcement_broadcast_timestamp_seconds", + "Unix time in seconds of the last node announcement broadcast.", + self.node_announcement_broadcast_timestamp_seconds.load(Ordering::Relaxed), + ); + buffer } } @@ -306,6 +414,13 @@ mod tests { assert!(result.contains("ldk_server_spendable_onchain_balance_sats 0")); assert!(result.contains("ldk_server_total_anchor_channels_reserve_sats 0")); assert!(result.contains("ldk_server_total_lightning_balance_sats 0")); + assert!(result.contains("ldk_server_best_block_height 0")); + assert!(!result.contains("ldk_server_lightning_wallet_sync_timestamp_seconds")); + assert!(!result.contains("ldk_server_onchain_wallet_sync_timestamp_seconds")); + assert!(!result.contains("ldk_server_fee_rate_cache_update_timestamp_seconds")); + assert!(!result.contains("ldk_server_rgs_snapshot_timestamp_seconds")); + assert!(!result.contains("ldk_server_pathfinding_scores_sync_timestamp_seconds")); + assert!(!result.contains("ldk_server_node_announcement_broadcast_timestamp_seconds")); } #[test] @@ -325,14 +440,35 @@ mod tests { metrics.spendable_onchain_balance_sats.store(50_000, Ordering::Relaxed); metrics.total_anchor_channels_reserve_sats.store(1_000, Ordering::Relaxed); metrics.total_lightning_balance_sats.store(250_000, Ordering::Relaxed); + metrics.best_block_height.store(800_000, Ordering::Relaxed); + metrics.lightning_wallet_sync_timestamp_seconds.store(1_700_000_000, Ordering::Relaxed); + metrics.onchain_wallet_sync_timestamp_seconds.store(1_700_000_000, Ordering::Relaxed); + metrics.fee_rate_cache_update_timestamp_seconds.store(1_700_000_000, Ordering::Relaxed); + metrics.rgs_snapshot_timestamp_seconds.store(1_700_000_000, Ordering::Relaxed); + metrics.pathfinding_scores_sync_timestamp_seconds.store(1_700_000_000, Ordering::Relaxed); + metrics + .node_announcement_broadcast_timestamp_seconds + .store(1_700_000_000, Ordering::Relaxed); let result = metrics.gather_metrics(); // Check that output contains updated values and correct Prometheus format assert!(result.contains("# HELP ldk_server_total_peers_count Total number of peers")); assert!(result.contains("# TYPE ldk_server_total_peers_count gauge")); + assert!(result.contains("# HELP ldk_server_best_block_height Height of the best block the Lightning wallet is synced to.")); + assert!(result.contains("# TYPE ldk_server_best_block_height gauge")); assert!(result.contains("ldk_server_total_peers_count 5")); + assert!(result.contains("# TYPE ldk_server_lightning_wallet_sync_timestamp_seconds gauge")); + assert!(result.contains("# TYPE ldk_server_onchain_wallet_sync_timestamp_seconds gauge")); + assert!(result.contains("# TYPE ldk_server_fee_rate_cache_update_timestamp_seconds gauge")); + assert!(result.contains("# TYPE ldk_server_rgs_snapshot_timestamp_seconds gauge")); + assert!( + result.contains("# TYPE ldk_server_pathfinding_scores_sync_timestamp_seconds gauge") + ); + assert!(result + .contains("# TYPE ldk_server_node_announcement_broadcast_timestamp_seconds gauge")); + assert!(result.contains("ldk_server_total_payments_count 10")); assert!(result.contains("ldk_server_total_pending_payments_count 1")); assert!(result.contains("ldk_server_total_successful_payments_count 8")); @@ -344,5 +480,14 @@ mod tests { assert!(result.contains("ldk_server_spendable_onchain_balance_sats 50000")); assert!(result.contains("ldk_server_total_anchor_channels_reserve_sats 1000")); assert!(result.contains("ldk_server_total_lightning_balance_sats 250000")); + assert!(result.contains("ldk_server_best_block_height 800000")); + assert!(result.contains("ldk_server_lightning_wallet_sync_timestamp_seconds 1700000000")); + assert!(result.contains("ldk_server_onchain_wallet_sync_timestamp_seconds 1700000000")); + assert!(result.contains("ldk_server_fee_rate_cache_update_timestamp_seconds 1700000000")); + assert!(result.contains("ldk_server_rgs_snapshot_timestamp_seconds 1700000000")); + assert!(result.contains("ldk_server_pathfinding_scores_sync_timestamp_seconds 1700000000")); + assert!( + result.contains("ldk_server_node_announcement_broadcast_timestamp_seconds 1700000000") + ); } } From 1291866701c6ce114d10ffe125eb40650b6fb77f Mon Sep 17 00:00:00 2001 From: Fernando Ledesma Date: Thu, 22 Oct 2026 11:12:14 -0500 Subject: [PATCH 2/2] Report connected peers separately from known peers ldk_server_total_peers_count stays list_peers().len(), which includes persisted peers that are disconnected. Add ldk_server_connected_peers for peers with an active connection, and describe the old series as known peers. --- ldk-server/src/util/metrics.rs | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/ldk-server/src/util/metrics.rs b/ldk-server/src/util/metrics.rs index 20ce7df0..a3637a18 100644 --- a/ldk-server/src/util/metrics.rs +++ b/ldk-server/src/util/metrics.rs @@ -81,6 +81,7 @@ pub struct Metrics { pub rgs_snapshot_timestamp_seconds: AtomicU64, pub pathfinding_scores_sync_timestamp_seconds: AtomicU64, pub node_announcement_broadcast_timestamp_seconds: AtomicU64, + pub connected_peers: AtomicI64, } /// `None` is stored as 0. Zero is not a real sample: ldk-node only records a @@ -111,12 +112,16 @@ impl Metrics { rgs_snapshot_timestamp_seconds: AtomicU64::new(0), pathfinding_scores_sync_timestamp_seconds: AtomicU64::new(0), node_announcement_broadcast_timestamp_seconds: AtomicU64::new(0), + connected_peers: AtomicI64::new(0), } } fn update_peer_count(&self, node: &Node) { - let total_peers_count = node.list_peers().len() as i64; + let list_peers = node.list_peers(); + let total_peers_count = list_peers.len() as i64; + let connect_peers = list_peers.iter().filter(|e| e.is_connected).count() as i64; self.total_peers_count.store(total_peers_count, Ordering::Relaxed); + self.connected_peers.store(connect_peers, Ordering::Relaxed); } fn update_node_status_metrics(&self, node: &Node) { @@ -245,11 +250,19 @@ impl Metrics { format_metric( &mut buffer, "ldk_server_total_peers_count", - "Total number of peers", + "Number of known peers, including persisted peers that are disconnected.", "gauge", self.total_peers_count.load(Ordering::Relaxed), ); + format_metric( + &mut buffer, + "ldk_server_connected_peers", + "Number of peers with an active connection.", + "gauge", + self.connected_peers.load(Ordering::Relaxed), + ); + format_metric( &mut buffer, "ldk_server_total_payments_count", @@ -421,6 +434,7 @@ mod tests { assert!(!result.contains("ldk_server_rgs_snapshot_timestamp_seconds")); assert!(!result.contains("ldk_server_pathfinding_scores_sync_timestamp_seconds")); assert!(!result.contains("ldk_server_node_announcement_broadcast_timestamp_seconds")); + assert!(result.contains("ldk_server_connected_peers 0")); } #[test] @@ -450,14 +464,21 @@ mod tests { .node_announcement_broadcast_timestamp_seconds .store(1_700_000_000, Ordering::Relaxed); + metrics.connected_peers.store(2, Ordering::Relaxed); + let result = metrics.gather_metrics(); // Check that output contains updated values and correct Prometheus format - assert!(result.contains("# HELP ldk_server_total_peers_count Total number of peers")); + assert!(result.contains("# HELP ldk_server_total_peers_count Number of known peers, including persisted peers that are disconnected.")); + assert!(result.contains( + "# HELP ldk_server_connected_peers Number of peers with an active connection." + )); assert!(result.contains("# TYPE ldk_server_total_peers_count gauge")); + assert!(result.contains("# TYPE ldk_server_connected_peers gauge")); assert!(result.contains("# HELP ldk_server_best_block_height Height of the best block the Lightning wallet is synced to.")); assert!(result.contains("# TYPE ldk_server_best_block_height gauge")); assert!(result.contains("ldk_server_total_peers_count 5")); + assert!(result.contains("ldk_server_connected_peers 2")); assert!(result.contains("# TYPE ldk_server_lightning_wallet_sync_timestamp_seconds gauge")); assert!(result.contains("# TYPE ldk_server_onchain_wallet_sync_timestamp_seconds gauge"));