Expose health and liveness metrics

ldk-server/src/util/metrics.rs

@@ -74,6 +74,20 @@ pub struct Metrics {
 	pub spendable_onchain_balance_sats: AtomicU64,
 	pub total_anchor_channels_reserve_sats: AtomicU64,
 	pub total_lightning_balance_sats: AtomicU64,
+	pub best_block_height: AtomicU64,
+	pub lightning_wallet_sync_timestamp_seconds: AtomicU64,
+	pub onchain_wallet_sync_timestamp_seconds: AtomicU64,
+	pub fee_rate_cache_update_timestamp_seconds: AtomicU64,
+	pub rgs_snapshot_timestamp_seconds: AtomicU64,
+	pub pathfinding_scores_sync_timestamp_seconds: AtomicU64,
+	pub node_announcement_broadcast_timestamp_seconds: AtomicU64,
+	pub connected_peers: AtomicI64,
+}
+
+/// `None` is stored as 0. Zero is not a real sample: ldk-node only records a
+/// timestamp after the Unix epoch, and `gather_metrics` omits a 0.
+fn store_optional_timestamp(atomic: &AtomicU64, timestamp: Option<u64>) {
+	atomic.store(timestamp.unwrap_or(0), Ordering::Relaxed);
 }
 
 impl Metrics {
@@ -91,12 +105,52 @@ impl Metrics {
 			spendable_onchain_balance_sats: AtomicU64::new(0),
 			total_anchor_channels_reserve_sats: AtomicU64::new(0),
 			total_lightning_balance_sats: AtomicU64::new(0),
+			best_block_height: AtomicU64::new(0),
+			lightning_wallet_sync_timestamp_seconds: AtomicU64::new(0),
+			onchain_wallet_sync_timestamp_seconds: AtomicU64::new(0),
+			fee_rate_cache_update_timestamp_seconds: AtomicU64::new(0),
+			rgs_snapshot_timestamp_seconds: AtomicU64::new(0),
+			pathfinding_scores_sync_timestamp_seconds: AtomicU64::new(0),
+			node_announcement_broadcast_timestamp_seconds: AtomicU64::new(0),
+			connected_peers: AtomicI64::new(0),
 		}
 	}
 
 	fn update_peer_count(&self, node: &Node) {
-		let total_peers_count = node.list_peers().len() as i64;
+		let list_peers = node.list_peers();
+		let total_peers_count = list_peers.len() as i64;
+		let connect_peers = list_peers.iter().filter(|e| e.is_connected).count() as i64;
 		self.total_peers_count.store(total_peers_count, Ordering::Relaxed);
+		self.connected_peers.store(connect_peers, Ordering::Relaxed);
+	}
+
+	fn update_node_status_metrics(&self, node: &Node) {
+		let status = node.status();
+		self.best_block_height.store(status.current_best_block.height as u64, Ordering::Relaxed);
+		store_optional_timestamp(
+			&self.lightning_wallet_sync_timestamp_seconds,
+			status.latest_lightning_wallet_sync_timestamp,
+		);
+		store_optional_timestamp(
+			&self.onchain_wallet_sync_timestamp_seconds,
+			status.latest_onchain_wallet_sync_timestamp,
+		);
+		store_optional_timestamp(
+			&self.fee_rate_cache_update_timestamp_seconds,
+			status.latest_fee_rate_cache_update_timestamp,
+		);
+		store_optional_timestamp(
+			&self.rgs_snapshot_timestamp_seconds,
+			status.latest_rgs_snapshot_timestamp,
+		);
+		store_optional_timestamp(
+			&self.pathfinding_scores_sync_timestamp_seconds,
+			status.latest_pathfinding_scores_sync_timestamp,
+		);
+		store_optional_timestamp(
+			&self.node_announcement_broadcast_timestamp_seconds,
+			status.latest_node_announcement_broadcast_timestamp,
+		);
 	}
 
 	fn store_payment_counts(&self, counts: PaymentCounts) {
@@ -127,6 +181,7 @@ impl Metrics {
 
 		self.update_peer_count(node);
 		self.update_all_balances(node);
+		self.update_node_status_metrics(node);
 	}
 
 	fn update_channel_visibility_counts(&self, channels: &[ChannelDetails]) {
@@ -163,6 +218,7 @@ impl Metrics {
 
 		self.update_peer_count(node);
 		self.update_all_balances(node);
+		self.update_node_status_metrics(node);
 	}
 
 	/// Gathers all metrics and formats them into the Prometheus text-based format.
@@ -184,14 +240,29 @@ impl Metrics {
 			let _ = writeln!(buffer, "{} {}", name, value);
 		}
 
+		fn format_timestamp_metric(buffer: &mut String, name: &str, help: &str, value: u64) {
+			if value == 0 {
+				return;
+			}
+			format_metric(buffer, name, help, "gauge", value);
+		}
+
 		format_metric(
 			&mut buffer,
 			"ldk_server_total_peers_count",
-			"Total number of peers",
+			"Number of known peers, including persisted peers that are disconnected.",
 			"gauge",
 			self.total_peers_count.load(Ordering::Relaxed),
 		);
 
+		format_metric(
+			&mut buffer,
+			"ldk_server_connected_peers",
+			"Number of peers with an active connection.",
+			"gauge",
+			self.connected_peers.load(Ordering::Relaxed),
+		);
+
 		format_metric(
 			&mut buffer,
 			"ldk_server_total_payments_count",
@@ -280,6 +351,56 @@ impl Metrics {
 			self.total_lightning_balance_sats.load(Ordering::Relaxed),
 		);
 
+		format_metric(
+			&mut buffer,
+			"ldk_server_best_block_height",
+			"Height of the best block the Lightning wallet is synced to.",
+			"gauge",
+			self.best_block_height.load(Ordering::Relaxed),
+		);
+
+		format_timestamp_metric(
+			&mut buffer,
+			"ldk_server_lightning_wallet_sync_timestamp_seconds",
+			"Unix time in seconds of the last successful Lightning wallet sync to the chain tip.",
+			self.lightning_wallet_sync_timestamp_seconds.load(Ordering::Relaxed),
+		);
+
+		format_timestamp_metric(
+			&mut buffer,
+			"ldk_server_onchain_wallet_sync_timestamp_seconds",
+			"Unix time in seconds of the last successful on-chain wallet sync to the chain tip.",
+			self.onchain_wallet_sync_timestamp_seconds.load(Ordering::Relaxed),
+		);
+
+		format_timestamp_metric(
+			&mut buffer,
+			"ldk_server_fee_rate_cache_update_timestamp_seconds",
+			"Unix time in seconds of the last successful fee-rate cache update.",
+			self.fee_rate_cache_update_timestamp_seconds.load(Ordering::Relaxed),
+		);
+
+		format_timestamp_metric(
+			&mut buffer,
+			"ldk_server_rgs_snapshot_timestamp_seconds",
+			"Unix time in seconds when the last applied rapid gossip sync snapshot was generated.",
+			self.rgs_snapshot_timestamp_seconds.load(Ordering::Relaxed),
+		);
+
+		format_timestamp_metric(
+			&mut buffer,
+			"ldk_server_pathfinding_scores_sync_timestamp_seconds",
+			"Unix time in seconds of the last successful merge of external pathfinding scores.",
+			self.pathfinding_scores_sync_timestamp_seconds.load(Ordering::Relaxed),
+		);
+
+		format_timestamp_metric(
+			&mut buffer,
+			"ldk_server_node_announcement_broadcast_timestamp_seconds",
+			"Unix time in seconds of the last node announcement broadcast.",
+			self.node_announcement_broadcast_timestamp_seconds.load(Ordering::Relaxed),
+		);
+
 		buffer
 	}
 }
@@ -306,6 +427,14 @@ mod tests {
 		assert!(result.contains("ldk_server_spendable_onchain_balance_sats 0"));
 		assert!(result.contains("ldk_server_total_anchor_channels_reserve_sats 0"));
 		assert!(result.contains("ldk_server_total_lightning_balance_sats 0"));
+		assert!(result.contains("ldk_server_best_block_height 0"));
+		assert!(!result.contains("ldk_server_lightning_wallet_sync_timestamp_seconds"));
+		assert!(!result.contains("ldk_server_onchain_wallet_sync_timestamp_seconds"));
+		assert!(!result.contains("ldk_server_fee_rate_cache_update_timestamp_seconds"));
+		assert!(!result.contains("ldk_server_rgs_snapshot_timestamp_seconds"));
+		assert!(!result.contains("ldk_server_pathfinding_scores_sync_timestamp_seconds"));
+		assert!(!result.contains("ldk_server_node_announcement_broadcast_timestamp_seconds"));
+		assert!(result.contains("ldk_server_connected_peers 0"));
 	}
 
 	#[test]
@@ -325,13 +454,41 @@ mod tests {
 		metrics.spendable_onchain_balance_sats.store(50_000, Ordering::Relaxed);
 		metrics.total_anchor_channels_reserve_sats.store(1_000, Ordering::Relaxed);
 		metrics.total_lightning_balance_sats.store(250_000, Ordering::Relaxed);
+		metrics.best_block_height.store(800_000, Ordering::Relaxed);
+		metrics.lightning_wallet_sync_timestamp_seconds.store(1_700_000_000, Ordering::Relaxed);
+		metrics.onchain_wallet_sync_timestamp_seconds.store(1_700_000_000, Ordering::Relaxed);
+		metrics.fee_rate_cache_update_timestamp_seconds.store(1_700_000_000, Ordering::Relaxed);
+		metrics.rgs_snapshot_timestamp_seconds.store(1_700_000_000, Ordering::Relaxed);
+		metrics.pathfinding_scores_sync_timestamp_seconds.store(1_700_000_000, Ordering::Relaxed);
+		metrics
+			.node_announcement_broadcast_timestamp_seconds
+			.store(1_700_000_000, Ordering::Relaxed);
+
+		metrics.connected_peers.store(2, Ordering::Relaxed);
 
 		let result = metrics.gather_metrics();
 
 		// Check that output contains updated values and correct Prometheus format
-		assert!(result.contains("# HELP ldk_server_total_peers_count Total number of peers"));
+		assert!(result.contains("# HELP ldk_server_total_peers_count Number of known peers, including persisted peers that are disconnected."));
+		assert!(result.contains(
+			"# HELP ldk_server_connected_peers Number of peers with an active connection."
+		));
 		assert!(result.contains("# TYPE ldk_server_total_peers_count gauge"));
+		assert!(result.contains("# TYPE ldk_server_connected_peers gauge"));
+		assert!(result.contains("# HELP ldk_server_best_block_height Height of the best block the Lightning wallet is synced to."));
+		assert!(result.contains("# TYPE ldk_server_best_block_height gauge"));
 		assert!(result.contains("ldk_server_total_peers_count 5"));
+		assert!(result.contains("ldk_server_connected_peers 2"));
+
+		assert!(result.contains("# TYPE ldk_server_lightning_wallet_sync_timestamp_seconds gauge"));
+		assert!(result.contains("# TYPE ldk_server_onchain_wallet_sync_timestamp_seconds gauge"));
+		assert!(result.contains("# TYPE ldk_server_fee_rate_cache_update_timestamp_seconds gauge"));
+		assert!(result.contains("# TYPE ldk_server_rgs_snapshot_timestamp_seconds gauge"));
+		assert!(
+			result.contains("# TYPE ldk_server_pathfinding_scores_sync_timestamp_seconds gauge")
+		);
+		assert!(result
+			.contains("# TYPE ldk_server_node_announcement_broadcast_timestamp_seconds gauge"));
 
 		assert!(result.contains("ldk_server_total_payments_count 10"));
 		assert!(result.contains("ldk_server_total_pending_payments_count 1"));
@@ -344,5 +501,14 @@ mod tests {
 		assert!(result.contains("ldk_server_spendable_onchain_balance_sats 50000"));
 		assert!(result.contains("ldk_server_total_anchor_channels_reserve_sats 1000"));
 		assert!(result.contains("ldk_server_total_lightning_balance_sats 250000"));
+		assert!(result.contains("ldk_server_best_block_height 800000"));
+		assert!(result.contains("ldk_server_lightning_wallet_sync_timestamp_seconds 1700000000"));
+		assert!(result.contains("ldk_server_onchain_wallet_sync_timestamp_seconds 1700000000"));
+		assert!(result.contains("ldk_server_fee_rate_cache_update_timestamp_seconds 1700000000"));
+		assert!(result.contains("ldk_server_rgs_snapshot_timestamp_seconds 1700000000"));
+		assert!(result.contains("ldk_server_pathfinding_scores_sync_timestamp_seconds 1700000000"));
+		assert!(
+			result.contains("ldk_server_node_announcement_broadcast_timestamp_seconds 1700000000")
+		);
 	}
 }