Update to LDK 0.0.123

Cargo.toml

@@ -8,13 +8,13 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-lightning = { version = "0.0.121", features = ["max_level_trace"] }
-lightning-block-sync = { version = "0.0.121", features = [ "rpc-client", "tokio" ] }
-lightning-invoice = { version = "0.29.0" }
-lightning-net-tokio = { version = "0.0.121" }
-lightning-persister = { version = "0.0.121" }
-lightning-background-processor = { version = "0.0.121", features = [ "futures" ] }
-lightning-rapid-gossip-sync = { version = "0.0.121" }
+lightning = { version = "0.0.123", features = ["max_level_trace"] }
+lightning-block-sync = { version = "0.0.123", features = [ "rpc-client", "tokio" ] }
+lightning-invoice = { version = "0.31.0" }
+lightning-net-tokio = { version = "0.0.123" }
+lightning-persister = { version = "0.0.123" }
+lightning-background-processor = { version = "0.0.123", features = [ "futures" ] }
+lightning-rapid-gossip-sync = { version = "0.0.123" }
 
 base64 = "0.13.0"
 bitcoin = "0.30.2"

src/bitcoind_client.rs

@@ -18,6 +18,7 @@ use bitcoin::{Network, OutPoint, TxOut, WPubkeyHash};
 use lightning::chain::chaininterface::{BroadcasterInterface, ConfirmationTarget, FeeEstimator};
 use lightning::events::bump_transaction::{Utxo, WalletSource};
 use lightning::log_error;
+use lightning::sign::ChangeDestinationSource;
 use lightning::util::logger::Logger;
 use lightning_block_sync::http::HttpEndpoint;
 use lightning_block_sync::rpc::RpcClient;
@@ -317,6 +318,14 @@ impl BroadcasterInterface for BitcoindClient {
 	}
 }
 
+impl ChangeDestinationSource for BitcoindClient {
+	fn get_change_destination_script(&self) -> Result<ScriptBuf, ()> {
+		tokio::task::block_in_place(move || {
+			Ok(self.handle.block_on(async move { self.get_new_address().await.script_pubkey() }))
+		})
+	}
+}
+
 impl WalletSource for BitcoindClient {
 	fn list_confirmed_utxos(&self) -> Result<Vec<Utxo>, ()> {
 		let utxos = tokio::task::block_in_place(move || {

src/cli.rs

@@ -286,7 +286,7 @@ pub(crate) fn poll_for_user_input(
 					);
 				},
 				"getoffer" => {
-					let offer_builder = channel_manager.create_offer_builder(String::new());
+					let offer_builder = channel_manager.create_offer_builder();
 					if let Err(e) = offer_builder {
 						println!("ERROR: Failed to initiate offer building: {:?}", e);
 						continue;
@@ -603,14 +603,14 @@ fn node_info(channel_manager: &Arc<ChannelManager>, peer_manager: &Arc<PeerManag
 	println!("\t\t num_usable_channels: {}", chans.iter().filter(|c| c.is_usable).count());
 	let local_balance_msat = chans.iter().map(|c| c.balance_msat).sum::<u64>();
 	println!("\t\t local_balance_msat: {}", local_balance_msat);
-	println!("\t\t num_peers: {}", peer_manager.get_peer_node_ids().len());
+	println!("\t\t num_peers: {}", peer_manager.list_peers().len());
 	println!("\t}},");
 }
 
 fn list_peers(peer_manager: Arc<PeerManager>) {
 	println!("\t{{");
-	for (pubkey, _) in peer_manager.get_peer_node_ids() {
-		println!("\t\t pubkey: {}", pubkey);
+	for peer_details in peer_manager.list_peers() {
+		println!("\t\t pubkey: {}", peer_details.counterparty_node_id);
 	}
 	println!("\t}},");
 }
@@ -701,8 +701,8 @@ fn list_payments(
 pub(crate) async fn connect_peer_if_necessary(
 	pubkey: PublicKey, peer_addr: SocketAddr, peer_manager: Arc<PeerManager>,
 ) -> Result<(), ()> {
-	for (node_pubkey, _) in peer_manager.get_peer_node_ids() {
-		if node_pubkey == pubkey {
+	for peer_details in peer_manager.list_peers() {
+		if peer_details.counterparty_node_id == pubkey {
 			return Ok(());
 		}
 	}
@@ -725,7 +725,7 @@ pub(crate) async fn do_connect_peer(
 					_ = &mut connection_closed_future => return Err(()),
 					_ = tokio::time::sleep(Duration::from_millis(10)) => {},
 				};
-				if peer_manager.get_peer_node_ids().iter().find(|(id, _)| *id == pubkey).is_some() {
+				if peer_manager.peer_by_node_id(&pubkey).is_some() {
 					return Ok(());
 				}
 			}
@@ -747,8 +747,7 @@ fn do_disconnect_peer(
 	}
 
 	//check the pubkey matches a valid connected peer
-	let peers = peer_manager.get_peer_node_ids();
-	if !peers.iter().any(|(pk, _)| &pubkey == pk) {
+	if peer_manager.peer_by_node_id(&pubkey).is_none() {
 		println!("Error: Could not find peer {}", pubkey);
 		return Err(());
 	}

src/main.rs

@@ -15,7 +15,7 @@ use bitcoin::BlockHash;
 use bitcoin_bech32::WitnessProgram;
 use disk::{INBOUND_PAYMENTS_FNAME, OUTBOUND_PAYMENTS_FNAME};
 use lightning::chain::{chainmonitor, ChannelMonitorUpdateStatus};
-use lightning::chain::{Filter, Watch};
+use lightning::chain::{BestBlock, Filter, Watch};
 use lightning::events::bump_transaction::{BumpTransactionEventHandler, Wallet};
 use lightning::events::{Event, PaymentFailureReason, PaymentPurpose};
 use lightning::ln::channelmanager::{self, RecentPaymentDetails};
@@ -30,10 +30,14 @@ use lightning::routing::gossip;
 use lightning::routing::gossip::{NodeId, P2PGossipSync};
 use lightning::routing::router::DefaultRouter;
 use lightning::routing::scoring::ProbabilisticScoringFeeParameters;
-use lightning::sign::{EntropySource, InMemorySigner, KeysManager, SpendableOutputDescriptor};
+use lightning::sign::{EntropySource, InMemorySigner, KeysManager};
 use lightning::util::config::UserConfig;
-use lightning::util::persist::{self, KVStore, MonitorUpdatingPersister};
+use lightning::util::persist::{
+	self, KVStore, MonitorUpdatingPersister, OUTPUT_SWEEPER_PERSISTENCE_KEY,
+	OUTPUT_SWEEPER_PERSISTENCE_PRIMARY_NAMESPACE, OUTPUT_SWEEPER_PERSISTENCE_SECONDARY_NAMESPACE,
+};
 use lightning::util::ser::{Readable, ReadableArgs, Writeable, Writer};
+use lightning::util::sweep as ldk_sweep;
 use lightning::{chain, impl_writeable_tlv_based, impl_writeable_tlv_based_enum};
 use lightning_background_processor::{process_events_async, GossipSync};
 use lightning_block_sync::init;
@@ -172,13 +176,26 @@ pub(crate) type BumpTxEventHandler = BumpTransactionEventHandler<
 	Arc<FilesystemLogger>,
 >;
 
+pub(crate) type OutputSweeper = ldk_sweep::OutputSweeper<
+	Arc<BitcoindClient>,
+	Arc<BitcoindClient>,
+	Arc<BitcoindClient>,
+	Arc<dyn Filter + Send + Sync>,
+	Arc<FilesystemStore>,
+	Arc<FilesystemLogger>,
+	Arc<KeysManager>,
+>;
+
+// Needed due to rust-lang/rust#63033.
+struct OutputSweeperWrapper(Arc<OutputSweeper>);
+
 async fn handle_ldk_events(
 	channel_manager: Arc<ChannelManager>, bitcoind_client: &BitcoindClient,
 	network_graph: &NetworkGraph, keys_manager: &KeysManager,
 	bump_tx_event_handler: &BumpTxEventHandler, peer_manager: Arc<PeerManager>,
 	inbound_payments: Arc<Mutex<InboundPaymentInfoStorage>>,
 	outbound_payments: Arc<Mutex<OutboundPaymentInfoStorage>>, fs_store: Arc<FilesystemStore>,
-	network: Network, event: Event,
+	output_sweeper: OutputSweeperWrapper, network: Network, event: Event,
 ) {
 	match event {
 		Event::FundingGenerationReady {
@@ -247,7 +264,9 @@ async fn handle_ldk_events(
 			print!("> ");
 			io::stdout().flush().unwrap();
 			let payment_preimage = match purpose {
-				PaymentPurpose::InvoicePayment { payment_preimage, .. } => payment_preimage,
+				PaymentPurpose::Bolt11InvoicePayment { payment_preimage, .. } => payment_preimage,
+				PaymentPurpose::Bolt12OfferPayment { payment_preimage, .. } => payment_preimage,
+				PaymentPurpose::Bolt12RefundPayment { payment_preimage, .. } => payment_preimage,
 				PaymentPurpose::SpontaneousPayment(preimage) => Some(preimage),
 			};
 			channel_manager.claim_funds(payment_preimage.unwrap());
@@ -267,9 +286,15 @@ async fn handle_ldk_events(
 			print!("> ");
 			io::stdout().flush().unwrap();
 			let (payment_preimage, payment_secret) = match purpose {
-				PaymentPurpose::InvoicePayment { payment_preimage, payment_secret, .. } => {
+				PaymentPurpose::Bolt11InvoicePayment {
+					payment_preimage, payment_secret, ..
+				} => (payment_preimage, Some(payment_secret)),
+				PaymentPurpose::Bolt12OfferPayment { payment_preimage, payment_secret, .. } => {
 					(payment_preimage, Some(payment_secret))
 				},
+				PaymentPurpose::Bolt12RefundPayment {
+					payment_preimage, payment_secret, ..
+				} => (payment_preimage, Some(payment_secret)),
 				PaymentPurpose::SpontaneousPayment(preimage) => (Some(preimage), None),
 			};
 			let mut inbound = inbound_payments.lock().unwrap();
@@ -381,9 +406,12 @@ async fn handle_ldk_events(
 		Event::PaymentForwarded {
 			prev_channel_id,
 			next_channel_id,
-			fee_earned_msat,
+			total_fee_earned_msat,
 			claim_from_onchain_tx,
 			outbound_amount_forwarded_msat,
+			skimmed_fee_msat: _,
+			prev_user_channel_id: _,
+			next_user_channel_id: _,
 		} => {
 			let read_only_network_graph = network_graph.read_only();
 			let nodes = read_only_network_graph.nodes();
@@ -426,7 +454,7 @@ async fn handle_ldk_events(
 			} else {
 				"?".to_string()
 			};
-			if let Some(fee_earned) = fee_earned_msat {
+			if let Some(fee_earned) = total_fee_earned_msat {
 				println!(
 					"\nEVENT: Forwarded payment for {} msat{}{}, earning {} msat {}",
 					amt_args, from_prev_str, to_next_str, fee_earned, from_onchain_str
@@ -450,22 +478,8 @@ async fn handle_ldk_events(
 				forwarding_channel_manager.process_pending_htlc_forwards();
 			});
 		},
-		Event::SpendableOutputs { outputs, channel_id: _ } => {
-			// SpendableOutputDescriptors, of which outputs is a vec of, are critical to keep track
-			// of! While a `StaticOutput` descriptor is just an output to a static, well-known key,
-			// other descriptors are not currently ever regenerated for you by LDK. Once we return
-			// from this method, the descriptor will be gone, and you may lose track of some funds.
-			//
-			// Here we simply persist them to disk, with a background task running which will try
-			// to spend them regularly (possibly duplicatively/RBF'ing them). These can just be
-			// treated as normal funds where possible - they are only spendable by us and there is
-			// no rush to claim them.
-			for output in outputs {
-				let key = hex_utils::hex_str(&keys_manager.get_secure_random_bytes());
-				// Note that if the type here changes our read code needs to change as well.
-				let output: SpendableOutputDescriptor = output;
-				fs_store.write(PENDING_SPENDABLE_OUTPUT_DIR, "", &key, &output.encode()).unwrap();
-			}
+		Event::SpendableOutputs { outputs, channel_id } => {
+			output_sweeper.0.track_spendable_outputs(outputs, channel_id, false, None).unwrap();
 		},
 		Event::ChannelPending { channel_id, counterparty_node_id, .. } => {
 			println!(
@@ -677,7 +691,7 @@ async fn start_ldk() {
 	let router = Arc::new(DefaultRouter::new(
 		network_graph.clone(),
 		logger.clone(),
-		keys_manager.get_secure_random_bytes(),
+		keys_manager.clone(),
 		scorer.clone(),
 		scoring_fee_params,
 	));
@@ -712,7 +726,7 @@ async fn start_ldk() {
 			restarting_node = false;
 
 			let polled_best_block = polled_chain_tip.to_best_block();
-			let polled_best_block_hash = polled_best_block.block_hash();
+			let polled_best_block_hash = polled_best_block.block_hash;
 			let chain_params =
 				ChainParameters { network: args.network, best_block: polled_best_block };
 			let fresh_channel_manager = channelmanager::ChannelManager::new(
@@ -732,14 +746,50 @@ async fn start_ldk() {
 		}
 	};
 
-	// Step 12: Sync ChannelMonitors and ChannelManager to chain tip
+	// Step 12: Initialize the OutputSweeper.
+	let (sweeper_best_block, output_sweeper) = match fs_store.read(
+		OUTPUT_SWEEPER_PERSISTENCE_PRIMARY_NAMESPACE,
+		OUTPUT_SWEEPER_PERSISTENCE_SECONDARY_NAMESPACE,
+		OUTPUT_SWEEPER_PERSISTENCE_KEY,
+	) {
+		Err(e) if e.kind() == io::ErrorKind::NotFound => {
+			let sweeper = OutputSweeper::new(
+				channel_manager.current_best_block(),
+				broadcaster.clone(),
+				fee_estimator.clone(),
+				None,
+				keys_manager.clone(),
+				bitcoind_client.clone(),
+				fs_store.clone(),
+				logger.clone(),
+			);
+			(channel_manager.current_best_block(), sweeper)
+		},
+		Ok(mut bytes) => {
+			let read_args = (
+				broadcaster.clone(),
+				fee_estimator.clone(),
+				None,
+				keys_manager.clone(),
+				bitcoind_client.clone(),
+				fs_store.clone(),
+				logger.clone(),
+			);
+			let mut reader = io::Cursor::new(&mut bytes);
+			<(BestBlock, OutputSweeper)>::read(&mut reader, read_args)
+				.expect("Failed to deserialize OutputSweeper")
+		},
+		Err(e) => panic!("Failed to read OutputSweeper with {}", e),
+	};
+
+	// Step 13: Sync ChannelMonitors, ChannelManager and OutputSweeper to chain tip
 	let mut chain_listener_channel_monitors = Vec::new();
 	let mut cache = UnboundedCache::new();
 	let chain_tip = if restarting_node {
-		let mut chain_listeners = vec![(
-			channel_manager_blockhash,
-			&channel_manager as &(dyn chain::Listen + Send + Sync),
-		)];
+		let mut chain_listeners = vec![
+			(channel_manager_blockhash, &channel_manager as &(dyn chain::Listen + Send + Sync)),
+			(sweeper_best_block.block_hash, &output_sweeper as &(dyn chain::Listen + Send + Sync)),
+		];
 
 		for (blockhash, channel_monitor) in channelmonitors.drain(..) {
 			let outpoint = channel_monitor.get_funding_txo().0;
@@ -769,7 +819,7 @@ async fn start_ldk() {
 		polled_chain_tip
 	};
 
-	// Step 13: Give ChannelMonitors to ChainMonitor
+	// Step 14: Give ChannelMonitors to ChainMonitor
 	for item in chain_listener_channel_monitors.drain(..) {
 		let channel_monitor = item.1 .0;
 		let funding_outpoint = item.2;
@@ -779,17 +829,18 @@ async fn start_ldk() {
 		);
 	}
 
-	// Step 14: Optional: Initialize the P2PGossipSync
+	// Step 15: Optional: Initialize the P2PGossipSync
 	let gossip_sync =
 		Arc::new(P2PGossipSync::new(Arc::clone(&network_graph), None, Arc::clone(&logger)));
 
-	// Step 15: Initialize the PeerManager
+	// Step 16: Initialize the PeerManager
 	let channel_manager: Arc<ChannelManager> = Arc::new(channel_manager);
 	let onion_messenger: Arc<OnionMessenger> = Arc::new(OnionMessenger::new(
 		Arc::clone(&keys_manager),
 		Arc::clone(&keys_manager),
 		Arc::clone(&logger),
-		Arc::new(DefaultMessageRouter::new(Arc::clone(&network_graph))),
+		Arc::clone(&channel_manager),
+		Arc::new(DefaultMessageRouter::new(Arc::clone(&network_graph), Arc::clone(&keys_manager))),
 		Arc::clone(&channel_manager),
 		IgnoringMessageHandler {},
 	));
@@ -820,7 +871,7 @@ async fn start_ldk() {
 	gossip_sync.add_utxo_lookup(Some(utxo_lookup));
 
 	// ## Running LDK
-	// Step 16: Initialize networking
+	// Step 17: Initialize networking
 
 	let peer_manager_connection_handler = peer_manager.clone();
 	let listening_port = args.ldk_peer_listening_port;
@@ -846,14 +897,17 @@ async fn start_ldk() {
 		}
 	});
 
-	// Step 17: Connect and Disconnect Blocks
+	// Step 18: Connect and Disconnect Blocks
+	let output_sweeper: Arc<OutputSweeper> = Arc::new(output_sweeper);
 	let channel_manager_listener = channel_manager.clone();
 	let chain_monitor_listener = chain_monitor.clone();
+	let output_sweeper_listener = output_sweeper.clone();
 	let bitcoind_block_source = bitcoind_client.clone();
 	let network = args.network;
 	tokio::spawn(async move {
 		let chain_poller = poll::ChainPoller::new(bitcoind_block_source.as_ref(), network);
-		let chain_listener = (chain_monitor_listener, channel_manager_listener);
+		let chain_listener =
+			(chain_monitor_listener, &(channel_manager_listener, output_sweeper_listener));
 		let mut spv_client = SpvClient::new(chain_tip, chain_poller, &mut cache, &chain_listener);
 		loop {
 			spv_client.poll_best_tip().await.unwrap();
@@ -892,7 +946,7 @@ async fn start_ldk() {
 		.write("", "", OUTBOUND_PAYMENTS_FNAME, &outbound_payments.lock().unwrap().encode())
 		.unwrap();
 
-	// Step 18: Handle LDK Events
+	// Step 19: Handle LDK Events
 	let channel_manager_event_listener = Arc::clone(&channel_manager);
 	let bitcoind_client_event_listener = Arc::clone(&bitcoind_client);
 	let network_graph_event_listener = Arc::clone(&network_graph);
@@ -901,6 +955,7 @@ async fn start_ldk() {
 	let outbound_payments_event_listener = Arc::clone(&outbound_payments);
 	let fs_store_event_listener = Arc::clone(&fs_store);
 	let peer_manager_event_listener = Arc::clone(&peer_manager);
+	let output_sweeper_event_listener = Arc::clone(&output_sweeper);
 	let network = args.network;
 	let event_handler = move |event: Event| {
 		let channel_manager_event_listener = Arc::clone(&channel_manager_event_listener);
@@ -912,6 +967,7 @@ async fn start_ldk() {
 		let outbound_payments_event_listener = Arc::clone(&outbound_payments_event_listener);
 		let fs_store_event_listener = Arc::clone(&fs_store_event_listener);
 		let peer_manager_event_listener = Arc::clone(&peer_manager_event_listener);
+		let output_sweeper_event_listener = Arc::clone(&output_sweeper_event_listener);
 		async move {
 			handle_ldk_events(
 				channel_manager_event_listener,
@@ -923,17 +979,18 @@ async fn start_ldk() {
 				inbound_payments_event_listener,
 				outbound_payments_event_listener,
 				fs_store_event_listener,
+				OutputSweeperWrapper(output_sweeper_event_listener),
 				network,
 				event,
 			)
 			.await;
 		}
 	};
 
-	// Step 19: Persist ChannelManager and NetworkGraph
+	// Step 20: Persist ChannelManager and NetworkGraph
 	let persister = Arc::new(FilesystemStore::new(ldk_data_dir.clone().into()));
 
-	// Step 20: Background Processing
+	// Step 21: Background Processing
 	let (bp_exit, bp_exit_check) = tokio::sync::watch::channel(());
 	let mut background_processor = tokio::spawn(process_events_async(
 		Arc::clone(&persister),
@@ -969,12 +1026,11 @@ async fn start_ldk() {
 			interval.tick().await;
 			match disk::read_channel_peer_data(Path::new(&peer_data_path)) {
 				Ok(info) => {
-					let peers = connect_pm.get_peer_node_ids();
 					for node_id in connect_cm
 						.list_channels()
 						.iter()
 						.map(|chan| chan.counterparty.node_id)
-						.filter(|id| !peers.iter().any(|(pk, _)| id == pk))
+						.filter(|id| connect_pm.peer_by_node_id(id).is_none())
 					{
 						if stop_connect.load(Ordering::Acquire) {
 							return;
@@ -1022,6 +1078,7 @@ async fn start_ldk() {
 		}
 	});
 
+	// TODO: remove this, since the new `OutputSweeper` was added in LDK v0.0.123.
 	tokio::spawn(sweep::periodic_sweep(
 		ldk_data_dir.clone(),
 		Arc::clone(&keys_manager),

src/sweep.rs

@@ -5,7 +5,7 @@ use std::time::Duration;
 use std::{fs, io};
 
 use lightning::chain::chaininterface::{BroadcasterInterface, ConfirmationTarget, FeeEstimator};
-use lightning::sign::{EntropySource, KeysManager, SpendableOutputDescriptor};
+use lightning::sign::{EntropySource, KeysManager, OutputSpender, SpendableOutputDescriptor};
 use lightning::util::logger::Logger;
 use lightning::util::persist::KVStore;
 use lightning::util::ser::{Readable, WithoutLength, Writeable};
@@ -118,7 +118,7 @@ pub(crate) async fn periodic_sweep(
 				// e.g. high-latency mix networks and some CoinJoin implementations, have
 				// better privacy.
 				// Logic copied from core: https://github.com/bitcoin/bitcoin/blob/1d4846a8443be901b8a5deb0e357481af22838d0/src/wallet/spend.cpp#L936
-				let mut cur_height = channel_manager.current_best_block().height();
+				let mut cur_height = channel_manager.current_best_block().height;
 
 				// 10% of the time
 				if thread_rng().gen_range(0, 10) == 0 {