[bug]: Multiple asset channels have been force-closed

[shuishihang]

litd -- v0.15.0-alpha
tapd -- v0.6.0
lnd -- v0.19.1-beta

My node has established one BTC channel + one asset channel with multiple peers. Every 10 minutes, I use Keysend to send an asset payment to all peer asset channels. Since each asset transfer also includes 354 sats, the peers respond by sending back 354 sats to me via Keysend.

However, I don’t understand why some of these asset channels suddenly entered a force-closed state. Out of 200 asset channels, 7 have been force-closed, and they remain stuck in an unresolved state (unable to fully close).

  "pending_force_closing_channels": [
        {
            "channel": {
                "remote_node_pub": "02094b89eca31f623af2d4fe2bf0fe52828c9816bf23573bb94f3938ce44c5a621",
                "channel_point": "02e194f26ed80c39363f3bf889271525ea98f37e1aba3656596c0a7d43d40c09:0",
                "capacity": "20000",
                "local_balance": "8855",
                "remote_balance": "10000",
                "local_chan_reserve_sat": "0",
                "remote_chan_reserve_sat": "0",
                "initiator": "INITIATOR_LOCAL",
                "commitment_type": "SIMPLE_TAPROOT_OVERLAY",
                "num_forwarding_packages": "0",
                "chan_status_flags": "",
                "private": true,
                "memo": "",
                "custom_channel_data": {
                    "funding_assets": [
                        {
                            "version": 1,
                            "asset_genesis": {
                                "genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
                                "name": "LNT",
                                "meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
                                "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
                            },
                            "amount": 10000000000,
                            "script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
                            "decimal_display": 6
                        }
                    ],
                    "local_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 9837900184
                        }
                    ],
                    "remote_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 162099816
                        }
                    ],
                    "outgoing_htlcs": [],
                    "incoming_htlcs": [],
                    "capacity": 10000000000,
                    "local_balance": 9837900184,
                    "remote_balance": 162099816,
                    "outgoing_htlc_balance": 0,
                    "incoming_htlc_balance": 0
                }
            },
            "closing_txid": "937ab4d5c847eb7bde242e16a64f0a6d794dd3dba976b1b327a06f77b2feacc6",
            "limbo_balance": "8855",
            "maturity_height": 904956,
            "blocks_til_maturity": -55,
            "recovered_balance": "330",
            "pending_htlcs": [],
            "anchor": "RECOVERED"
        },
        {
            "channel": {
                "remote_node_pub": "020686ae2a3a54897ef8098ff3ac0541cd09b2ce6296dc8d5c8b757d5ce14d6d96",
                "channel_point": "afc7179f513ed1651d0a950a184a05c68fc2b080b777d021418d7a9071dc4964:0",
                "capacity": "20000",
                "local_balance": "8088",
                "remote_balance": "10708",
                "local_chan_reserve_sat": "0",
                "remote_chan_reserve_sat": "0",
                "initiator": "INITIATOR_LOCAL",
                "commitment_type": "SIMPLE_TAPROOT_OVERLAY",
                "num_forwarding_packages": "0",
                "chan_status_flags": "",
                "private": true,
                "memo": "",
                "custom_channel_data": {
                    "funding_assets": [
                        {
                            "version": 1,
                            "asset_genesis": {
                                "genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
                                "name": "LNT",
                                "meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
                                "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
                            },
                            "amount": 10000000000,
                            "script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
                            "decimal_display": 6
                        }
                    ],
                    "local_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 9999619484
                        }
                    ],
                    "remote_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 380516
                        }
                    ],
                    "outgoing_htlcs": [],
                    "incoming_htlcs": [],
                    "capacity": 10000000000,
                    "local_balance": 9999619484,
                    "remote_balance": 380516,
                    "outgoing_htlc_balance": 0,
                    "incoming_htlc_balance": 0
                }
            },
            "closing_txid": "70aa87e5a77013fcc51af00c3b2fb0c17aefc9116951d3a7b35f31d42a187bc7",
            "limbo_balance": "8418",
            "maturity_height": 904806,
            "blocks_til_maturity": -205,
            "recovered_balance": "0",
            "pending_htlcs": [],
            "anchor": "LIMBO"
        },
        {
            "channel": {
                "remote_node_pub": "021b161cc6bbdce722d07514f9c72badb6843433084d4c67bd1842fc48c2e363b6",
                "channel_point": "db5b335293c4f7d736e5dab6ac2972f7c7023d0680dde71d9dccfb74d9125581:0",
                "capacity": "20000",
                "local_balance": "8855",
                "remote_balance": "10000",
                "local_chan_reserve_sat": "0",
                "remote_chan_reserve_sat": "0",
                "initiator": "INITIATOR_LOCAL",
                "commitment_type": "SIMPLE_TAPROOT_OVERLAY",
                "num_forwarding_packages": "0",
                "chan_status_flags": "",
                "private": true,
                "memo": "",
                "custom_channel_data": {
                    "funding_assets": [
                        {
                            "version": 1,
                            "asset_genesis": {
                                "genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
                                "name": "LNT",
                                "meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
                                "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
                            },
                            "amount": 10000000000,
                            "script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
                            "decimal_display": 6
                        }
                    ],
                    "local_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 9814878966
                        }
                    ],
                    "remote_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 185121034
                        }
                    ],
                    "outgoing_htlcs": [],
                    "incoming_htlcs": [],
                    "capacity": 10000000000,
                    "local_balance": 9814878966,
                    "remote_balance": 185121034,
                    "outgoing_htlc_balance": 0,
                    "incoming_htlc_balance": 0
                }
            },
            "closing_txid": "6233e886a9d39c0d40cac8997c2f91b8224b70eb055440982c38580b5ca68596",
            "limbo_balance": "8855",
            "maturity_height": 904956,
            "blocks_til_maturity": -55,
            "recovered_balance": "330",
            "pending_htlcs": [],
            "anchor": "RECOVERED"
        },
        {
            "channel": {
                "remote_node_pub": "0319698624fba54edce6626bbfd7d0a74e2359436272efedbf35467d866cb57c01",
                "channel_point": "00a26943dd10aa14368f7214a2710596cae67ec237ddb94ae7994e5e9909a5c2:0",
                "capacity": "20000",
                "local_balance": "8855",
                "remote_balance": "10000",
                "local_chan_reserve_sat": "0",
                "remote_chan_reserve_sat": "0",
                "initiator": "INITIATOR_LOCAL",
                "commitment_type": "SIMPLE_TAPROOT_OVERLAY",
                "num_forwarding_packages": "0",
                "chan_status_flags": "",
                "private": true,
                "memo": "",
                "custom_channel_data": {
                    "funding_assets": [
                        {
                            "version": 1,
                            "asset_genesis": {
                                "genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
                                "name": "LNT",
                                "meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
                                "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
                            },
                            "amount": 10000000000,
                            "script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
                            "decimal_display": 6
                        }
                    ],
                    "local_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 9976978782
                        }
                    ],
                    "remote_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 23021218
                        }
                    ],
                    "outgoing_htlcs": [],
                    "incoming_htlcs": [],
                    "capacity": 10000000000,
                    "local_balance": 9976978782,
                    "remote_balance": 23021218,
                    "outgoing_htlc_balance": 0,
                    "incoming_htlc_balance": 0
                }
            },
            "closing_txid": "ad547d90dc2463d3eb55dc72baedc10d4bfef88d41c38381b5fcf3373f65bf9b",
            "limbo_balance": "8855",
            "maturity_height": 905068,
            "blocks_til_maturity": 57,
            "recovered_balance": "330",
            "pending_htlcs": [],
            "anchor": "RECOVERED"
        },
        {
            "channel": {
                "remote_node_pub": "03c1ea3853a96531f72954e0f4f66bcb3e0542f10c46c8b0da0e6f3191e97dbb88",
                "channel_point": "d733afb11ef653f8063770605a23931258b886df77a4f9c91b2fc14db2447bce:0",
                "capacity": "20000",
                "local_balance": "8796",
                "remote_balance": "10000",
                "local_chan_reserve_sat": "0",
                "remote_chan_reserve_sat": "0",
                "initiator": "INITIATOR_LOCAL",
                "commitment_type": "SIMPLE_TAPROOT_OVERLAY",
                "num_forwarding_packages": "0",
                "chan_status_flags": "",
                "private": true,
                "memo": "",
                "custom_channel_data": {
                    "funding_assets": [
                        {
                            "version": 1,
                            "asset_genesis": {
                                "genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
                                "name": "LNT",
                                "meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
                                "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
                            },
                            "amount": 10000000000,
                            "script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
                            "decimal_display": 6
                        }
                    ],
                    "local_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 9950913436
                        }
                    ],
                    "remote_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 49086564
                        }
                    ],
                    "outgoing_htlcs": [],
                    "incoming_htlcs": [],
                    "capacity": 10000000000,
                    "local_balance": 9950913436,
                    "remote_balance": 49086564,
                    "outgoing_htlc_balance": 0,
                    "incoming_htlc_balance": 0
                }
            },
            "closing_txid": "af7313d030694612df2ab0e363062413d274860fca844e74febea1e4df99cdbd",
            "limbo_balance": "9126",
            "maturity_height": 904835,
            "blocks_til_maturity": -176,
            "recovered_balance": "0",
            "pending_htlcs": [],
            "anchor": "LIMBO"
        },
        {
            "channel": {
                "remote_node_pub": "03c8a8d79e54eaee93c2b9ecc0fd4d2224c1a4e31c5e12bedc62ee0fb9b6e2ca69",
                "channel_point": "032d3b9165353be967a2891a2b6e4c578b181ceae62165a0423e1da2f76cdada:0",
                "capacity": "20000",
                "local_balance": "8856",
                "remote_balance": "10000",
                "local_chan_reserve_sat": "0",
                "remote_chan_reserve_sat": "0",
                "initiator": "INITIATOR_LOCAL",
                "commitment_type": "SIMPLE_TAPROOT_OVERLAY",
                "num_forwarding_packages": "0",
                "chan_status_flags": "",
                "private": true,
                "memo": "",
                "custom_channel_data": {
                    "funding_assets": [
                        {
                            "version": 1,
                            "asset_genesis": {
                                "genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
                                "name": "LNT",
                                "meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
                                "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
                            },
                            "amount": 10000000000,
                            "script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
                            "decimal_display": 6
                        }
                    ],
                    "local_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 9999238968
                        }
                    ],
                    "remote_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 761032
                        }
                    ],
                    "outgoing_htlcs": [],
                    "incoming_htlcs": [],
                    "capacity": 10000000000,
                    "local_balance": 9999238968,
                    "remote_balance": 761032,
                    "outgoing_htlc_balance": 0,
                    "incoming_htlc_balance": 0
                }
            },
            "closing_txid": "ef93c27b2fa151959f0d711a9498c0896bd607d04671963da095cf96bc43740b",
            "limbo_balance": "9186",
            "maturity_height": 905005,
            "blocks_til_maturity": -6,
            "recovered_balance": "0",
            "pending_htlcs": [],
            "anchor": "LIMBO"
        },
        {
            "channel": {
                "remote_node_pub": "0213843878d8c02d753966c985a1159ea6d7b61f92a7e8193142258ff3666bb180",
                "channel_point": "f290d6b997fd0294cdb847d4beb4655c364319d910c6c1cd08e77d40aa39c1e6:0",
                "capacity": "20000",
                "local_balance": "8855",
                "remote_balance": "10000",
                "local_chan_reserve_sat": "0",
                "remote_chan_reserve_sat": "0",
                "initiator": "INITIATOR_LOCAL",
                "commitment_type": "SIMPLE_TAPROOT_OVERLAY",
                "num_forwarding_packages": "0",
                "chan_status_flags": "",
                "private": true,
                "memo": "",
                "custom_channel_data": {
                    "funding_assets": [
                        {
                            "version": 1,
                            "asset_genesis": {
                                "genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
                                "name": "LNT",
                                "meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
                                "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
                            },
                            "amount": 10000000000,
                            "script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
                            "decimal_display": 6
                        }
                    ],
                    "local_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 9896119132
                        }
                    ],
                    "remote_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 103880868
                        }
                    ],
                    "outgoing_htlcs": [],
                    "incoming_htlcs": [],
                    "capacity": 10000000000,
                    "local_balance": 9896119132,
                    "remote_balance": 103880868,
                    "outgoing_htlc_balance": 0,
                    "incoming_htlc_balance": 0
                }
            },
            "closing_txid": "44529b0324f3f94f60db65bbed09dce0dc9d15adf2b7d8b183ae9f2e225255f5",
            "limbo_balance": "8855",
            "maturity_height": 905068,
            "blocks_til_maturity": 57,
            "recovered_balance": "330",
            "pending_htlcs": [],
            "anchor": "RECOVERED"
        }
    ],
    "waiting_close_channels": []
}

This is the log from the time period when one of the asset channels was force-closed. I extracted the logs from 2 minutes before and after the timestamp of the closing_txid (44529b0324f3f94f60db65bbed09dce0dc9d15adf2b7d8b183ae9f2e225255f5) on my node.

lnd.txt

This is the log snippet from the time window when one of the asset channels was force-closed. I’ve extracted logs covering 2 minutes before and after the timestamp of the closing_txid (44529b0324f3f94f60db65bbed09dce0dc9d15adf2b7d8b183ae9f2e225255f5) on my node.

I haven't yet obtained the counterparty's logs, but I'll share them as soon as they become available. Please let me know if you need any additional information - I'll be happy to provide whatever is needed for troubleshooting.

[shuishihang]

I located the counterparty's log records, but their node appears to have been offline during the channel closure timeframe - there are no corresponding log entries from that period. The channel that was force-closed is shown in the following segment:

 "pending_force_closing_channels": [
        {
            "channel": {
                "remote_node_pub": "02c953421bc7f07be6052920e46843d11e6d3ffc9986177c91f140d76c6ed3a3d4",
                "channel_point": "00a26943dd10aa14368f7214a2710596cae67ec237ddb94ae7994e5e9909a5c2:0",
                "capacity": "20000",
                "local_balance": "10000",
                "remote_balance": "8855",
                "local_chan_reserve_sat": "0",
                "remote_chan_reserve_sat": "0",
                "initiator": "INITIATOR_REMOTE",
                "commitment_type": "SIMPLE_TAPROOT_OVERLAY",
                "num_forwarding_packages": "0",
                "chan_status_flags": "",
                "private": true,
                "memo": "",
                "custom_channel_data": {
                    "funding_assets": [
                        {
                            "version": 1,
                            "asset_genesis": {
                                "genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
                                "name": "LNT",
                                "meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
                                "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
                            },
                            "amount": 10000000000,
                            "script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
                            "decimal_display": 6
                        }
                    ],
                    "local_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 23021218
                        }
                    ],
                    "remote_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 9976978782
                        }
                    ],
                    "outgoing_htlcs": [],
                    "incoming_htlcs": [],
                    "capacity": 10000000000,
                    "local_balance": 23021218,
                    "remote_balance": 9976978782,
                    "outgoing_htlc_balance": 0,
                    "incoming_htlc_balance": 0
                }
            },
            "closing_txid": "ad547d90dc2463d3eb55dc72baedc10d4bfef88d41c38381b5fcf3373f65bf9b",
            "limbo_balance": "10330",
            "maturity_height": 0,
            "blocks_til_maturity": 0,
            "recovered_balance": "0",
            "pending_htlcs": [],
            "anchor": "LIMBO"
        }
    ],
    "waiting_close_channels": []
}

I observed that the closing transaction (closing_txid: ad547d90dc2463d3eb55dc72baedc10d4bfef88d41c38381b5fcf3373f65bf9b) was initiated at 2025-07-10 21:45:42, but there are no corresponding log entries from that exact timestamp in node's records.

lnd.log

[guggero]

Hi. Thanks for the report. It's possible you ran into the channel connection issue with lnd v0.19.0. This has been fixed in v0.19.1, but we haven't bundled that into a litd version yet. We'll do that soon.

But to confirm, can you please send us more of your log? The actual force close seems to have happened before the timestamp you mentioned, so it's not in the file you sent.

[Roasbeef]

@shuishihang how many peers does your node have?

Looking at the latest logs you sent, it seems like the node may have having some network connectivity issues:

2025-07-11 11:10:46.572 [ERR] CNCT: Notify blockbeat for ChannelArbitrator failed: consumer ChannelArbitrator(00a26943dd10aa14368f7214a2710596cae67ec237ddb94ae7994e5e9909a5c2:0): process block timeout
2025-07-11 11:10:46.572 [ERR] CHIO: Height[905016]: Failed to process block: consumer ChainArbitrator: process block timeout
2025-07-11 11:10:46.572 [ERR] CHIO: Height[905016]: Notify block failed: queue=1 got err: consumer ChainArbitrator: process block timeout
2025-07-11 11:10:46.572 [INF] CNCT: ChannelArbitrator(c0a0d3a5735ccbdf9ff226fb847c98b80f55eb28a578f8aca93174a196462bc3:1) no close event
2025-07-11 11:10:47.164 [INF] SRVR: Established connection to: 02c953421bc7f07be6052920e46843d11e6d3ffc9986177c91f140d76c6ed3a3d4@X:9735

AFAICT, none of the on-chain resolves that handle output sweeping are getting notifications for new blocks.

[shuishihang]

My node may have over 300 peer connections.Why are these force-closed channels stuck in an unresolved state? Attached are the complete logs from the timeframe of the problematic node (closing_txid: ad547d90dc2463d3eb55dc72baedc10d4bfef88d41c38381b5fcf3373f65bf9b). I’m unsure if this contains the specific information you need—the logs are extensive, and I can’t pinpoint the exact timestamp.

lnd.log1.txt

Another channel on my node has entered the waiting_close_channels state. Below are its details and the logs recorded during that timeframe.

    "waiting_close_channels": [
        {
            "channel": {
                "remote_node_pub": "025eb73588b8a32b063a6d9eb0fdf22d82ee850c35f779aa682361a807a464ff32",
                "channel_point": "b042ab0ab4d880ab5dba703a8dd4df587ac46ca6eab47fab0a380d58a40868e2:0",
                "capacity": "20000",
                "local_balance": "7252",
                "remote_balance": "10000",
                "local_chan_reserve_sat": "354",
                "remote_chan_reserve_sat": "1062",
                "initiator": "INITIATOR_LOCAL",
                "commitment_type": "SIMPLE_TAPROOT_OVERLAY",
                "num_forwarding_packages": "0",
                "chan_status_flags": "ChanStatusBorked|ChanStatusCommitBroadcasted|ChanStatusLocalCloseInitiator",
                "private": true,
                "memo": "",
                "custom_channel_data": {
                    "funding_assets": [
                        {
                            "version": 1,
                            "asset_genesis": {
                                "genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
                                "name": "LNT",
                                "meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
                                "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
                            },
                            "amount": 10000000000,
                            "script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
                            "decimal_display": 6
                        }
                    ],
                    "local_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 9985730650
                        }
                    ],
                    "remote_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 14269350
                        }
                    ],
                    "outgoing_htlcs": [],
                    "incoming_htlcs": [],
                    "capacity": 10000000000,
                    "local_balance": 9985730650,
                    "remote_balance": 14269350,
                    "outgoing_htlc_balance": 0,
                    "incoming_htlc_balance": 0
                }
            },
            "limbo_balance": "7252",
            "commitments": {
                "local_txid": "3be78770eb223e9a7930d8698811185bf9f975fd00ca9861426b03f4cf9d975b",
                "remote_txid": "daef6206d1a8cedf8032108805826b102a9a87e50ec72ddc7a8fd1156f6095fc",
                "remote_pending_txid": "428bc2b60ba86532e139c08ff66dfef356f5331023511cbf8e19bfc030c29e5d",
                "local_commit_fee_sat": "672",
                "remote_commit_fee_sat": "485",
                "remote_pending_commit_fee_sat": "0"
            },
            "closing_txid": "3be78770eb223e9a7930d8698811185bf9f975fd00ca9861426b03f4cf9d975b",
            "closing_tx_hex": ""
        }
    ]
}

lnd.log.txt

[guggero]

Thanks for the logs. Unfortunately they still don't contain the actual cause for the force close. The earliest mention of one of the channels (for example afc7179f513ed1651d0a950a184a05c68fc2b080b777d021418d7a9071dc4964:0) is at 2025-07-10 20:29:28.784, but it looks like at that point the channel was already force closed:

2025-07-10 20:29:28.784 [INF] CNCT: ChannelArbitrator(afc7179f513ed1651d0a950a184a05c68fc2b080b777d021418d7a9071dc4964:0): skipping reading close events in state=StateWaitingFullResolution

Do you have log files that go further back than that date?

I'm still looking into why the channel state isn't resolved. Unfortunately there are no error messages related to that in either log file.

I also saw that you're using a custom build of litd with modifications. Can you confirm you're running the correct commit of tapd? If yes, can you please confirm the actual commit you compiled your litd with?

[shuishihang]

litd -- v0.15.0-alpha
I have a closed asset channel (closing_txid: bbc9e6c24018088c58a2be957c07c76836cbcc8feb2d70b44f541d2bd2e505a8, timestamped 2025-07-14 02:18:55). Attached are the logs generated during the 3-hour window around its closure. These are the only recent logs I could retrieve for a closed asset channel, as older logs were automatically purged.

lnd.log.txt

lnd.log1.txt

[shuishihang]

litd -- v0.15.0-alpha  litd——v0.15.0-alpha I have a closed asset channel (closing_txid: bbc9e6c24018088c58a2be957c07c76836cbcc8feb2d70b44f541d2bd2e505a8, timestamped 2025-07-14 02:18:55). Attached are the logs generated during the 3-hour window around its closure. These are the only recent logs I could retrieve for a closed asset channel, as older logs were automatically purged.我有一个已关闭的资产通道（closing_txid：bbc9e6c24018088c58a2be957c07c76836cbcc8feb2d70b44f541d2bd2e505a8，时间戳为 2025-07-14 02:18:55）。附件是该通道关闭前后 3 小时内生成的日志。这些是我能检索到的关于已关闭资产通道的最新日志，因为旧日志已被自动清除。

lnd.log.txt

lnd.log1.txt

If another asset channel gets force-closed again, I will carefully monitor the situation and provide you with all relevant logs for analysis

[guggero]

I'm not sure how you timestamp the closing transaction. But the channel 0b5fc3a49ecaa2a3420e84c7a7085df69fc41326abf5ee1c6d212d9ac4d95e54:0 was closed on 2025-07-13 (the log file is always in UTC). So again, the actual force close of 0b5fc3a49ecaa2a3420e84c7a7085df69fc41326abf5ee1c6d212d9ac4d95e54:0 (closing txid bbc9e6c24018088c58a2be957c07c76836cbcc8feb2d70b44f541d2bd2e505a8) is not in the files you sent...

What about the other questions above?

[shuishihang]

😅I searched for the closing_txid (ad547d90dc2463d3eb55dc72baedc10d4bfef88d41c38381b5fcf3373f65bf9b) in the mempool to check its timestamp, and I might have indeed mistaken the actual closure time. I’ll pay closer attention to this issue if another force-close occurs. However, I still don’t understand why the previously force-closed asset channels remain stuck in an unresolved state.

[shuishihang]

 {
            "channel": {
                "remote_node_pub": "03e13380639fbf0a3a990e74185ff966bc87780f448d2e8653900eff8e47b047bf",
                "channel_point": "9fa4ccbf4e0d48ae32e8bcbc0292e4b4c95aa8b948c70af528e49bd6d0faa0c8:0",
                "capacity": "20000",
                "local_balance": "8102",
                "remote_balance": "10000",
                "local_chan_reserve_sat": "0",
                "remote_chan_reserve_sat": "0",
                "initiator": "INITIATOR_LOCAL",
                "commitment_type": "SIMPLE_TAPROOT_OVERLAY",
                "num_forwarding_packages": "0",
                "chan_status_flags": "",
                "private": true,
                "memo": "",
                "custom_channel_data": {
                    "funding_assets": [
                        {
                            "version": 1,
                            "asset_genesis": {
                                "genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
                                "name": "LNT",
                                "meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
                                "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
                            },
                            "amount": 10000000000,
                            "script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
                            "decimal_display": 6
                        }
                    ],
                    "local_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 9784437686
                        }
                    ],
                    "remote_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 215562314
                        }
                    ],
                    "outgoing_htlcs": [],
                    "incoming_htlcs": [],
                    "capacity": 10000000000,
                    "local_balance": 9784437686,
                    "remote_balance": 215562314,
                    "outgoing_htlc_balance": 0,
                    "incoming_htlc_balance": 0
                }
            },
            "closing_txid": "68e3255acfd9a3060a7afa3fe2db4c2e31deb9e688f2cbc56a5d6689bcb4b505",
            "limbo_balance": "8432",
            "maturity_height": 905673,
            "blocks_til_maturity": 81,
            "recovered_balance": "0",
            "pending_htlcs": [],
            "anchor": "LIMBO"
        },
 {
            "channel": {
                "remote_node_pub": "0280d7d16f739a5d137a0ba5ddc244a520337b787628e6c33c1a7194b1640b7176",
                "channel_point": "9fb7c2c513cddfe03ede9fd5fdb1a17aa667e3eb51cbcbf5ee2c4cce0d7eb0be:0",
                "capacity": "20000",
                "local_balance": "8039",
                "remote_balance": "10000",
                "local_chan_reserve_sat": "0",
                "remote_chan_reserve_sat": "0",
                "initiator": "INITIATOR_LOCAL",
                "commitment_type": "SIMPLE_TAPROOT_OVERLAY",
                "num_forwarding_packages": "0",
                "chan_status_flags": "",
                "private": true,
                "memo": "",
                "custom_channel_data": {
                    "funding_assets": [
                        {
                            "version": 1,
                            "asset_genesis": {
                                "genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
                                "name": "LNT",
                                "meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
                                "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
                            },
                            "amount": 10000000000,
                            "script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
                            "decimal_display": 6
                        }
                    ],
                    "local_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 9786720782
                        }
                    ],
                    "remote_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 213279218
                        }
                    ],
                    "outgoing_htlcs": [],
                    "incoming_htlcs": [],
                    "capacity": 10000000000,
                    "local_balance": 9786720782,
                    "remote_balance": 213279218,
                    "outgoing_htlc_balance": 0,
                    "incoming_htlc_balance": 0
                }
            },
            "closing_txid": "597029f2f4268f32069e1134b1e97c82469feac9a3a66180fba2264542bf19ca",
            "limbo_balance": "8369",
            "maturity_height": 905674,
            "blocks_til_maturity": 29,
            "recovered_balance": "0",
            "pending_htlcs": [],
            "anchor": "LIMBO"
        },

This time, I found the force-closure information of this asset channel in the logs.
2025-07-14 23:17:59.730 [INF] CNCT: ChannelArbitrator(9fa4ccbf4e0d48ae32e8bcbc0292e4b4c95aa8b948c70af528e49bd6d0faa0c8:0): fail dangling htlc=0c052c70aaffcade1096a4a74dd9a13ad9dcf1cd382a0596edb7ab789bb68226 from local/remote commitments diff

lnd.log1.txt

[guggero]

Can you please just give me the full log file with everything as far back as possible? You can zip it if there's a size limit (or send it to me on another platform). It's really hard to find the cause for something that happens sporadically when we don't have the full log.

Also, please answer the other questions above. And having a bit more detail about your node also wouldn't help. What chain backend? Are some of your peers also nodes you control? For example, the log you provided of one peer that runs on Neutrino with many hundreds of channels... That's not really a stable configuration IMO, is that also one of your nodes?

[shuishihang]

I don’t have complete log records because my node automatically purges oversized LND logs. When the issue recurs, I’ll compile and share all existing intact logs. My node runs on Bitcoind, while my ~300 peers use Neutrino. These peers only open one asset channel and one BTC channel with my node. I did use a custom-modified LiTd version (v0.15.0-alpha), but without altering core functionalities. The Tapd commit is correct, and LiTd was compiled from the v0.15.0-alpha source.
In Tapd, I only modified the satoshi capacity parameter for asset channel openings, without altering any other channel-related functionalities

[shuishihang]

Below are the forcibly closed asset channels that occurred today. The earliest logs I currently have cover from last night at 8 PM up to the present.

{
            "channel": {
                "remote_node_pub": "02246a884d38c94bdb3101890d2500749f734d843ea891c5399072f5237ce1300f",
                "channel_point": "515a775c47984d8a72fb50f189fb36fdf7c1c3059bfb012a632b602592232147:0",
                "capacity": "20000",
                "local_balance": "7598",
                "remote_balance": "10000",
                "local_chan_reserve_sat": "0",
                "remote_chan_reserve_sat": "0",
                "initiator": "INITIATOR_LOCAL",
                "commitment_type": "SIMPLE_TAPROOT_OVERLAY",
                "num_forwarding_packages": "0",
                "chan_status_flags": "",
                "private": true,
                "memo": "",
                "custom_channel_data": {
                    "funding_assets": [
                        {
                            "version": 1,
                            "asset_genesis": {
                                "genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
                                "name": "LNT",
                                "meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
                                "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
                            },
                            "amount": 10000000000,
                            "script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
                            "decimal_display": 6
                        }
                    ],
                    "local_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 9990106584
                        }
                    ],
                    "remote_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 9893416
                        }
                    ],
                    "outgoing_htlcs": [],
                    "incoming_htlcs": [],
                    "capacity": 10000000000,
                    "local_balance": 9990106584,
                    "remote_balance": 9893416,
                    "outgoing_htlc_balance": 0,
                    "incoming_htlc_balance": 0
                }
            },
            "closing_txid": "7dbc18e80df8c0fd12d0cff54e7a946eea78cace4189daf3ae23a313bc36f0f7",
            "limbo_balance": "7928",
            "maturity_height": 906164,
            "blocks_til_maturity": 82,
            "recovered_balance": "0",
            "pending_htlcs": [],
            "anchor": "LIMBO"
        },
 {
            "channel": {
                "remote_node_pub": "02ce0e0d0b8d48535f979c1823ed500b7280885615b41a7bfdf4596536273644aa",
                "channel_point": "d2ae128cbfa80a16cd45f9b5e84ac82627e90a4712dece668d24d604f9a5915c:0",
                "capacity": "20000",
                "local_balance": "7659",
                "remote_balance": "10000",
                "local_chan_reserve_sat": "0",
                "remote_chan_reserve_sat": "0",
                "initiator": "INITIATOR_LOCAL",
                "commitment_type": "SIMPLE_TAPROOT_OVERLAY",
                "num_forwarding_packages": "0",
                "chan_status_flags": "",
                "private": true,
                "memo": "",
                "custom_channel_data": {
                    "funding_assets": [
                        {
                            "version": 1,
                            "asset_genesis": {
                                "genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
                                "name": "LNT",
                                "meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
                                "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
                            },
                            "amount": 10000000000,
                            "script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
                            "decimal_display": 6
                        }
                    ],
                    "local_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 9733448542
                        }
                    ],
                    "remote_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 266551458
                        }
                    ],
                    "outgoing_htlcs": [],
                    "incoming_htlcs": [],
                    "capacity": 10000000000,
                    "local_balance": 9733448542,
                    "remote_balance": 266551458,
                    "outgoing_htlc_balance": 0,
                    "incoming_htlc_balance": 0
                }
            },
            "closing_txid": "6f751a1c6997f73471bbba3bfdcd4fde2ce6986b9d1a962b6405a58f0fced42f",
            "limbo_balance": "7989",
            "maturity_height": 906163,
            "blocks_til_maturity": 81,
            "recovered_balance": "0",
            "pending_htlcs": [],
            "anchor": "LIMBO"
        },
 {
            "channel": {
                "remote_node_pub": "0284430163650adb660c69ac55a58e1b697ca6524bf8a43f12ab67ac089c9e48b5",
                "channel_point": "82d08f9b1ae941ca7cbd8b1abe09c81e463e4f4971c48e873b24138e253715c5:0",
                "capacity": "20000",
                "local_balance": "7660",
                "remote_balance": "10000",
                "local_chan_reserve_sat": "0",
                "remote_chan_reserve_sat": "0",
                "initiator": "INITIATOR_LOCAL",
                "commitment_type": "SIMPLE_TAPROOT_OVERLAY",
                "num_forwarding_packages": "0",
                "chan_status_flags": "",
                "private": true,
                "memo": "",
                "custom_channel_data": {
                    "funding_assets": [
                        {
                            "version": 1,
                            "asset_genesis": {
                                "genesis_point": "6163c6dd64dc8394a14a752b6de480c8da8991cc7f367a467135c1a2bb38c2c9:5",
                                "name": "LNT",
                                "meta_hash": "83acf6a925f04714026da39016d568c47c53aff1cb0f6d67f7c3f5bb6cf05be4",
                                "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c"
                            },
                            "amount": 10000000000,
                            "script_key": "0250aaeb166f4234650d84a2d8a130987aeaf6950206e0905401ee74ff3f8d18e6",
                            "decimal_display": 6
                        }
                    ],
                    "local_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 9948630340
                        }
                    ],
                    "remote_assets": [
                        {
                            "asset_id": "97b98f3c45f926057d430ef71f20a6d3e25d7a00fbd1d7b72b306a49d48c9d8c",
                            "amount": 51369660
                        }
                    ],
                    "outgoing_htlcs": [],
                    "incoming_htlcs": [],
                    "capacity": 10000000000,
                    "local_balance": 9948630340,
                    "remote_balance": 51369660,
                    "outgoing_htlc_balance": 0,
                    "incoming_htlc_balance": 0
                }
            },
            "closing_txid": "f318fe0be64ce75d215357cef3c908e1b7f88f2f2c25af47d97400567abe6f68",
            "limbo_balance": "7990",
            "maturity_height": 906166,
            "blocks_til_maturity": 84,
            "recovered_balance": "0",
            "pending_htlcs": [],
            "anchor": "LIMBO"
        }