diff --git a/electrum/commands.py b/electrum/commands.py index 01af4aea644f..988ba1b77bdc 100644 --- a/electrum/commands.py +++ b/electrum/commands.py @@ -71,7 +71,7 @@ from .plugin import run_hook, DeviceMgr, Plugins from .version import ELECTRUM_VERSION from .simple_config import SimpleConfig -from .fee_policy import FeePolicy +from .fee_policy import FeePolicy, FEE_ETA_TARGETS, FEERATE_DEFAULT_RELAY from . import GuiImportError from . import crypto from . import constants @@ -1577,6 +1577,27 @@ async def getfeerate(self): 'tooltip': tooltip, } + @command('n') + async def test_inject_fee_etas(self, fee_est): + """ + Inject fee estimates into the network object, as if they were coming from connected servers. + Useful on regtest. + + arg:str:fee_est:dict of ETA-based fee estimates, encoded as str + """ + if not isinstance(fee_est, dict): + fee_est = ast.literal_eval(fee_est) + assert isinstance(fee_est, dict), f"unexpected type for fee_est. got {repr(fee_est)}" + # populate missing high-block-number estimates using default relay fee. + # e.g. {"25": 2222} -> {"25": 2222, "144": 1000, "1008": 1000} + furthest_estimate = max(fee_est.keys()) if fee_est else 0 + further_fee_est = { + eta_target: FEERATE_DEFAULT_RELAY for eta_target in FEE_ETA_TARGETS + if eta_target > furthest_estimate + } + fee_est.update(further_fee_est) + self.network.update_fee_estimates(fee_est=fee_est) + @command('w') async def removelocaltx(self, txid, wallet: Abstract_Wallet = None): """Remove a 'local' transaction from the wallet, and its dependent @@ -1622,7 +1643,8 @@ async def add_peer(self, connection_string, timeout=20, gossip=False, wallet: Ab arg:int:timeout:Timeout in seconds (default=20) """ lnworker = self.network.lngossip if gossip else wallet.lnworker - await lnworker.add_peer(connection_string) + async with util.async_timeout(timeout): + await lnworker.add_peer(connection_string) return True @command('wnl') @@ -1698,19 +1720,20 @@ async def decode_invoice(self, invoice: str): return invoice.to_debug_json() @command('wnpl') - async def lnpay(self, invoice, timeout=120, password=None, wallet: Abstract_Wallet = None): + async def lnpay(self, invoice, timeout=0, password=None, wallet: Abstract_Wallet = None): """ Pay a lightning invoice arg:str:invoice:Lightning invoice (bolt 11) - arg:int:timeout:Timeout in seconds (default=20) + arg:int:timeout:Timeout in seconds. Note: *not* safe to try paying same invoice multiple times with a timeout. """ lnworker = wallet.lnworker lnaddr = lnworker._check_bolt11_invoice(invoice) payment_hash = lnaddr.paymenthash invoice_obj = Invoice.from_bech32(invoice) wallet.save_invoice(invoice_obj) - success, log = await lnworker.pay_invoice(invoice_obj) + async with util.async_timeout(timeout or None): + success, log = await lnworker.pay_invoice(invoice_obj) return { 'payment_hash': payment_hash.hex(), 'success': success, diff --git a/electrum/fee_policy.py b/electrum/fee_policy.py index 958c35854549..33b7eb384739 100644 --- a/electrum/fee_policy.py +++ b/electrum/fee_policy.py @@ -375,8 +375,9 @@ def has_data(self) -> bool: just try to do the estimate and handle a potential None result. That way, estimation works for targets we have, even if some targets are missing. """ - # we do not request estimate for next block fee, hence -1 - return len(self.data) == len(FEE_ETA_TARGETS) - 1 + targets = set(FEE_ETA_TARGETS) + targets.discard(1) # rm "next block" target + return all(target in self.data for target in targets) def set_data(self, nblock_target: int, fee_per_kb: int): assert isinstance(nblock_target, int), f"expected int, got {nblock_target!r}" diff --git a/electrum/lnwatcher.py b/electrum/lnwatcher.py index 8b55b27e80fc..419abc92c841 100644 --- a/electrum/lnwatcher.py +++ b/electrum/lnwatcher.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Optional from . import util -from .util import TxMinedInfo, BelowDustLimit +from .util import TxMinedInfo, BelowDustLimit, NoDynamicFeeEstimates from .util import EventListener, event_listener, log_exceptions, ignore_exceptions from .transaction import Transaction, TxOutpoint from .logging import Logger @@ -99,6 +99,7 @@ async def check_onchain_situation(self, address: str, funding_outpoint: str) -> if not self.adb.is_mine(address): return # inspect_tx_candidate might have added new addresses, in which case we return early + # note: maybe we should wait until adb.is_up_to_date... (?) funding_txid = funding_outpoint.split(':')[0] funding_height = self.adb.get_tx_height(funding_txid) closing_txid = self.adb.get_spender(funding_outpoint) @@ -159,7 +160,9 @@ async def sweep_commitment_transaction(self, funding_outpoint: str, closing_tx: return False # detect who closed and get information about how to claim outputs is_local_ctx, sweep_info_dict = chan.get_ctx_sweep_info(closing_tx) - keep_watching = False if sweep_info_dict else not self.adb.is_deeply_mined(closing_tx.txid()) + # note: we need to keep watching *at least* until the closing tx is deeply mined, + # possibly longer if there are TXOs to sweep + keep_watching = not self.adb.is_deeply_mined(closing_tx.txid()) # create and broadcast transactions for prevout, sweep_info in sweep_info_dict.items(): prev_txid, prev_index = prevout.split(':') @@ -169,38 +172,44 @@ async def sweep_commitment_transaction(self, funding_outpoint: str, closing_tx: # do not keep watching if prevout does not exist self.logger.info(f'prevout does not exist for {name}: {prevout}') continue - was_added = self.maybe_redeem(sweep_info) - spender_txid = self.adb.get_spender(prevout) + watch_sweep_info = self.maybe_redeem(sweep_info) + spender_txid = self.adb.get_spender(prevout) # note: LOCAL spenders don't count spender_tx = self.adb.get_transaction(spender_txid) if spender_txid else None if spender_tx: # the spender might be the remote, revoked or not htlc_sweepinfo = chan.maybe_sweep_htlcs(closing_tx, spender_tx) for prevout2, htlc_sweep_info in htlc_sweepinfo.items(): - htlc_was_added = self.maybe_redeem(htlc_sweep_info) + watch_htlc_sweep_info = self.maybe_redeem(htlc_sweep_info) htlc_tx_spender = self.adb.get_spender(prevout2) self.lnworker.wallet.set_default_label(prevout2, htlc_sweep_info.name) if htlc_tx_spender: keep_watching |= not self.adb.is_deeply_mined(htlc_tx_spender) self.maybe_add_accounting_address(htlc_tx_spender, htlc_sweep_info) else: - keep_watching |= htlc_was_added + keep_watching |= watch_htlc_sweep_info keep_watching |= not self.adb.is_deeply_mined(spender_txid) self.maybe_extract_preimage(chan, spender_tx, prevout) self.maybe_add_accounting_address(spender_txid, sweep_info) else: - keep_watching |= was_added + keep_watching |= watch_sweep_info self.maybe_add_pending_forceclose( - chan=chan, spender_txid=spender_txid, is_local_ctx=is_local_ctx, sweep_info=sweep_info, was_added=was_added) + chan=chan, spender_txid=spender_txid, is_local_ctx=is_local_ctx, sweep_info=sweep_info) return keep_watching def get_pending_force_closes(self): return self._pending_force_closes def maybe_redeem(self, sweep_info: 'SweepInfo') -> bool: - """ returns False if it was dust """ + """ returns 'keep_watching' """ try: self.lnworker.wallet.txbatcher.add_sweep_input('lnwatcher', sweep_info) except BelowDustLimit: + # utxo is considered dust at *current* fee estimates. + # but maybe the fees atm are very high? We will retry later. + pass + except NoDynamicFeeEstimates: + pass # will retry later + if sweep_info.is_anchor(): return False return True @@ -251,10 +260,9 @@ def maybe_add_pending_forceclose( spender_txid: Optional[str], is_local_ctx: bool, sweep_info: 'SweepInfo', - was_added: bool, ): """ we are waiting for ctx to be confirmed and there are received htlcs """ - if was_added and is_local_ctx and sweep_info.name == 'received-htlc' and chan.has_anchors(): + if is_local_ctx and sweep_info.name == 'received-htlc' and chan.has_anchors(): tx_mined_status = self.adb.get_tx_height(spender_txid) if tx_mined_status.height == TX_HEIGHT_LOCAL: self._pending_force_closes.add(chan) diff --git a/electrum/plugins/watchtower/watchtower.py b/electrum/plugins/watchtower/watchtower.py index a8dacb686645..99a48c28d7b7 100644 --- a/electrum/plugins/watchtower/watchtower.py +++ b/electrum/plugins/watchtower/watchtower.py @@ -202,9 +202,10 @@ def inspect_tx_candidate(self, outpoint, n: int) -> Dict[str, str]: result.update(r) return result - async def sweep_commitment_transaction(self, funding_outpoint, closing_tx): + async def sweep_commitment_transaction(self, funding_outpoint: str, closing_tx: Transaction) -> bool: + assert closing_tx spenders = self.inspect_tx_candidate(funding_outpoint, 0) - keep_watching = False + keep_watching = not self.adb.is_deeply_mined(closing_tx.txid()) for prevout, spender in spenders.items(): if spender is not None: keep_watching |= not self.adb.is_deeply_mined(spender) diff --git a/electrum/simple_config.py b/electrum/simple_config.py index 4f83abfc3ef2..b1cdf465bcca 100644 --- a/electrum/simple_config.py +++ b/electrum/simple_config.py @@ -891,6 +891,7 @@ def __setattr__(self, name, value): # connect to remote submarine swap server SWAPSERVER_URL = ConfigVar('swapserver_url', default='', type_=str) TEST_SWAPSERVER_REFUND = ConfigVar('test_swapserver_refund', default=False, type_=bool) + TEST_SWAPSERVER_SKIP_ONCHAIN_FUNDING = ConfigVar('test_swapserver_skip_onchain_funding', default=False, type_=bool) SWAPSERVER_NPUB = ConfigVar('swapserver_npub', default=None, type_=str) SWAPSERVER_POW_TARGET = ConfigVar('swapserver_pow_target', default=30, type_=int) diff --git a/electrum/submarine_swaps.py b/electrum/submarine_swaps.py index d8edc14232e1..9bf34f07bfe0 100644 --- a/electrum/submarine_swaps.py +++ b/electrum/submarine_swaps.py @@ -32,7 +32,7 @@ from .util import ( log_exceptions, ignore_exceptions, BelowDustLimit, OldTaskGroup, ca_path, gen_nostr_ann_pow, get_nostr_ann_pow_amount, make_aiohttp_proxy_connector, get_running_loop, get_asyncio_loop, wait_for2, - run_sync_function_on_asyncio_thread, trigger_callback + run_sync_function_on_asyncio_thread, trigger_callback, NoDynamicFeeEstimates ) from . import lnutil from .lnutil import hex_to_bytes, REDEEM_AFTER_DOUBLE_SPENT_DELAY, Keypair @@ -485,6 +485,9 @@ async def _claim_swap(self, swap: SwapData) -> None: except BelowDustLimit: self.logger.info('utxo value below dust threshold') return + except NoDynamicFeeEstimates: + self.logger.info('got NoDynamicFeeEstimates') + return def get_fee_for_txbatcher(self): return self._get_tx_fee(self.config.FEE_POLICY_SWAPS) @@ -512,6 +515,8 @@ async def hold_invoice_callback(self, payment_hash: bytes) -> None: key = payment_hash.hex() if swap := self._swaps.get(key): if not swap.is_funded(): + if self.network.config.TEST_SWAPSERVER_SKIP_ONCHAIN_FUNDING: # for testing + return output = self.create_funding_output(swap) self.wallet.txbatcher.add_payment_output('swaps', output) swap._payment_pending = True diff --git a/electrum/txbatcher.py b/electrum/txbatcher.py index 2cb3b0e86dfb..dcc0ebccb3d7 100644 --- a/electrum/txbatcher.py +++ b/electrum/txbatcher.py @@ -99,6 +99,7 @@ def add_payment_output(self, key: str, output: 'PartialTxOutput') -> None: @locked def add_sweep_input(self, key: str, sweep_info: 'SweepInfo') -> None: + """Can raise BelowDustLimit or NoDynamicFeeEstimates.""" if sweep_info.txin and sweep_info.txout: # detect legacy htlc using name and csv delay if sweep_info.name in ['received-htlc', 'offered-htlc'] and sweep_info.csv_delay == 0: @@ -263,20 +264,25 @@ def add_payment_output(self, output: 'PartialTxOutput') -> None: self.batch_payments.append(output) def is_dust(self, sweep_info: SweepInfo) -> bool: + """Can raise BelowDustLimit or NoDynamicFeeEstimates.""" if sweep_info.is_anchor(): return False if sweep_info.txout is not None: return False - value = sweep_info.txin._trusted_value_sats + value = sweep_info.txin.value_sats() witness_size = len(sweep_info.txin.make_witness(71*b'\x00')) tx_size_vbytes = 84 + witness_size//4 # assumes no batching, sweep to p2wpkh self.logger.info(f'{sweep_info.name} size = {tx_size_vbytes}') - fee = self.fee_policy.estimate_fee(tx_size_vbytes, network=self.wallet.network, allow_fallback_to_static_rates=True) + fee = self.fee_policy.estimate_fee(tx_size_vbytes, network=self.wallet.network) return value - fee <= dust_threshold() @locked def add_sweep_input(self, sweep_info: 'SweepInfo') -> None: + """Can raise BelowDustLimit or NoDynamicFeeEstimates.""" if self.is_dust(sweep_info): + # note: this uses the current fee estimates. Just because something is dust + # at the current fee levels, if fees go down, it might still become + # worthwhile to sweep. So callers might want to retry later. raise BelowDustLimit txin = sweep_info.txin if txin.prevout in self._unconfirmed_sweeps: diff --git a/tests/regtest.py b/tests/regtest.py index 87aff21a82ca..e192dd06bd04 100644 --- a/tests/regtest.py +++ b/tests/regtest.py @@ -82,6 +82,9 @@ def test_breach_with_unspent_htlc(self): def test_breach_with_spent_htlc(self): self.run_shell(['breach_with_spent_htlc']) + def test_lnwatcher_waits_until_fees_go_down(self): + self.run_shell(['lnwatcher_waits_until_fees_go_down']) + class TestLightningSwapserver(TestLightning): agents = { @@ -107,6 +110,9 @@ def test_swapserver_forceclose(self): def test_swapserver_refund(self): self.run_shell(['swapserver_refund']) + def test_swapserver_server_skip_onchain_funding(self): + self.run_shell(['swapserver_server_skip_onchain_funding']) + class TestLightningWatchtower(TestLightning): diff --git a/tests/regtest/regtest.sh b/tests/regtest/regtest.sh index be4f9cfa5a81..5ad21566c25f 100755 --- a/tests/regtest/regtest.sh +++ b/tests/regtest/regtest.sh @@ -90,6 +90,15 @@ function wait_until_spent() printf "\n" } +function assert_utxo_exists() +{ + utxo=$($bitcoin_cli gettxout $1 $2) + if [[ -z "$utxo" ]]; then + echo "utxo $1:$2 does not exist" + exit 1 + fi +} + if [[ $# -eq 0 ]]; then echo "syntax: init|start|open|status|pay|close|stop" exit 1 @@ -308,6 +317,116 @@ if [[ $1 == "swapserver_refund" ]]; then fi +if [[ $1 == "swapserver_server_skip_onchain_funding" ]]; then + # Alice starts reverse-swap with Bob. + # Alice sends hold-HTLCs via LN. Bob does NOT fund locking script onchain. + # After a while, Alice requests Bob to force-close chan, and Bob does. + # Alice will broadcast HTLC-timeout tx to reclaim the swap amount from Bob's commitment tx. + $bob setconfig test_swapserver_skip_onchain_funding true + wait_for_balance alice 1 + echo "alice opens channel" + bob_node=$($bob nodeid) + channel=$($alice open_channel $bob_node 0.15 --password='') + chan_funding_txid=$(echo "$channel" | cut -d ":" -f 1) + chan_funding_outidx=$(echo "$channel" | cut -d ":" -f 2) + new_blocks 3 + wait_until_channel_open alice + echo "alice initiates swap" + dryrun=$($alice reverse_swap 0.02 dryrun) + onchain_amount=$(echo $dryrun| jq -r ".onchain_amount") + # Alice starts a reverse-swap, but will time out waiting for Bob's swap-funding-tx to appear in mempool. + $alice setconfig timeout 10 + set +e + swap=$($alice reverse_swap 0.02 $onchain_amount) + set -e + $alice unsetconfig timeout + # After a while, Alice gets impatient and gets Bob to close the channel. + new_blocks 20 + $alice request_force_close $channel + wait_until_spent $chan_funding_txid $chan_funding_outidx + new_blocks 1 + wait_until_channel_closed alice + ctx_id=$($alice list_channels | jq -r ".[0].closing_txid") + # need more blocks to reach CLTV of HTLC-output in ctx + new_blocks 20 + if [ $TEST_ANCHOR_CHANNELS = True ] ; then + htlc_output_index=3 # FIXME index depends on Alice not using MPP # FIXME presence of fee prepayment depends on fee-levels + else + htlc_output_index=1 + fi + assert_utxo_exists $ctx_id $htlc_output_index + new_blocks 110 + wait_until_spent $ctx_id $htlc_output_index + new_blocks 1 + wait_for_balance alice 0.997 +fi + + +if [[ $1 == "lnwatcher_waits_until_fees_go_down" ]]; then + # Alice sends two HTLCs to Bob (one for small invoice, one for large invoice), which Bob will hold. + # Alice requests Bob to force-close the channel, while the HTLCs are pending. Bob force-closes. + # Fee levels rise, to the point where the small HTLC is not economical to claim. + # Alice sweeps the large HTLC (via onchain timeout), but not the small one. + # Then, fee levels go back down, and Alice sweeps the small HTLC. + # This test checks Alice does not abandon channel outputs that are temporarily ~dust due to + # mempool spikes, and keeps watching the channel in hope of fees going down. + $alice setconfig test_force_disable_mpp true + $alice setconfig test_force_mpp false + wait_for_balance alice 1 + $alice test_inject_fee_etas "{2:1000}" + $bob test_inject_fee_etas "{2:1000}" + echo "alice opens channel" + bob_node=$($bob nodeid) + channel=$($alice open_channel $bob_node 0.15 --password='') + chan_funding_txid=$(echo "$channel" | cut -d ":" -f 1) + chan_funding_outidx=$(echo "$channel" | cut -d ":" -f 2) + new_blocks 3 + wait_until_channel_open alice + # Alice sends an HTLC to Bob, which Bob will hold indefinitely. Alice's lnpay will time out. + invoice1=$($bob add_hold_invoice deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbee1 \ + --amount 0.0004 --min_final_cltv_expiry_delta 300 | jq -r ".invoice") + invoice2=$($bob add_hold_invoice deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbee2 \ + --amount 0.04 --min_final_cltv_expiry_delta 300 | jq -r ".invoice") + set +e + $alice lnpay $invoice1 --timeout 3 + $alice lnpay $invoice2 --timeout 3 + set -e + # After a while, Alice gets impatient and gets Bob to close the channel. + new_blocks 20 + $alice request_force_close $channel + wait_until_spent $chan_funding_txid $chan_funding_outidx + $bob stop # bob closes and then disappears. FIXME this is a hack to prevent Bob claiming the fake-hold-invoice-htlc onchain + new_blocks 1 + wait_until_channel_closed alice + ctx_id=$($alice list_channels | jq -r ".[0].closing_txid") + if [ $TEST_ANCHOR_CHANNELS = True ] ; then + htlc_output_index1=2 + htlc_output_index2=3 + to_alice_index=4 # Bob's to_remote + wait_until_spent $ctx_id $to_alice_index + else + htlc_output_index1=0 + htlc_output_index2=1 + to_alice_index=2 + fi + new_blocks 1 + assert_utxo_exists $ctx_id $htlc_output_index1 + assert_utxo_exists $ctx_id $htlc_output_index2 + # fee levels rise. now small htlc is ~dust + $alice test_inject_fee_etas "{2:300000}" + new_blocks 300 # this goes past the CLTV of the HTLC-output in ctx + wait_until_spent $ctx_id $htlc_output_index2 + assert_utxo_exists $ctx_id $htlc_output_index1 + new_blocks 1 + # fee levels go down. time to claim the small htlc + $alice test_inject_fee_etas "{2:1000}" + new_blocks 1 + wait_until_spent $ctx_id $htlc_output_index1 + new_blocks 1 + wait_for_balance alice 0.9995 +fi + + if [[ $1 == "extract_preimage" ]]; then # Alice sends htlc1 to Bob. Bob sends htlc2 to Alice. # Neither one of them settles, they hold the htlcs, and Bob force-closes.