regtests: add test "swapserver_server_skip_onchain_funding"

electrum/commands.py

@@ -71,7 +71,7 @@
 from .plugin import run_hook, DeviceMgr, Plugins
 from .version import ELECTRUM_VERSION
 from .simple_config import SimpleConfig
-from .fee_policy import FeePolicy
+from .fee_policy import FeePolicy, FEE_ETA_TARGETS, FEERATE_DEFAULT_RELAY
 from . import GuiImportError
 from . import crypto
 from . import constants
@@ -1577,6 +1577,27 @@ async def getfeerate(self):
             'tooltip': tooltip,
         }
 
+    @command('n')
+    async def test_inject_fee_etas(self, fee_est):
+        """
+        Inject fee estimates into the network object, as if they were coming from connected servers.
+        Useful on regtest.
+
+        arg:str:fee_est:dict of ETA-based fee estimates, encoded as str
+        """
+        if not isinstance(fee_est, dict):
+            fee_est = ast.literal_eval(fee_est)
+        assert isinstance(fee_est, dict), f"unexpected type for fee_est. got {repr(fee_est)}"
+        # populate missing high-block-number estimates using default relay fee.
+        # e.g. {"25": 2222} -> {"25": 2222, "144": 1000, "1008": 1000}
+        furthest_estimate = max(fee_est.keys()) if fee_est else 0
+        further_fee_est = {
+            eta_target: FEERATE_DEFAULT_RELAY for eta_target in FEE_ETA_TARGETS
+            if eta_target > furthest_estimate
+        }
+        fee_est.update(further_fee_est)
+        self.network.update_fee_estimates(fee_est=fee_est)
+
     @command('w')
     async def removelocaltx(self, txid, wallet: Abstract_Wallet = None):
         """Remove a 'local' transaction from the wallet, and its dependent
@@ -1622,7 +1643,8 @@ async def add_peer(self, connection_string, timeout=20, gossip=False, wallet: Ab
         arg:int:timeout:Timeout in seconds (default=20)
         """
         lnworker = self.network.lngossip if gossip else wallet.lnworker
-        await lnworker.add_peer(connection_string)
+        async with util.async_timeout(timeout):
+            await lnworker.add_peer(connection_string)
         return True
 
     @command('wnl')
@@ -1698,19 +1720,20 @@ async def decode_invoice(self, invoice: str):
         return invoice.to_debug_json()
 
     @command('wnpl')
-    async def lnpay(self, invoice, timeout=120, password=None, wallet: Abstract_Wallet = None):
+    async def lnpay(self, invoice, timeout=0, password=None, wallet: Abstract_Wallet = None):
         """
         Pay a lightning invoice
 
         arg:str:invoice:Lightning invoice (bolt 11)
-        arg:int:timeout:Timeout in seconds (default=20)
+        arg:int:timeout:Timeout in seconds. Note: *not* safe to try paying same invoice multiple times with a timeout.
         """
         lnworker = wallet.lnworker
         lnaddr = lnworker._check_bolt11_invoice(invoice)
         payment_hash = lnaddr.paymenthash
         invoice_obj = Invoice.from_bech32(invoice)
         wallet.save_invoice(invoice_obj)
-        success, log = await lnworker.pay_invoice(invoice_obj)
+        async with util.async_timeout(timeout or None):
+            success, log = await lnworker.pay_invoice(invoice_obj)
         return {
             'payment_hash': payment_hash.hex(),
             'success': success,

electrum/fee_policy.py

@@ -375,8 +375,9 @@ def has_data(self) -> bool:
               just try to do the estimate and handle a potential None result. That way,
               estimation works for targets we have, even if some targets are missing.
         """
-        # we do not request estimate for next block fee, hence -1
-        return len(self.data) == len(FEE_ETA_TARGETS) - 1
+        targets = set(FEE_ETA_TARGETS)
+        targets.discard(1)  # rm "next block" target
+        return all(target in self.data for target in targets)
 
     def set_data(self, nblock_target: int, fee_per_kb: int):
         assert isinstance(nblock_target, int), f"expected int, got {nblock_target!r}"

electrum/lnwatcher.py

@@ -5,7 +5,7 @@
 from typing import TYPE_CHECKING, Optional
 
 from . import util
-from .util import TxMinedInfo, BelowDustLimit
+from .util import TxMinedInfo, BelowDustLimit, NoDynamicFeeEstimates
 from .util import EventListener, event_listener, log_exceptions, ignore_exceptions
 from .transaction import Transaction, TxOutpoint
 from .logging import Logger
@@ -99,6 +99,7 @@ async def check_onchain_situation(self, address: str, funding_outpoint: str) ->
         if not self.adb.is_mine(address):
             return
         # inspect_tx_candidate might have added new addresses, in which case we return early
+        # note: maybe we should wait until adb.is_up_to_date... (?)
         funding_txid = funding_outpoint.split(':')[0]
         funding_height = self.adb.get_tx_height(funding_txid)
         closing_txid = self.adb.get_spender(funding_outpoint)
@@ -159,7 +160,9 @@ async def sweep_commitment_transaction(self, funding_outpoint: str, closing_tx:
             return False
         # detect who closed and get information about how to claim outputs
         is_local_ctx, sweep_info_dict = chan.get_ctx_sweep_info(closing_tx)
-        keep_watching = False if sweep_info_dict else not self.adb.is_deeply_mined(closing_tx.txid())
+        # note: we need to keep watching *at least* until the closing tx is deeply mined,
+        #       possibly longer if there are TXOs to sweep
+        keep_watching = not self.adb.is_deeply_mined(closing_tx.txid())
         # create and broadcast transactions
         for prevout, sweep_info in sweep_info_dict.items():
             prev_txid, prev_index = prevout.split(':')
@@ -169,38 +172,44 @@ async def sweep_commitment_transaction(self, funding_outpoint: str, closing_tx:
                 # do not keep watching if prevout does not exist
                 self.logger.info(f'prevout does not exist for {name}: {prevout}')
                 continue
-            was_added = self.maybe_redeem(sweep_info)
-            spender_txid = self.adb.get_spender(prevout)
+            watch_sweep_info = self.maybe_redeem(sweep_info)
+            spender_txid = self.adb.get_spender(prevout)  # note: LOCAL spenders don't count
             spender_tx = self.adb.get_transaction(spender_txid) if spender_txid else None
             if spender_tx:
                 # the spender might be the remote, revoked or not
                 htlc_sweepinfo = chan.maybe_sweep_htlcs(closing_tx, spender_tx)
                 for prevout2, htlc_sweep_info in htlc_sweepinfo.items():
-                    htlc_was_added = self.maybe_redeem(htlc_sweep_info)
+                    watch_htlc_sweep_info = self.maybe_redeem(htlc_sweep_info)
                     htlc_tx_spender = self.adb.get_spender(prevout2)
                     self.lnworker.wallet.set_default_label(prevout2, htlc_sweep_info.name)
                     if htlc_tx_spender:
                         keep_watching |= not self.adb.is_deeply_mined(htlc_tx_spender)
                         self.maybe_add_accounting_address(htlc_tx_spender, htlc_sweep_info)
                     else:
-                        keep_watching |= htlc_was_added
+                        keep_watching |= watch_htlc_sweep_info
                 keep_watching |= not self.adb.is_deeply_mined(spender_txid)
                 self.maybe_extract_preimage(chan, spender_tx, prevout)
                 self.maybe_add_accounting_address(spender_txid, sweep_info)
             else:
-                keep_watching |= was_added
+                keep_watching |= watch_sweep_info
             self.maybe_add_pending_forceclose(
-                chan=chan, spender_txid=spender_txid, is_local_ctx=is_local_ctx, sweep_info=sweep_info, was_added=was_added)
+                chan=chan, spender_txid=spender_txid, is_local_ctx=is_local_ctx, sweep_info=sweep_info)
         return keep_watching
 
     def get_pending_force_closes(self):
         return self._pending_force_closes
 
     def maybe_redeem(self, sweep_info: 'SweepInfo') -> bool:
-        """ returns False if it was dust """
+        """ returns 'keep_watching' """
         try:
             self.lnworker.wallet.txbatcher.add_sweep_input('lnwatcher', sweep_info)
         except BelowDustLimit:
+            # utxo is considered dust at *current* fee estimates.
+            # but maybe the fees atm are very high? We will retry later.
+            pass
+        except NoDynamicFeeEstimates:
+            pass  # will retry later
+        if sweep_info.is_anchor():
             return False
         return True
 
@@ -251,10 +260,9 @@ def maybe_add_pending_forceclose(
         spender_txid: Optional[str],
         is_local_ctx: bool,
         sweep_info: 'SweepInfo',
-        was_added: bool,
     ):
         """ we are waiting for ctx to be confirmed and there are received htlcs """
-        if was_added and is_local_ctx and sweep_info.name == 'received-htlc' and chan.has_anchors():
+        if is_local_ctx and sweep_info.name == 'received-htlc' and chan.has_anchors():
             tx_mined_status = self.adb.get_tx_height(spender_txid)
             if tx_mined_status.height == TX_HEIGHT_LOCAL:
                 self._pending_force_closes.add(chan)

electrum/plugins/watchtower/watchtower.py

@@ -202,9 +202,10 @@ def inspect_tx_candidate(self, outpoint, n: int) -> Dict[str, str]:
                 result.update(r)
         return result
 
-    async def sweep_commitment_transaction(self, funding_outpoint, closing_tx):
+    async def sweep_commitment_transaction(self, funding_outpoint: str, closing_tx: Transaction) -> bool:
+        assert closing_tx
         spenders = self.inspect_tx_candidate(funding_outpoint, 0)
-        keep_watching = False
+        keep_watching = not self.adb.is_deeply_mined(closing_tx.txid())
         for prevout, spender in spenders.items():
             if spender is not None:
                 keep_watching |= not self.adb.is_deeply_mined(spender)

electrum/simple_config.py

@@ -891,6 +891,7 @@ def __setattr__(self, name, value):
     # connect to remote submarine swap server
     SWAPSERVER_URL = ConfigVar('swapserver_url', default='', type_=str)
     TEST_SWAPSERVER_REFUND = ConfigVar('test_swapserver_refund', default=False, type_=bool)
+    TEST_SWAPSERVER_SKIP_ONCHAIN_FUNDING = ConfigVar('test_swapserver_skip_onchain_funding', default=False, type_=bool)
     SWAPSERVER_NPUB = ConfigVar('swapserver_npub', default=None, type_=str)
     SWAPSERVER_POW_TARGET = ConfigVar('swapserver_pow_target', default=30, type_=int)
 

electrum/submarine_swaps.py

@@ -32,7 +32,7 @@
 from .util import (
     log_exceptions, ignore_exceptions, BelowDustLimit, OldTaskGroup, ca_path, gen_nostr_ann_pow,
     get_nostr_ann_pow_amount, make_aiohttp_proxy_connector, get_running_loop, get_asyncio_loop, wait_for2,
-    run_sync_function_on_asyncio_thread, trigger_callback
+    run_sync_function_on_asyncio_thread, trigger_callback, NoDynamicFeeEstimates
 )
 from . import lnutil
 from .lnutil import hex_to_bytes, REDEEM_AFTER_DOUBLE_SPENT_DELAY, Keypair
@@ -485,6 +485,9 @@ async def _claim_swap(self, swap: SwapData) -> None:
             except BelowDustLimit:
                 self.logger.info('utxo value below dust threshold')
                 return
+            except NoDynamicFeeEstimates:
+                self.logger.info('got NoDynamicFeeEstimates')
+                return
 
     def get_fee_for_txbatcher(self):
         return self._get_tx_fee(self.config.FEE_POLICY_SWAPS)
@@ -512,6 +515,8 @@ async def hold_invoice_callback(self, payment_hash: bytes) -> None:
         key = payment_hash.hex()
         if swap := self._swaps.get(key):
             if not swap.is_funded():
+                if self.network.config.TEST_SWAPSERVER_SKIP_ONCHAIN_FUNDING:  # for testing
+                    return
                 output = self.create_funding_output(swap)
                 self.wallet.txbatcher.add_payment_output('swaps', output)
                 swap._payment_pending = True

electrum/txbatcher.py

@@ -99,6 +99,7 @@ def add_payment_output(self, key: str, output: 'PartialTxOutput') -> None:
 
     @locked
     def add_sweep_input(self, key: str, sweep_info: 'SweepInfo') -> None:
+        """Can raise BelowDustLimit or NoDynamicFeeEstimates."""
         if sweep_info.txin and sweep_info.txout:
             # detect legacy htlc using name and csv delay
             if sweep_info.name in ['received-htlc', 'offered-htlc'] and sweep_info.csv_delay == 0:
@@ -263,20 +264,25 @@ def add_payment_output(self, output: 'PartialTxOutput') -> None:
         self.batch_payments.append(output)
 
     def is_dust(self, sweep_info: SweepInfo) -> bool:
+        """Can raise BelowDustLimit or NoDynamicFeeEstimates."""
         if sweep_info.is_anchor():
             return False
         if sweep_info.txout is not None:
             return False
-        value = sweep_info.txin._trusted_value_sats
+        value = sweep_info.txin.value_sats()
         witness_size = len(sweep_info.txin.make_witness(71*b'\x00'))
         tx_size_vbytes = 84 + witness_size//4     # assumes no batching, sweep to p2wpkh
         self.logger.info(f'{sweep_info.name} size = {tx_size_vbytes}')
-        fee = self.fee_policy.estimate_fee(tx_size_vbytes, network=self.wallet.network, allow_fallback_to_static_rates=True)
+        fee = self.fee_policy.estimate_fee(tx_size_vbytes, network=self.wallet.network)
         return value - fee <= dust_threshold()
 
     @locked
     def add_sweep_input(self, sweep_info: 'SweepInfo') -> None:
+        """Can raise BelowDustLimit or NoDynamicFeeEstimates."""
         if self.is_dust(sweep_info):
+            # note: this uses the current fee estimates. Just because something is dust
+            #       at the current fee levels, if fees go down, it might still become
+            #       worthwhile to sweep. So callers might want to retry later.
             raise BelowDustLimit
         txin = sweep_info.txin
         if txin.prevout in self._unconfirmed_sweeps:

tests/regtest.py

@@ -82,6 +82,9 @@ def test_breach_with_unspent_htlc(self):
     def test_breach_with_spent_htlc(self):
         self.run_shell(['breach_with_spent_htlc'])
 
+    def test_lnwatcher_waits_until_fees_go_down(self):
+        self.run_shell(['lnwatcher_waits_until_fees_go_down'])
+
 
 class TestLightningSwapserver(TestLightning):
     agents = {
@@ -107,6 +110,9 @@ def test_swapserver_forceclose(self):
     def test_swapserver_refund(self):
         self.run_shell(['swapserver_refund'])
 
+    def test_swapserver_server_skip_onchain_funding(self):
+        self.run_shell(['swapserver_server_skip_onchain_funding'])
+
 
 
 class TestLightningWatchtower(TestLightning):