From 6dbea6f4ab50a9c3d8ecbd15d64d69acc90588ae Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 19 May 2020 18:42:56 -0400 Subject: [PATCH] add sqlite coin chooser, set it as the default coin selection strategy --- lbry/conf.py | 2 +- lbry/extras/daemon/daemon.py | 2 +- lbry/wallet/coinselection.py | 2 +- lbry/wallet/database.py | 65 +++++++++++++++++++++++++++++++++++- lbry/wallet/ledger.py | 11 ++++-- 5 files changed, 75 insertions(+), 7 deletions(-) diff --git a/lbry/conf.py b/lbry/conf.py index 6945e537a..220e5bda9 100644 --- a/lbry/conf.py +++ b/lbry/conf.py @@ -634,7 +634,7 @@ class Config(CLIConfig): coin_selection_strategy = StringChoice( "Strategy to use when selecting UTXOs for a transaction", - STRATEGIES, "standard") + STRATEGIES, "sqlite") transaction_cache_size = Integer("Transaction cache size", 100_000) save_resolved_claims = Toggle( diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index 756ff6bc7..3fabb5ddf 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -3269,7 +3269,7 @@ class Daemon(metaclass=JSONRPCServerType): self.component_manager.loop.create_task(self.analytics_manager.send_claim_action('publish')) else: await account.ledger.release_tx(tx) - + log.info("successful publish %s", tx.id) return tx @requires(WALLET_COMPONENT, FILE_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT) diff --git a/lbry/wallet/coinselection.py b/lbry/wallet/coinselection.py index 63bbb6977..182c6d0d4 100644 --- a/lbry/wallet/coinselection.py +++ b/lbry/wallet/coinselection.py @@ -5,7 +5,7 @@ from lbry.wallet.transaction import OutputEffectiveAmountEstimator MAXIMUM_TRIES = 100000 -STRATEGIES = [] +STRATEGIES = ['sqlite'] # sqlite coin chooser is in database.py def strategy(method): diff --git a/lbry/wallet/database.py b/lbry/wallet/database.py index 15c866017..9430d87f3 100644 --- a/lbry/wallet/database.py +++ b/lbry/wallet/database.py @@ -4,6 +4,7 @@ import asyncio import sqlite3 import platform from binascii import hexlify +from collections import defaultdict from dataclasses import dataclass from contextvars import ContextVar from concurrent.futures.thread import ThreadPoolExecutor @@ -14,7 +15,7 @@ from prometheus_client import Gauge, Counter, Histogram from lbry.utils import LockWithMetrics from .bip32 import PubKey -from .transaction import Transaction, Output, OutputScript, TXRefImmutable +from .transaction import Transaction, Output, OutputScript, TXRefImmutable, Input from .constants import TXO_TYPES, CLAIM_TYPES from .util import date_to_julian_day @@ -466,6 +467,55 @@ def dict_row_factory(cursor, row): return d +def get_spendable_utxos(transaction: sqlite3.Connection, accounts: List, reserve_amount: int, floor: int, + fee_per_byte: int): + txs = defaultdict(list) + decoded_transactions = {} + + accumulated = 0 + multiplier = 10 + gap_count = 0 + accounts_fmt = ",".join(["?"] * len(accounts)) + txo_query = f""" + SELECT tx.txid, txo.txoid, tx.raw, tx.height, txo.position as nout, tx.is_verified, txo.amount FROM txo + INNER JOIN account_address USING (address) + LEFT JOIN txi USING (txoid) + INNER JOIN tx USING (txid) + WHERE txo.txo_type=0 AND txi.txoid IS NULL AND tx.txid IS NOT NULL AND NOT txo.is_reserved + AND txo.amount BETWEEN ? AND ? + """ + if accounts: + txo_query += f""" + AND account_address.account {'= ?' if len(accounts_fmt) == 1 else 'IN (' + accounts_fmt + ')'} + """ + reserved = [] + while accumulated < reserve_amount: + found_txs = False + for row in transaction.execute(txo_query, (floor, floor * multiplier, *accounts)): + (txid, txoid, raw, height, nout, verified, amount) = row.values() + found_txs = True + if txid not in decoded_transactions: + decoded_transactions[txid] = Transaction(raw) + decoded_tx = decoded_transactions[txid] + accumulated += amount + accumulated -= Input.spend(decoded_tx.outputs[nout]).size * fee_per_byte + txs[(raw, height, verified)].append(nout) + reserved.append(txoid) + if accumulated >= reserve_amount: + break + if not found_txs: + gap_count += 1 + if gap_count == 5: + break + floor *= multiplier + # reserve the accumulated txos if enough were found + if accumulated >= reserve_amount: + transaction.executemany("UPDATE txo SET is_reserved = ? WHERE txoid = ?", + [(True, txoid) for txoid in reserved]).fetchall() + + return txs + + class Database(SQLiteMixin): SCHEMA_VERSION = "1.3" @@ -666,6 +716,19 @@ class Database(SQLiteMixin): # 2. update address histories removing deleted TXs return True + async def get_spendable_utxos(self, ledger, reserve_amount, accounts: Optional[Iterable], min_amount: int = 100000, + fee_per_byte: int = 50) -> List: + to_spend = await self.db.run( + get_spendable_utxos, tuple(account.id for account in accounts), reserve_amount, min_amount, + fee_per_byte + ) + txos = [] + for (raw, height, verified), positions in to_spend.items(): + tx = Transaction(raw, height=height, is_verified=verified) + for nout in positions: + txos.append(tx.outputs[nout].get_estimator(ledger)) + return txos + async def select_transactions(self, cols, accounts=None, read_only=False, **constraints): if not {'txid', 'txid__in'}.intersection(constraints): assert accounts, "'accounts' argument required when no 'txid' constraint is present" diff --git a/lbry/wallet/ledger.py b/lbry/wallet/ledger.py index 50adf7467..194f1baa7 100644 --- a/lbry/wallet/ledger.py +++ b/lbry/wallet/ledger.py @@ -244,11 +244,16 @@ class Ledger(metaclass=LedgerRegistry): def get_address_count(self, **constraints): return self.db.get_address_count(**constraints) - async def get_spendable_utxos(self, amount: int, funding_accounts): + async def get_spendable_utxos(self, amount: int, funding_accounts: Optional[Iterable['Account']], + min_amount=100000): + min_amount = min(amount // 10, min_amount) + fee = Output.pay_pubkey_hash(COIN, NULL_HASH32).get_fee(self) + selector = CoinSelector(amount, fee) async with self._utxo_reservation_lock: + if self.coin_selection_strategy == 'sqlite': + return await self.db.get_spendable_utxos(self, amount + fee, funding_accounts, min_amount=min_amount, + fee_per_byte=self.fee_per_byte) txos = await self.get_effective_amount_estimators(funding_accounts) - fee = Output.pay_pubkey_hash(COIN, NULL_HASH32).get_fee(self) - selector = CoinSelector(amount, fee) spendables = selector.select(txos, self.coin_selection_strategy) if spendables: await self.reserve_outputs(s.txo for s in spendables)