diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index 6f31b5e6a..4e285f88f 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -4283,6 +4283,62 @@ class Daemon(metaclass=JSONRPCServerType): read_only=True, **self._constrain_txo_from_kwargs({}, **kwargs) ) + @requires(WALLET_COMPONENT) + def jsonrpc_txo_plot( + self, account_id=None, wallet_id=None, + days_back=0, start_day=None, days_after=None, end_day=None, **kwargs): + """ + Plot transaction output sum over days. + + Usage: + txo_plot [--account_id=] [--type=...] [--txid=...] + [--claim_id=...] [--name=...] [--unspent] + [--is_my_input_or_output | + [[--is_my_output | --is_not_my_output] [--is_my_input | --is_not_my_input]] + ] + [--exclude_internal_transfers] [--wallet_id=] + [--days_back= | + [--start_day= [--days_after= | --end_day=]] + ] + + Options: + --type= : (str or list) claim type: stream, channel, support, + purchase, collection, repost, other + --txid= : (str or list) transaction id of outputs + --claim_id= : (str or list) claim id + --name= : (str or list) claim name + --unspent : (bool) hide spent outputs, show only unspent ones + --is_my_input_or_output : (bool) txos which have your inputs or your outputs, + if using this flag the other related flags + are ignored (--is_my_output, --is_my_input, etc) + --is_my_output : (bool) show outputs controlled by you + --is_not_my_output : (bool) show outputs not controlled by you + --is_my_input : (bool) show outputs created by you + --is_not_my_input : (bool) show outputs not created by you + --exclude_internal_transfers: (bool) excludes any outputs that are exactly this combination: + "--is_my_input --is_my_output --type=other" + this allows to exclude "change" payments, this + flag can be used in combination with any of the other flags + --account_id= : (str) id of the account to query + --wallet_id= : (str) restrict results to specific wallet + --days_back= : (int) number of days back from today + (not compatible with --start_day, --days_after, --end_day) + --start_day= : (date) start on specific date (YYYY-MM-DD) + (instead of --days_back) + --days_after= : (int) end number of days after --start_day + (instead of --end_day) + --end_day= : (date) end on specific date (YYYY-MM-DD) + (instead of --days_after) + + Returns: List[Dict] + """ + wallet = self.wallet_manager.get_wallet_or_default(wallet_id) + return self.ledger.get_txo_plot( + wallet=wallet, accounts=[wallet.get_account_or_error(account_id)] if account_id else wallet.accounts, + read_only=True, days_back=days_back, start_day=start_day, days_after=days_after, end_day=end_day, + **self._constrain_txo_from_kwargs({}, **kwargs) + ) + UTXO_DOC = """ Unspent transaction management. """ diff --git a/lbry/testcase.py b/lbry/testcase.py index 56c9cb4c6..c0f8fa449 100644 --- a/lbry/testcase.py +++ b/lbry/testcase.py @@ -580,6 +580,9 @@ class CommandTestCase(IntegrationTestCase): async def txo_sum(self, *args, **kwargs): return await self.out(self.daemon.jsonrpc_txo_sum(*args, **kwargs)) + async def txo_plot(self, *args, **kwargs): + return await self.out(self.daemon.jsonrpc_txo_plot(*args, **kwargs)) + async def claim_list(self, *args, **kwargs): return (await self.out(self.daemon.jsonrpc_claim_list(*args, **kwargs)))['items'] diff --git a/lbry/wallet/database.py b/lbry/wallet/database.py index 218e54cc5..949442247 100644 --- a/lbry/wallet/database.py +++ b/lbry/wallet/database.py @@ -9,10 +9,12 @@ from contextvars import ContextVar from concurrent.futures.thread import ThreadPoolExecutor from concurrent.futures.process import ProcessPoolExecutor from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable, Dict, Optional +from datetime import date from .bip32 import PubKey from .transaction import Transaction, Output, OutputScript, TXRefImmutable from .constants import TXO_TYPES, CLAIM_TYPES +from .util import date_to_julian_day log = logging.getLogger(__name__) @@ -254,6 +256,7 @@ def query(select, **constraints) -> Tuple[str, Dict[str, Any]]: limit = constraints.pop('limit', None) offset = constraints.pop('offset', None) order_by = constraints.pop('order_by', None) + group_by = constraints.pop('group_by', None) accounts = constraints.pop('accounts', []) if accounts: @@ -264,6 +267,9 @@ def query(select, **constraints) -> Tuple[str, Dict[str, Any]]: sql.append('WHERE') sql.append(where) + if group_by is not None: + sql.append(f'GROUP BY {group_by}') + if order_by: sql.append('ORDER BY') if isinstance(order_by, str): @@ -387,7 +393,7 @@ def dict_row_factory(cursor, row): class Database(SQLiteMixin): - SCHEMA_VERSION = "1.2" + SCHEMA_VERSION = "1.3" PRAGMAS = """ pragma journal_mode=WAL; @@ -422,7 +428,8 @@ class Database(SQLiteMixin): height integer not null, position integer not null, is_verified boolean not null default 0, - purchased_claim_id text + purchased_claim_id text, + day integer ); create index if not exists tx_purchased_claim_id_idx on tx (purchased_claim_id); """ @@ -506,14 +513,14 @@ class Database(SQLiteMixin): row['claim_name'] = txo.claim_name return row - @staticmethod - def tx_to_row(tx): + def tx_to_row(self, tx): row = { 'txid': tx.id, 'raw': sqlite3.Binary(tx.raw), 'height': tx.height, 'position': tx.position, - 'is_verified': tx.is_verified + 'is_verified': tx.is_verified, + 'day': tx.get_julian_day(self.ledger), } txos = tx.outputs if len(txos) >= 2 and txos[1].can_decode_purchase_data: @@ -863,6 +870,7 @@ class Database(SQLiteMixin): return txos def _clean_txo_constraints_for_aggregation(self, unspent, constraints): + constraints.pop('include_is_spent', None) constraints.pop('include_is_my_input', None) constraints.pop('include_is_my_output', None) constraints.pop('include_received_tips', None) @@ -876,14 +884,36 @@ class Database(SQLiteMixin): async def get_txo_count(self, unspent=False, **constraints): self._clean_txo_constraints_for_aggregation(unspent, constraints) - count = await self.select_txos('COUNT(*) as total', **constraints) + count = await self.select_txos('COUNT(*) AS total', **constraints) return count[0]['total'] or 0 async def get_txo_sum(self, unspent=False, **constraints): self._clean_txo_constraints_for_aggregation(unspent, constraints) - result = await self.select_txos('SUM(amount) as total', **constraints) + result = await self.select_txos('SUM(amount) AS total', **constraints) return result[0]['total'] or 0 + async def get_txo_plot( + self, unspent=False, start_day=None, days_back=0, end_day=None, days_after=None, **constraints): + self._clean_txo_constraints_for_aggregation(unspent, constraints) + if start_day is None: + constraints['day__gte'] = self.ledger.headers.estimated_julian_day( + self.ledger.headers.height + ) - days_back + else: + constraints['day__gte'] = date_to_julian_day( + date.fromisoformat(start_day) + ) + if end_day is not None: + constraints['day__lte'] = date_to_julian_day( + date.fromisoformat(end_day) + ) + elif days_after is not None: + constraints['day__lte'] = constraints['day__gte'] + days_after + return await self.select_txos( + "DATE(day) AS day, SUM(amount) AS total", + group_by='day', order_by='day', **constraints + ) + def get_utxos(self, read_only=False, **constraints): return self.get_txos(unspent=True, read_only=read_only, **constraints) diff --git a/lbry/wallet/header.py b/lbry/wallet/header.py index 6af943e3b..d8ea47665 100644 --- a/lbry/wallet/header.py +++ b/lbry/wallet/header.py @@ -4,6 +4,7 @@ import struct import asyncio import logging import zlib +from datetime import date from concurrent.futures.thread import ThreadPoolExecutor from io import BytesIO @@ -11,7 +12,7 @@ from typing import Optional, Iterator, Tuple, Callable from binascii import hexlify, unhexlify from lbry.crypto.hash import sha512, double_sha256, ripemd160 -from lbry.wallet.util import ArithUint256 +from lbry.wallet.util import ArithUint256, date_to_julian_day from .checkpoints import HASHES @@ -129,6 +130,9 @@ class Headers: def estimated_timestamp(self, height): return self.first_block_timestamp + (height * self.timestamp_average_offset) + def estimated_julian_day(self, height): + return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height))) + async def get_raw_header(self, height) -> bytes: if self.chunk_getter: await self.ensure_chunk_at(height) diff --git a/lbry/wallet/ledger.py b/lbry/wallet/ledger.py index ebdfc7612..7563e3270 100644 --- a/lbry/wallet/ledger.py +++ b/lbry/wallet/ledger.py @@ -277,6 +277,9 @@ class Ledger(metaclass=LedgerRegistry): def get_txo_sum(self, **constraints): return self.db.get_txo_sum(**constraints) + def get_txo_plot(self, **constraints): + return self.db.get_txo_plot(**constraints) + def get_transactions(self, **constraints): return self.db.get_transactions(**constraints) diff --git a/lbry/wallet/transaction.py b/lbry/wallet/transaction.py index 693ccc91b..a7ad79b43 100644 --- a/lbry/wallet/transaction.py +++ b/lbry/wallet/transaction.py @@ -4,6 +4,7 @@ import logging import typing from binascii import hexlify, unhexlify from typing import List, Iterable, Optional, Tuple +from datetime import date import ecdsa from cryptography.hazmat.backends import default_backend @@ -494,7 +495,7 @@ class Output(InputOutput): class Transaction: def __init__(self, raw=None, version: int = 1, locktime: int = 0, is_verified: bool = False, - height: int = -2, position: int = -1) -> None: + height: int = -2, position: int = -1, julian_day: int = None) -> None: self._raw = raw self._raw_sans_segwit = None self.is_segwit_flag = 0 @@ -512,6 +513,7 @@ class Transaction: # +num: confirmed in a specific block (height) self.height = height self.position = position + self._day = julian_day if raw is not None: self._deserialize() @@ -535,6 +537,11 @@ class Transaction: def hash(self): return self.ref.hash + def get_julian_day(self, ledger): + if self._day is None and self.height > 0: + self._day = ledger.headers.estimated_julian_day(self.height) + return self._day + @property def raw(self): if self._raw is None: diff --git a/lbry/wallet/util.py b/lbry/wallet/util.py index a9504bff1..cb57bc694 100644 --- a/lbry/wallet/util.py +++ b/lbry/wallet/util.py @@ -3,6 +3,10 @@ from typing import TypeVar, Sequence, Optional from .constants import COIN +def date_to_julian_day(d): + return d.toordinal() + 1721424.5 + + def coins_to_satoshis(coins): if not isinstance(coins, str): raise ValueError("{coins} must be a string") diff --git a/tests/integration/blockchain/test_claim_commands.py b/tests/integration/blockchain/test_claim_commands.py index 1156917c5..6fee0bd67 100644 --- a/tests/integration/blockchain/test_claim_commands.py +++ b/tests/integration/blockchain/test_claim_commands.py @@ -9,7 +9,7 @@ from lbry.error import InsufficientFundsError from lbry.extras.daemon.daemon import DEFAULT_PAGE_SIZE from lbry.testcase import CommandTestCase from lbry.wallet.transaction import Transaction -from lbry.wallet.util import satoshis_to_coins as lbc +from lbry.wallet.util import satoshis_to_coins as lbc, coins_to_satoshis as dewies log = logging.getLogger(__name__) @@ -555,6 +555,61 @@ class TransactionOutputCommands(ClaimTestCase): r = await self.txo_list(is_not_my_output=True) self.assertEqual([sent_channel], r) + async def test_txo_plot(self): + day_blocks = int((24 * 60 * 60) / self.ledger.headers.timestamp_average_offset) + stream_id = self.get_claim_id(await self.stream_create()) + await self.support_create(stream_id, '0.3') + await self.support_create(stream_id, '0.2') + await self.generate(day_blocks) + await self.support_create(stream_id, '0.4') + await self.support_create(stream_id, '0.5') + await self.generate(day_blocks) + await self.support_create(stream_id, '0.6') + + plot = await self.txo_plot(type='support') + self.assertEqual([ + {'day': '2016-06-25', 'total': dewies('0.6')}, + ], plot) + plot = await self.txo_plot(type='support', days_back=1) + self.assertEqual([ + {'day': '2016-06-24', 'total': dewies('0.9')}, + {'day': '2016-06-25', 'total': dewies('0.6')}, + ], plot) + plot = await self.txo_plot(type='support', days_back=2) + self.assertEqual([ + {'day': '2016-06-23', 'total': dewies('0.5')}, + {'day': '2016-06-24', 'total': dewies('0.9')}, + {'day': '2016-06-25', 'total': dewies('0.6')}, + ], plot) + + plot = await self.txo_plot(type='support', start_day='2016-06-23') + self.assertEqual([ + {'day': '2016-06-23', 'total': dewies('0.5')}, + {'day': '2016-06-24', 'total': dewies('0.9')}, + {'day': '2016-06-25', 'total': dewies('0.6')}, + ], plot) + plot = await self.txo_plot(type='support', start_day='2016-06-24') + self.assertEqual([ + {'day': '2016-06-24', 'total': dewies('0.9')}, + {'day': '2016-06-25', 'total': dewies('0.6')}, + ], plot) + plot = await self.txo_plot(type='support', start_day='2016-06-23', end_day='2016-06-24') + self.assertEqual([ + {'day': '2016-06-23', 'total': dewies('0.5')}, + {'day': '2016-06-24', 'total': dewies('0.9')}, + ], plot) + plot = await self.txo_plot(type='support', start_day='2016-06-23', days_after=1) + self.assertEqual([ + {'day': '2016-06-23', 'total': dewies('0.5')}, + {'day': '2016-06-24', 'total': dewies('0.9')}, + ], plot) + plot = await self.txo_plot(type='support', start_day='2016-06-23', days_after=2) + self.assertEqual([ + {'day': '2016-06-23', 'total': dewies('0.5')}, + {'day': '2016-06-24', 'total': dewies('0.9')}, + {'day': '2016-06-25', 'total': dewies('0.6')}, + ], plot) + class ClaimCommands(ClaimTestCase):