added txo_plot command to allow plotting txo sums over time

lbry/extras/daemon/daemon.py

@@ -4283,6 +4283,62 @@ class Daemon(metaclass=JSONRPCServerType):
             read_only=True, **self._constrain_txo_from_kwargs({}, **kwargs)
         )
 
+    @requires(WALLET_COMPONENT)
+    def jsonrpc_txo_plot(
+            self, account_id=None, wallet_id=None,
+            days_back=0, start_day=None, days_after=None, end_day=None, **kwargs):
+        """
+        Plot transaction output sum over days.
+
+        Usage:
+            txo_plot [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...]
+                     [--claim_id=<claim_id>...] [--name=<name>...] [--unspent]
+                     [--is_my_input_or_output |
+                         [[--is_my_output | --is_not_my_output] [--is_my_input | --is_not_my_input]]
+                     ]
+                     [--exclude_internal_transfers] [--wallet_id=<wallet_id>]
+                     [--days_back=<days_back> |
+                        [--start_day=<start_day> [--days_after=<days_after> | --end_day=<end_day>]]
+                     ]
+
+        Options:
+            --type=<type>              : (str or list) claim type: stream, channel, support,
+                                         purchase, collection, repost, other
+            --txid=<txid>              : (str or list) transaction id of outputs
+            --claim_id=<claim_id>      : (str or list) claim id
+            --name=<name>              : (str or list) claim name
+            --unspent                  : (bool) hide spent outputs, show only unspent ones
+            --is_my_input_or_output    : (bool) txos which have your inputs or your outputs,
+                                                if using this flag the other related flags
+                                                are ignored (--is_my_output, --is_my_input, etc)
+            --is_my_output             : (bool) show outputs controlled by you
+            --is_not_my_output         : (bool) show outputs not controlled by you
+            --is_my_input              : (bool) show outputs created by you
+            --is_not_my_input          : (bool) show outputs not created by you
+           --exclude_internal_transfers: (bool) excludes any outputs that are exactly this combination:
+                                                "--is_my_input --is_my_output --type=other"
+                                                this allows to exclude "change" payments, this
+                                                flag can be used in combination with any of the other flags
+            --account_id=<account_id>  : (str) id of the account to query
+            --wallet_id=<wallet_id>    : (str) restrict results to specific wallet
+            --days_back=<days_back>    : (int) number of days back from today
+                                               (not compatible with --start_day, --days_after, --end_day)
+            --start_day=<start_day>    : (date) start on specific date (YYYY-MM-DD)
+                                               (instead of --days_back)
+            --days_after=<days_after>  : (int) end number of days after --start_day
+                                               (instead of --end_day)
+            --end_day=<end_day>        : (date) end on specific date (YYYY-MM-DD)
+                                               (instead of --days_after)
+
+        Returns: List[Dict]
+        """
+        wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
+        return self.ledger.get_txo_plot(
+            wallet=wallet, accounts=[wallet.get_account_or_error(account_id)] if account_id else wallet.accounts,
+            read_only=True, days_back=days_back, start_day=start_day, days_after=days_after, end_day=end_day,
+            **self._constrain_txo_from_kwargs({}, **kwargs)
+        )
+
     UTXO_DOC = """
     Unspent transaction management.
     """

lbry/testcase.py

@@ -580,6 +580,9 @@ class CommandTestCase(IntegrationTestCase):
     async def txo_sum(self, *args, **kwargs):
         return await self.out(self.daemon.jsonrpc_txo_sum(*args, **kwargs))
 
+    async def txo_plot(self, *args, **kwargs):
+        return await self.out(self.daemon.jsonrpc_txo_plot(*args, **kwargs))
+
     async def claim_list(self, *args, **kwargs):
         return (await self.out(self.daemon.jsonrpc_claim_list(*args, **kwargs)))['items']
 

lbry/wallet/database.py

@@ -9,10 +9,12 @@ from contextvars import ContextVar
 from concurrent.futures.thread import ThreadPoolExecutor
 from concurrent.futures.process import ProcessPoolExecutor
 from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable, Dict, Optional
+from datetime import date
 
 from .bip32 import PubKey
 from .transaction import Transaction, Output, OutputScript, TXRefImmutable
 from .constants import TXO_TYPES, CLAIM_TYPES
+from .util import date_to_julian_day
 
 
 log = logging.getLogger(__name__)
@@ -254,6 +256,7 @@ def query(select, **constraints) -> Tuple[str, Dict[str, Any]]:
     limit = constraints.pop('limit', None)
     offset = constraints.pop('offset', None)
     order_by = constraints.pop('order_by', None)
+    group_by = constraints.pop('group_by', None)
 
     accounts = constraints.pop('accounts', [])
     if accounts:
@@ -264,6 +267,9 @@ def query(select, **constraints) -> Tuple[str, Dict[str, Any]]:
         sql.append('WHERE')
         sql.append(where)
 
+    if group_by is not None:
+        sql.append(f'GROUP BY {group_by}')
+
     if order_by:
         sql.append('ORDER BY')
         if isinstance(order_by, str):
@@ -387,7 +393,7 @@ def dict_row_factory(cursor, row):
 
 class Database(SQLiteMixin):
 
-    SCHEMA_VERSION = "1.2"
+    SCHEMA_VERSION = "1.3"
 
     PRAGMAS = """
         pragma journal_mode=WAL;
@@ -422,7 +428,8 @@ class Database(SQLiteMixin):
             height integer not null,
             position integer not null,
             is_verified boolean not null default 0,
-            purchased_claim_id text
+            purchased_claim_id text,
+            day integer
         );
         create index if not exists tx_purchased_claim_id_idx on tx (purchased_claim_id);
     """
@@ -506,14 +513,14 @@ class Database(SQLiteMixin):
             row['claim_name'] = txo.claim_name
         return row
 
-    @staticmethod
+    def tx_to_row(self, tx):
-    def tx_to_row(tx):
         row = {
             'txid': tx.id,
             'raw': sqlite3.Binary(tx.raw),
             'height': tx.height,
             'position': tx.position,
-            'is_verified': tx.is_verified
+            'is_verified': tx.is_verified,
+            'day': tx.get_julian_day(self.ledger),
         }
         txos = tx.outputs
         if len(txos) >= 2 and txos[1].can_decode_purchase_data:
@@ -863,6 +870,7 @@ class Database(SQLiteMixin):
         return txos
 
     def _clean_txo_constraints_for_aggregation(self, unspent, constraints):
+        constraints.pop('include_is_spent', None)
         constraints.pop('include_is_my_input', None)
         constraints.pop('include_is_my_output', None)
         constraints.pop('include_received_tips', None)
@@ -876,14 +884,36 @@ class Database(SQLiteMixin):
 
     async def get_txo_count(self, unspent=False, **constraints):
         self._clean_txo_constraints_for_aggregation(unspent, constraints)
-        count = await self.select_txos('COUNT(*) as total', **constraints)
+        count = await self.select_txos('COUNT(*) AS total', **constraints)
         return count[0]['total'] or 0
 
     async def get_txo_sum(self, unspent=False, **constraints):
         self._clean_txo_constraints_for_aggregation(unspent, constraints)
-        result = await self.select_txos('SUM(amount) as total', **constraints)
+        result = await self.select_txos('SUM(amount) AS total', **constraints)
         return result[0]['total'] or 0
 
+    async def get_txo_plot(
+            self, unspent=False, start_day=None, days_back=0, end_day=None, days_after=None, **constraints):
+        self._clean_txo_constraints_for_aggregation(unspent, constraints)
+        if start_day is None:
+            constraints['day__gte'] = self.ledger.headers.estimated_julian_day(
+                self.ledger.headers.height
+            ) - days_back
+        else:
+            constraints['day__gte'] = date_to_julian_day(
+                date.fromisoformat(start_day)
+            )
+            if end_day is not None:
+                constraints['day__lte'] = date_to_julian_day(
+                    date.fromisoformat(end_day)
+                )
+            elif days_after is not None:
+                constraints['day__lte'] = constraints['day__gte'] + days_after
+        return await self.select_txos(
+            "DATE(day) AS day, SUM(amount) AS total",
+            group_by='day', order_by='day', **constraints
+        )
+
     def get_utxos(self, read_only=False, **constraints):
         return self.get_txos(unspent=True, read_only=read_only, **constraints)
 

lbry/wallet/header.py

@@ -4,6 +4,7 @@ import struct
 import asyncio
 import logging
 import zlib
+from datetime import date
 from concurrent.futures.thread import ThreadPoolExecutor
 
 from io import BytesIO
@@ -11,7 +12,7 @@ from typing import Optional, Iterator, Tuple, Callable
 from binascii import hexlify, unhexlify
 
 from lbry.crypto.hash import sha512, double_sha256, ripemd160
-from lbry.wallet.util import ArithUint256
+from lbry.wallet.util import ArithUint256, date_to_julian_day
 from .checkpoints import HASHES
 
 
@@ -129,6 +130,9 @@ class Headers:
     def estimated_timestamp(self, height):
         return self.first_block_timestamp + (height * self.timestamp_average_offset)
 
+    def estimated_julian_day(self, height):
+        return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height)))
+
     async def get_raw_header(self, height) -> bytes:
         if self.chunk_getter:
             await self.ensure_chunk_at(height)

lbry/wallet/ledger.py

@@ -277,6 +277,9 @@ class Ledger(metaclass=LedgerRegistry):
     def get_txo_sum(self, **constraints):
         return self.db.get_txo_sum(**constraints)
 
+    def get_txo_plot(self, **constraints):
+        return self.db.get_txo_plot(**constraints)
+
     def get_transactions(self, **constraints):
         return self.db.get_transactions(**constraints)
 

lbry/wallet/transaction.py

@@ -4,6 +4,7 @@ import logging
 import typing
 from binascii import hexlify, unhexlify
 from typing import List, Iterable, Optional, Tuple
+from datetime import date
 
 import ecdsa
 from cryptography.hazmat.backends import default_backend
@@ -494,7 +495,7 @@ class Output(InputOutput):
 class Transaction:
 
     def __init__(self, raw=None, version: int = 1, locktime: int = 0, is_verified: bool = False,
-                 height: int = -2, position: int = -1) -> None:
+                 height: int = -2, position: int = -1, julian_day: int = None) -> None:
         self._raw = raw
         self._raw_sans_segwit = None
         self.is_segwit_flag = 0
@@ -512,6 +513,7 @@ class Transaction:
         # +num: confirmed in a specific block (height)
         self.height = height
         self.position = position
+        self._day = julian_day
         if raw is not None:
             self._deserialize()
 
@@ -535,6 +537,11 @@ class Transaction:
     def hash(self):
         return self.ref.hash
 
+    def get_julian_day(self, ledger):
+        if self._day is None and self.height > 0:
+            self._day = ledger.headers.estimated_julian_day(self.height)
+        return self._day
+
     @property
     def raw(self):
         if self._raw is None:

lbry/wallet/util.py

@@ -3,6 +3,10 @@ from typing import TypeVar, Sequence, Optional
 from .constants import COIN
 
 
+def date_to_julian_day(d):
+    return d.toordinal() + 1721424.5
+
+
 def coins_to_satoshis(coins):
     if not isinstance(coins, str):
         raise ValueError("{coins} must be a string")

tests/integration/blockchain/test_claim_commands.py

@@ -9,7 +9,7 @@ from lbry.error import InsufficientFundsError
 from lbry.extras.daemon.daemon import DEFAULT_PAGE_SIZE
 from lbry.testcase import CommandTestCase
 from lbry.wallet.transaction import Transaction
-from lbry.wallet.util import satoshis_to_coins as lbc
+from lbry.wallet.util import satoshis_to_coins as lbc, coins_to_satoshis as dewies
 
 
 log = logging.getLogger(__name__)
@@ -555,6 +555,61 @@ class TransactionOutputCommands(ClaimTestCase):
         r = await self.txo_list(is_not_my_output=True)
         self.assertEqual([sent_channel], r)
 
+    async def test_txo_plot(self):
+        day_blocks = int((24 * 60 * 60) / self.ledger.headers.timestamp_average_offset)
+        stream_id = self.get_claim_id(await self.stream_create())
+        await self.support_create(stream_id, '0.3')
+        await self.support_create(stream_id, '0.2')
+        await self.generate(day_blocks)
+        await self.support_create(stream_id, '0.4')
+        await self.support_create(stream_id, '0.5')
+        await self.generate(day_blocks)
+        await self.support_create(stream_id, '0.6')
+
+        plot = await self.txo_plot(type='support')
+        self.assertEqual([
+            {'day': '2016-06-25', 'total': dewies('0.6')},
+        ], plot)
+        plot = await self.txo_plot(type='support', days_back=1)
+        self.assertEqual([
+            {'day': '2016-06-24', 'total': dewies('0.9')},
+            {'day': '2016-06-25', 'total': dewies('0.6')},
+        ], plot)
+        plot = await self.txo_plot(type='support', days_back=2)
+        self.assertEqual([
+            {'day': '2016-06-23', 'total': dewies('0.5')},
+            {'day': '2016-06-24', 'total': dewies('0.9')},
+            {'day': '2016-06-25', 'total': dewies('0.6')},
+        ], plot)
+
+        plot = await self.txo_plot(type='support', start_day='2016-06-23')
+        self.assertEqual([
+            {'day': '2016-06-23', 'total': dewies('0.5')},
+            {'day': '2016-06-24', 'total': dewies('0.9')},
+            {'day': '2016-06-25', 'total': dewies('0.6')},
+        ], plot)
+        plot = await self.txo_plot(type='support', start_day='2016-06-24')
+        self.assertEqual([
+            {'day': '2016-06-24', 'total': dewies('0.9')},
+            {'day': '2016-06-25', 'total': dewies('0.6')},
+        ], plot)
+        plot = await self.txo_plot(type='support', start_day='2016-06-23', end_day='2016-06-24')
+        self.assertEqual([
+            {'day': '2016-06-23', 'total': dewies('0.5')},
+            {'day': '2016-06-24', 'total': dewies('0.9')},
+        ], plot)
+        plot = await self.txo_plot(type='support', start_day='2016-06-23', days_after=1)
+        self.assertEqual([
+            {'day': '2016-06-23', 'total': dewies('0.5')},
+            {'day': '2016-06-24', 'total': dewies('0.9')},
+        ], plot)
+        plot = await self.txo_plot(type='support', start_day='2016-06-23', days_after=2)
+        self.assertEqual([
+            {'day': '2016-06-23', 'total': dewies('0.5')},
+            {'day': '2016-06-24', 'total': dewies('0.9')},
+            {'day': '2016-06-25', 'total': dewies('0.6')},
+        ], plot)
+
 
 class ClaimCommands(ClaimTestCase):