LBRYCommunity/lbry-sdk
1
0
Fork 1
Code Issues Pull requests Projects Releases 282 Packages Wiki Activity

Merge pull request #2884 from lbryio/txo_plot

Browse source 
added `txo_plot` command to allow plotting txo sums over time
This commit is contained in:
Lex Berezhny 2020-03-26 10:14:02 -04:00 committed by GitHub
commit b0aee3d335
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 173 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
lbry/extras/daemon/daemon.py
View file

@ -4283,6 +4283,65 @@ class Daemon(metaclass=JSONRPCServerType):
read_only=True, **self._constrain_txo_from_kwargs({}, **kwargs)
)
@requires(WALLET_COMPONENT)
async def jsonrpc_txo_plot(
self, account_id=None, wallet_id=None,
days_back=0, start_day=None, days_after=None, end_day=None, **kwargs):
"""
Plot transaction output sum over days.
Usage:
txo_plot [--account_id=<account_id>] [--type=<type>...] [--txid=<txid>...]
[--claim_id=<claim_id>...] [--name=<name>...] [--unspent]
[--is_my_input_or_output |
[[--is_my_output | --is_not_my_output] [--is_my_input | --is_not_my_input]]
]
[--exclude_internal_transfers] [--wallet_id=<wallet_id>]
[--days_back=<days_back> |
[--start_day=<start_day> [--days_after=<days_after> | --end_day=<end_day>]]
]
Options:
--type=<type> : (str or list) claim type: stream, channel, support,
purchase, collection, repost, other
--txid=<txid> : (str or list) transaction id of outputs
--claim_id=<claim_id> : (str or list) claim id
--name=<name> : (str or list) claim name
--unspent : (bool) hide spent outputs, show only unspent ones
--is_my_input_or_output : (bool) txos which have your inputs or your outputs,
if using this flag the other related flags
are ignored (--is_my_output, --is_my_input, etc)
--is_my_output : (bool) show outputs controlled by you
--is_not_my_output : (bool) show outputs not controlled by you
--is_my_input : (bool) show outputs created by you
--is_not_my_input : (bool) show outputs not created by you
--exclude_internal_transfers: (bool) excludes any outputs that are exactly this combination:
"--is_my_input --is_my_output --type=other"
this allows to exclude "change" payments, this
flag can be used in combination with any of the other flags
--account_id=<account_id> : (str) id of the account to query
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
--days_back=<days_back> : (int) number of days back from today
(not compatible with --start_day, --days_after, --end_day)
--start_day=<start_day> : (date) start on specific date (YYYY-MM-DD)
(instead of --days_back)
--days_after=<days_after> : (int) end number of days after --start_day
(instead of --end_day)
--end_day=<end_day> : (date) end on specific date (YYYY-MM-DD)
(instead of --days_after)
Returns: List[Dict]
"""
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
plot = await self.ledger.get_txo_plot(
wallet=wallet, accounts=[wallet.get_account_or_error(account_id)] if account_id else wallet.accounts,
read_only=True, days_back=days_back, start_day=start_day, days_after=days_after, end_day=end_day,
**self._constrain_txo_from_kwargs({}, **kwargs)
)
for row in plot:
row['total'] = dewies_to_lbc(row['total'])
return plot
UTXO_DOC = """
Unspent transaction management.
"""

3
lbry/testcase.py
View file

@ -580,6 +580,9 @@ class CommandTestCase(IntegrationTestCase):
async def txo_sum(self, *args, **kwargs):
return await self.out(self.daemon.jsonrpc_txo_sum(*args, **kwargs))
async def txo_plot(self, *args, **kwargs):
return await self.out(self.daemon.jsonrpc_txo_plot(*args, **kwargs))
async def claim_list(self, *args, **kwargs):
return (await self.out(self.daemon.jsonrpc_claim_list(*args, **kwargs)))['items']

44
lbry/wallet/database.py
View file

@ -9,10 +9,12 @@ from contextvars import ContextVar
from concurrent.futures.thread import ThreadPoolExecutor
from concurrent.futures.process import ProcessPoolExecutor
from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable, Dict, Optional
from datetime import date
from .bip32 import PubKey
from .transaction import Transaction, Output, OutputScript, TXRefImmutable
from .constants import TXO_TYPES, CLAIM_TYPES
from .util import date_to_julian_day
log = logging.getLogger(__name__)
@ -254,6 +256,7 @@ def query(select, **constraints) -> Tuple[str, Dict[str, Any]]:
limit = constraints.pop('limit', None)
offset = constraints.pop('offset', None)
order_by = constraints.pop('order_by', None)
group_by = constraints.pop('group_by', None)
accounts = constraints.pop('accounts', [])
if accounts:
@ -264,6 +267,9 @@ def query(select, **constraints) -> Tuple[str, Dict[str, Any]]:
sql.append('WHERE')
sql.append(where)
if group_by is not None:
sql.append(f'GROUP BY {group_by}')
if order_by:
sql.append('ORDER BY')
if isinstance(order_by, str):
@ -387,7 +393,7 @@ def dict_row_factory(cursor, row):
class Database(SQLiteMixin):
SCHEMA_VERSION = "1.2"
SCHEMA_VERSION = "1.3"
PRAGMAS = """
pragma journal_mode=WAL;
@ -422,7 +428,8 @@ class Database(SQLiteMixin):
height integer not null,
position integer not null,
is_verified boolean not null default 0,
purchased_claim_id text
purchased_claim_id text,
day integer
);
create index if not exists tx_purchased_claim_id_idx on tx (purchased_claim_id);
"""
@ -506,14 +513,14 @@ class Database(SQLiteMixin):
row['claim_name'] = txo.claim_name
return row
@staticmethod
def tx_to_row(tx):
def tx_to_row(self, tx):
row = {
'txid': tx.id,
'raw': sqlite3.Binary(tx.raw),
'height': tx.height,
'position': tx.position,
'is_verified': tx.is_verified
'is_verified': tx.is_verified,
'day': tx.get_julian_day(self.ledger),
}
txos = tx.outputs
if len(txos) >= 2 and txos[1].can_decode_purchase_data:
@ -863,6 +870,7 @@ class Database(SQLiteMixin):
return txos
def _clean_txo_constraints_for_aggregation(self, unspent, constraints):
constraints.pop('include_is_spent', None)
constraints.pop('include_is_my_input', None)
constraints.pop('include_is_my_output', None)
constraints.pop('include_received_tips', None)
@ -876,14 +884,36 @@ class Database(SQLiteMixin):
async def get_txo_count(self, unspent=False, **constraints):
self._clean_txo_constraints_for_aggregation(unspent, constraints)
count = await self.select_txos('COUNT(*) as total', **constraints)
count = await self.select_txos('COUNT(*) AS total', **constraints)
return count[0]['total'] or 0
async def get_txo_sum(self, unspent=False, **constraints):
self._clean_txo_constraints_for_aggregation(unspent, constraints)
result = await self.select_txos('SUM(amount) as total', **constraints)
result = await self.select_txos('SUM(amount) AS total', **constraints)
return result[0]['total'] or 0
async def get_txo_plot(
self, unspent=False, start_day=None, days_back=0, end_day=None, days_after=None, **constraints):
self._clean_txo_constraints_for_aggregation(unspent, constraints)
if start_day is None:
constraints['day__gte'] = self.ledger.headers.estimated_julian_day(
self.ledger.headers.height
) - days_back
else:
constraints['day__gte'] = date_to_julian_day(
date.fromisoformat(start_day)
)
if end_day is not None:
constraints['day__lte'] = date_to_julian_day(
date.fromisoformat(end_day)
)
elif days_after is not None:
constraints['day__lte'] = constraints['day__gte'] + days_after
return await self.select_txos(
"DATE(day) AS day, SUM(amount) AS total",
group_by='day', order_by='day', **constraints
)
def get_utxos(self, read_only=False, **constraints):
return self.get_txos(unspent=True, read_only=read_only, **constraints)

6
lbry/wallet/header.py
View file

@ -4,6 +4,7 @@ import struct
import asyncio
import logging
import zlib
from datetime import date
from concurrent.futures.thread import ThreadPoolExecutor
from io import BytesIO
@ -11,7 +12,7 @@ from typing import Optional, Iterator, Tuple, Callable
from binascii import hexlify, unhexlify
from lbry.crypto.hash import sha512, double_sha256, ripemd160
from lbry.wallet.util import ArithUint256
from lbry.wallet.util import ArithUint256, date_to_julian_day
from .checkpoints import HASHES
@ -129,6 +130,9 @@ class Headers:
def estimated_timestamp(self, height):
return self.first_block_timestamp + (height * self.timestamp_average_offset)
def estimated_julian_day(self, height):
return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height)))
async def get_raw_header(self, height) -> bytes:
if self.chunk_getter:
await self.ensure_chunk_at(height)

3
lbry/wallet/ledger.py
View file

@ -277,6 +277,9 @@ class Ledger(metaclass=LedgerRegistry):
def get_txo_sum(self, **constraints):
return self.db.get_txo_sum(**constraints)
def get_txo_plot(self, **constraints):
return self.db.get_txo_plot(**constraints)
def get_transactions(self, **constraints):
return self.db.get_transactions(**constraints)

8
lbry/wallet/transaction.py
View file

@ -494,7 +494,7 @@ class Output(InputOutput):
class Transaction:
def __init__(self, raw=None, version: int = 1, locktime: int = 0, is_verified: bool = False,
height: int = -2, position: int = -1) -> None:
height: int = -2, position: int = -1, julian_day: int = None) -> None:
self._raw = raw
self._raw_sans_segwit = None
self.is_segwit_flag = 0
@ -512,6 +512,7 @@ class Transaction:
# +num: confirmed in a specific block (height)
self.height = height
self.position = position
self._day = julian_day
if raw is not None:
self._deserialize()
@ -535,6 +536,11 @@ class Transaction:
def hash(self):
return self.ref.hash
def get_julian_day(self, ledger):
if self._day is None and self.height > 0:
self._day = ledger.headers.estimated_julian_day(self.height)
return self._day
@property
def raw(self):
if self._raw is None:

4
lbry/wallet/util.py
View file

@ -3,6 +3,10 @@ from typing import TypeVar, Sequence, Optional
from .constants import COIN
def date_to_julian_day(d):
return d.toordinal() + 1721424.5
def coins_to_satoshis(coins):
if not isinstance(coins, str):
raise ValueError("{coins} must be a string")

55
tests/integration/blockchain/test_claim_commands.py
View file

@ -555,6 +555,61 @@ class TransactionOutputCommands(ClaimTestCase):
r = await self.txo_list(is_not_my_output=True)
self.assertEqual([sent_channel], r)
async def test_txo_plot(self):
day_blocks = int((24 * 60 * 60) / self.ledger.headers.timestamp_average_offset)
stream_id = self.get_claim_id(await self.stream_create())
await self.support_create(stream_id, '0.3')
await self.support_create(stream_id, '0.2')
await self.generate(day_blocks)
await self.support_create(stream_id, '0.4')
await self.support_create(stream_id, '0.5')
await self.generate(day_blocks)
await self.support_create(stream_id, '0.6')
plot = await self.txo_plot(type='support')
self.assertEqual([
{'day': '2016-06-25', 'total': '0.6'},
], plot)
plot = await self.txo_plot(type='support', days_back=1)
self.assertEqual([
{'day': '2016-06-24', 'total': '0.9'},
{'day': '2016-06-25', 'total': '0.6'},
], plot)
plot = await self.txo_plot(type='support', days_back=2)
self.assertEqual([
{'day': '2016-06-23', 'total': '0.5'},
{'day': '2016-06-24', 'total': '0.9'},
{'day': '2016-06-25', 'total': '0.6'},
], plot)
plot = await self.txo_plot(type='support', start_day='2016-06-23')
self.assertEqual([
{'day': '2016-06-23', 'total': '0.5'},
{'day': '2016-06-24', 'total': '0.9'},
{'day': '2016-06-25', 'total': '0.6'},
], plot)
plot = await self.txo_plot(type='support', start_day='2016-06-24')
self.assertEqual([
{'day': '2016-06-24', 'total': '0.9'},
{'day': '2016-06-25', 'total': '0.6'},
], plot)
plot = await self.txo_plot(type='support', start_day='2016-06-23', end_day='2016-06-24')
self.assertEqual([
{'day': '2016-06-23', 'total': '0.5'},
{'day': '2016-06-24', 'total': '0.9'},
], plot)
plot = await self.txo_plot(type='support', start_day='2016-06-23', days_after=1)
self.assertEqual([
{'day': '2016-06-23', 'total': '0.5'},
{'day': '2016-06-24', 'total': '0.9'},
], plot)
plot = await self.txo_plot(type='support', start_day='2016-06-23', days_after=2)
self.assertEqual([
{'day': '2016-06-23', 'total': '0.5'},
{'day': '2016-06-24', 'total': '0.9'},
{'day': '2016-06-25', 'total': '0.6'},
], plot)
class ClaimCommands(ClaimTestCase):