Compare commits

...

2027 commits

Author SHA1 Message Date
Jonathan Moody
eb5da9511e Revert "TEMP: Try python 3.8."
This reverts commit 8def4d5177.
2023-04-03 13:34:36 -04:00
Jonathan Moody
8722ef840e Bump python_requires >= 3.8.
Code to handle CancelledError (as subclass of Exception) was removed.
2023-04-03 13:34:36 -04:00
Jonathan Moody
6e75a1a89b TEMP: Try python 3.8. 2023-04-03 13:34:36 -04:00
Jonathan Moody
ef3189de1d Work on some DeprecationWarnings: The explicit passing of coroutine objects to asyncio.wait() is deprecated since Python 3.8. 2023-04-03 13:34:36 -04:00
Jonathan Moody
c2d2080034 Try to suppress asyncio.CancelledError in a different way in test_streaming.py. 2023-04-03 13:34:36 -04:00
Jonathan Moody
d0b5a0a8fd TEMP: Add workflow_dispatch. 2023-04-03 13:34:36 -04:00
Jonathan Moody
1d0e17be21 Another place generalized to Exception or asyncio.CancelledError. 2023-04-03 13:34:36 -04:00
Jonathan Moody
4ef03bb1f4 Try separate file_manager.stop() and start() calls to better
control order of events in test.
While file_manager is stopped, we get no response to file_list().
2023-04-03 13:34:36 -04:00
Jonathan Moody
4bd4bcdc27 Try ubuntu-20.04 to resolve missing libffi.so.7 issue. 2023-04-03 13:34:36 -04:00
Jonathan Moody
e5ca967fa2 Make FileManager.stop() async because SourceManager.stop() is now async. 2023-04-03 13:34:36 -04:00
Jonathan Moody
eed7d02e8b Tweak aiohttp version to be compatible with hub repository. 2023-04-03 13:34:36 -04:00
Jonathan Moody
02aecad52b CancelledError derives from BaseException in Python >= 3.8. The significant functional
change here is in upload_to_reflector(). Unit tests in TestReflector were failing.
Deal with lint related to CancelledError cleanup.
2023-04-03 13:34:36 -04:00
Jonathan Moody
585962d930 Make stop(), stop_tasks() consistently async routines, and have stop_tasks()
wait for file_output_task completion. This fixes a problem with
test_download_stop_resume_delete.
2023-04-03 13:34:36 -04:00
Jonathan Moody
ea4fba39a6 Fix Transport, DatagramTransport mockup issues. 2023-04-03 13:34:36 -04:00
Jonathan Moody
7a86406746 Fix and enable lint no-self-use & try-except-raise. 2023-04-03 13:34:36 -04:00
Jonathan Moody
c8a3eb97a4 Bump pylint version. Old pylint did not find standard library stuff on 3.9.12. 2023-04-03 13:34:36 -04:00
Lex Berezhny
20213628d7 upgrade cryptography 2023-04-03 13:34:36 -04:00
Lex Berezhny
2d1649f972 pylint disable shuffle() arg check 2023-04-03 13:34:36 -04:00
Lex Berezhny
5cb04b86a0 shuffle() needs custom random, removed loop from Event()/Queue() 2023-04-03 13:34:36 -04:00
Lex Berezhny
93ab6b3be3 passing loop to asyncio functions is deprecated 2023-04-03 13:34:36 -04:00
Lex Berezhny
b9762c3e64 update plyvel 2023-04-03 13:34:36 -04:00
Lex Berezhny
82592d00ef try building 3.9 2023-04-03 13:34:36 -04:00
Jonathan Moody
c118174c1a Try shell: bash to simplify. 2023-02-02 14:16:07 -05:00
Jonathan Moody
d284acd8b8 Remove "debug pip cache". 2023-02-02 14:16:07 -05:00
Jonathan Moody
235c98372d Fix syntax. 2023-02-02 14:16:07 -05:00
Jonathan Moody
d2f5073ef4 Single "set pip cache dir" task with conditional inside. 2023-02-02 14:16:07 -05:00
Jonathan Moody
84e5e43117 Bump upload-artifact version too. 2023-02-02 14:16:07 -05:00
Jonathan Moody
7bd025ae54 Upgrade change-string-case. Use startsWith() to test runner.os.
Bump change-string-case-action version again.
2023-02-02 14:16:07 -05:00
Jonathan Moody
8f28ce65b0 Switch to environment vars in $GITHUB_ENV. 2023-02-02 14:16:07 -05:00
Jonathan Moody
d36e305129 Functions save-state, set-output deprecated. Use new mechanism. 2023-02-02 14:16:07 -05:00
Jonathan Moody
2609dee8fb Bump checkout, setup-python, cache action verions. 2023-02-02 14:16:07 -05:00
Lex Berezhny
a2da86d4b5 v0.113.0 2023-01-23 10:43:02 -05:00
Alex Grin
aa16c7fee5 Update conf.py 2023-01-23 10:30:25 -05:00
Alex Grin
3266f72b82 add s1.lbry.network 2023-01-23 10:30:25 -05:00
Jack Robison
77cd2a3f8a add more non lbry.com hubs/bootstrap dht nodes 2023-01-23 10:30:25 -05:00
Alex Grin
308e586e9a add grin's domain to bootstrap hubs list 2023-01-23 10:30:25 -05:00
84beddfd77 Added tracker and dht from pigg.es
Added tracker and dht from pigg.es
2023-01-22 19:09:17 -05:00
Victor Shyba
6258651650
Merge pull request #3716 from lbryio/dht_exceptions
handle remote exceptions on routing table ping
2022-12-13 17:18:47 -03:00
Victor Shyba
cc5f0b6630 handle remote exception on routing table ping 2022-12-13 16:56:58 -03:00
Jonathan Moody
f64d507d39 TEMP: Pin workflows to ubuntu-20.04 to work around missing ripemd160 issue. 2022-12-12 21:47:41 -05:00
Jonathan Moody
001819d5c2 Bump Hub to include fix for supports with wrong names. 2022-11-20 20:34:30 -05:00
Jonathan Moody
8b4c046d28 Try pyinstaller==4.6 to fix MacOS build failure. 2022-11-20 20:34:30 -05:00
Jonathan Moody
2c20ad6c43 Add another zlib.error mapped to InvalidPasswordError. 2022-11-20 20:34:30 -05:00
Jonathan Moody
9e610cc54c Update test for Hub rename of method stage_put() -> stash_put(). 2022-11-20 20:34:30 -05:00
Jonathan Moody
b9d25c6d01 Bump hub to latest, getting fix for TX negative caching issue and others. 2022-11-20 20:34:30 -05:00
Jonathan Moody
419b5b45f2 Allow a few initial "transaction not found" responses from Hub. 2022-11-20 20:34:30 -05:00
Jonathan Moody
516c2dd5d0 Bump hub to fix subscribe race + EADDRINUSE issue. 2022-11-20 20:34:30 -05:00
Jonathan Moody
b99102f9c9 Bump max_misuse_attempts by 50% to 120000. 2022-11-20 20:34:30 -05:00
Lex Berezhny
8c6c7b655c v0.112.0 2022-10-30 21:56:17 -04:00
Lex Berezhny
48c6873fc4 channel_sign command has customizeable salt 2022-10-30 21:53:53 -04:00
Victor Shyba
15dc52bd9a
Merge pull request #3695 from lbryio/3690
Fix claim fields fallback raising errors before download is saved on database
2022-10-28 11:16:52 -03:00
Victor Shyba
52d555078f initialize stored claim field for fallback earlier 2022-10-19 15:13:47 -03:00
Victor Shyba
cc976bd010 test for early fallback of suggested_file_name 2022-10-19 15:13:47 -03:00
Lex Berezhny
9cc6992011 torrents needs loop 2022-10-18 17:23:56 -04:00
Lex Berezhny
a1b87460c5 passing loop to asyncio functions is deprecated 2022-10-18 17:23:56 -04:00
jessopb
007e1115c4 v0.111.0 2022-10-18 11:18:26 -04:00
jessopb
20ae51b949
Merge pull request #3692 from lbryio/fix-import/export-backwards-compat
fix backwards compatibility in wallet import/export
2022-10-18 11:16:34 -04:00
zeppi
24e9c7b435 fix backwards compatibility in wallet import/export 2022-10-18 10:53:51 -04:00
zeppi
b452e76e1d reverting version to 0.110.0 2022-10-18 10:27:44 -04:00
Lex Berezhny
341834c30d v0.111.0 2022-10-18 00:37:44 -04:00
Victor Shyba
12bac730bd tests: check mime type as well 2022-10-18 00:31:10 -04:00
Victor Shyba
1027337833 fallback for stream name and tests 2022-10-18 00:31:10 -04:00
Victor Shyba
97fef21f75 fallback for suggested file name and tests 2022-10-18 00:31:10 -04:00
Lex Berezhny
9dafd5f69b added claim_list filtering by reposted_claim_id and fix for claim_id of reposted claim in JSON output 2022-10-18 00:28:13 -04:00
Victor Shyba
fd4f0b2049 bump first-start checkpoint 2022-10-18 00:26:10 -04:00
Lex Berezhny
734f0651a4 minor refactor 2022-10-18 00:25:41 -04:00
zeppi
94deaf55df lint 2022-10-18 00:25:41 -04:00
zeppi
d957d46f96 lint 2022-10-18 00:25:41 -04:00
zeppi
0217aede3d update docs 2022-10-18 00:25:41 -04:00
zeppi
e4e1600f51 Enable unencrypted wallet import and export 2022-10-18 00:25:41 -04:00
Jonathan Moody
d0aad8ccaf Add zlib.error string just observed for the first time. 2022-09-29 22:18:54 -04:00
Jonathan Moody
ab50cfa5c1 Add test steps to repeatedly sync_apply() using a bad password. 2022-09-29 22:18:54 -04:00
Jonathan Moody
5a26aea398 Feedback: Reuse IntegrationTestcase.generate() in generate_and_wait(). 2022-09-29 22:18:54 -04:00
Jonathan Moody
bd1cebdb4c Bump hub to include TaskGroup fix. 2022-09-29 22:18:54 -04:00
Jonathan Moody
ec433f069f Substitute InvalidPasswordError for zlib.error. 2022-09-29 22:18:54 -04:00
Jonathan Moody
cd6d3fec9c Wait for initial sync in test_wallet_syncing_status(). 2022-09-29 22:18:54 -04:00
Jonathan Moody
8c474a69de Hub error message changed to include blocked/filtered. 2022-09-29 22:18:54 -04:00
Jonathan Moody
8903056648 Bump hub version to latest. 2022-09-29 22:18:54 -04:00
Jonathan Moody
749a92d0e5 Wait on block height too in generate_and_wait(). 2022-09-29 22:18:54 -04:00
Jonathan Moody
a7d7efecc7 Logging level back to INFO. 2022-09-20 10:04:23 -04:00
Jonathan Moody
c88f0797a3 Log f.exception() if present instead of "Stopped". 2022-09-20 10:04:23 -04:00
Jonathan Moody
137ebd503d Test insufficient funds behavior. 2022-09-20 10:04:23 -04:00
Jonathan Moody
c3f5dd780e Revise exception handling. 2022-09-20 10:04:23 -04:00
Jonathan Moody
20b1865879 Don't use retriable_call(). Add handling for InsufficientFundsError. 2022-09-20 10:04:23 -04:00
Jonathan Moody
231b982422 Wait on usage payement TX to be processed. 2022-09-20 10:04:23 -04:00
Jonathan Moody
fd69401791 Catch and log exceptions coming from the pay() task.
Change test to reproduce failure.
2022-09-20 10:04:23 -04:00
Jonathan Moody
718d046833 Logging for test_single_server_payment debug. 2022-09-20 10:04:23 -04:00
Victor Shyba
e10f57d1ed
Merge pull request #3642 from lbryio/libtorrent
use official libtorrent, fix tests, make it a normal dependency
2022-09-09 19:58:38 -03:00
Victor Shyba
8a033d58df fix torrent component 2022-09-09 12:21:59 -03:00
Victor Shyba
c07c369a28 add libtorrent pyinstaller hook 2022-09-09 12:21:59 -03:00
Victor Shyba
5be990fc55 do not ignore libtorrent import error 2022-09-09 12:21:59 -03:00
Victor Shyba
8f26010c04 make libtorrent a normal dependency 2022-09-09 12:21:59 -03:00
Victor Shyba
3021962e3d tests: add peer directly instead of relying on torrent dht 2022-09-09 12:21:59 -03:00
Victor Shyba
84d89ce5af torrent: disable upnp, natpmp. 2022-09-09 12:21:59 -03:00
Victor Shyba
0961cad716 remove dead code 2022-09-09 12:21:59 -03:00
Jonathan Moody
5c543cb374 Wait for hub to update with all 100 new blocks
before proceeding with initial_headers_sync().
2022-09-08 22:40:09 -04:00
Jonathan Moody
f78d7896a5 Revert "Add more failure message details for debugging."
This reverts commit 5e00a79751.
2022-09-08 18:13:26 -04:00
Jonathan Moody
78a28de2aa Align style of generate() with generate_and_wait(). 2022-09-08 18:13:26 -04:00
Jonathan Moody
45a255e7a2 Reuse generate() logic to wait on hub
instead of half-baked reorg() logic.
2022-09-08 18:13:26 -04:00
Jonathan Moody
d2738c2e72 Add more failure message details for debugging. 2022-09-08 18:13:26 -04:00
Jonathan Moody
a7c7ab7f7b Correct the terminal height we wait for in generate(). 2022-09-08 18:13:26 -04:00
Jonathan Moody
988f288715 Lint fix for _es_height checks. 2022-09-08 18:13:26 -04:00
Jonathan Moody
38e9b5b432 Wait for _es_height in addition to db_height. 2022-09-08 18:13:26 -04:00
Victor Shyba
f7455600cc
Merge pull request #3625 from lbryio/dht_crawler
Add script to collect DHT metrics
2022-09-07 12:56:41 -03:00
Victor Shyba
c7c2d6fe5a collect connections reachability 2022-09-07 12:03:11 -03:00
Victor Shyba
c6c0228970 fix crawler startup query 2022-09-07 12:03:11 -03:00
Victor Shyba
8d9d2c76ae routing table sizes as histogram 2022-09-07 12:03:11 -03:00
Victor Shyba
0b059a5445 use a histogram for latency, remove labels 2022-09-07 12:03:11 -03:00
Victor Shyba
ab67f417ee dht_crawler: wait and retry during port switch 2022-09-07 12:03:11 -03:00
Victor Shyba
0e7a1aee0a dht_crawler: clean in memory set for expired peers 2022-09-07 12:03:11 -03:00
Victor Shyba
d0497cf6b5 dht_crawler: skip saving connections for now 2022-09-07 12:03:11 -03:00
Victor Shyba
c38573d5de dht_crawler: gather both loops, avoid task exceptions being hidden 2022-09-07 12:03:11 -03:00
Victor Shyba
f077e56cec dht_crawler:only count latency during findNode 2022-09-07 12:03:11 -03:00
Victor Shyba
5e58c2f224 fix hosting metrics, improve logging 2022-09-07 12:03:11 -03:00
Victor Shyba
cc64789e96 dht_crawler: fix logging for missing ports 2022-09-07 12:03:11 -03:00
Victor Shyba
b5c390ca04 docker: add volume declaration 2022-09-07 12:03:11 -03:00
Victor Shyba
da2ffb000e skip peers with bad ports without raising 2022-09-07 12:03:11 -03:00
Victor Shyba
df77392fe0 dht crawler:improve logging, metrics, make startup concurrent 2022-09-07 12:03:11 -03:00
Victor Shyba
9aa9ecdc0a add arg for db path 2022-09-07 12:03:11 -03:00
Victor Shyba
43b45a939b format logging 2022-09-07 12:03:11 -03:00
Victor Shyba
e2922a434f add script to generate probe dataset 2022-09-07 12:03:11 -03:00
Victor Shyba
0d6125de0b add sd_hash prober 2022-09-07 12:03:11 -03:00
Victor Shyba
13af7800c2 refactor script, remove dep 2022-09-07 12:03:11 -03:00
Victor Shyba
47a5d37d7c change default metric port, add sqlalchemy to dockerfile 2022-09-07 12:03:11 -03:00
Victor Shyba
4a3a7e318d update pip and setuptools on dht dockerfile 2022-09-07 12:03:11 -03:00
Victor Shyba
85ff487af5 dht_crawler: randomize port when idle 2022-09-07 12:03:11 -03:00
Victor Shyba
62eb9d5c75 dht_crawler: only count non zero connections 2022-09-07 12:03:11 -03:00
Victor Shyba
cfe5c8de8a dht_crawler: serve prometheus metrics at 7070 2022-09-07 12:03:11 -03:00
Victor Shyba
0497698c5b dht_crawler: skip ping if known node_id 2022-09-07 12:03:11 -03:00
Victor Shyba
508bdb8e94 dht_crawler: keep working set in memory, flush to db on intervals 2022-09-07 12:03:11 -03:00
Victor Shyba
cd42f0d726 dht_crawler: fix node id store 2022-09-07 12:03:11 -03:00
Victor Shyba
2706b66a92 dht_crawler: dont re-bootstrap. try known reachable even when they expire 2022-09-07 12:03:11 -03:00
Victor Shyba
29c2d5715d dht_crawler: fix last_seen update 2022-09-07 12:03:11 -03:00
Victor Shyba
965389b759 dht_crawler: process older first, avoid discarding 2022-09-07 12:03:11 -03:00
Victor Shyba
174439f517 dht_crawler: cleanup, try not to reset key 2022-09-07 12:03:11 -03:00
Victor Shyba
baf422fc03 dht_crawler: extract refresh_limit, bump to 1h 2022-09-07 12:03:11 -03:00
Victor Shyba
61f7fbe230 dht_crawler: avoid reads 2022-09-07 12:03:11 -03:00
Victor Shyba
c6c27925b7 dht_crawler: flush/commit only when finished 2022-09-07 12:03:11 -03:00
Victor Shyba
be4c62cf32 check membership instead of one update per peer 2022-09-07 12:03:11 -03:00
Victor Shyba
443a1c32fa dht_crawler: save a set of connections to avoid dupes, enable initial crawl 2022-09-07 12:03:11 -03:00
Victor Shyba
90c2a58470 dht_crawler: dont gather empty, fix crash 2022-09-07 12:03:11 -03:00
Victor Shyba
adc79ec404 dht_crawler: only warn for missing key if it replied 2022-09-07 12:03:11 -03:00
Victor Shyba
137d8ca4ac dht_crawler: enable WAL 2022-09-07 12:03:11 -03:00
Victor Shyba
abf4d888af dht_crawler: warn if we cannot get node id 2022-09-07 12:03:11 -03:00
Victor Shyba
6c350e57dd dht_crawler: query recently checked as stats 2022-09-07 12:03:11 -03:00
Victor Shyba
fb7a93096e only count checked unreachable 2022-09-07 12:03:11 -03:00
Victor Shyba
7ea88e7b31 dht_crawler: store data 2022-09-07 12:03:11 -03:00
Victor Shyba
2361e34541 dht crawler, initial version 2022-09-07 12:03:11 -03:00
Victor Shyba
be06378437 add method for getting the node_id from a known peer on peer manager 2022-09-07 12:03:11 -03:00
Victor Shyba
a334a93757
Merge pull request #3631 from lbryio/bootstrap_node
Add all peers when running as a bootstrap node
2022-08-29 11:07:29 -03:00
Victor Shyba
e3ee3892b2 better test name 2022-08-22 18:45:18 -03:00
Victor Shyba
d61accea1a simplify bucket refresh loop 2022-08-11 21:14:56 -03:00
Victor Shyba
e887453aa5 remove unused last_accessed 2022-08-11 20:39:51 -03:00
Victor Shyba
c3e4f0b988 add 'is_bootstrap_node' conf 2022-08-11 20:38:42 -03:00
Victor Shyba
318728aebd add bootstrap flag to routing table 2022-08-11 20:38:42 -03:00
Victor Shyba
d8c1aaebc2 routing table: mark private methods 2022-08-11 20:38:42 -03:00
Victor Shyba
d7b65c15d2 return none instead of raising 2022-08-11 20:38:42 -03:00
Victor Shyba
972db80246 move add peer logic to routing table 2022-08-11 20:38:42 -03:00
Victor Shyba
0d343ecb2f simplify iterative find constructor 2022-08-11 20:38:42 -03:00
Lex Berezhny
01cd95fe46 v0.110.0 2022-08-11 10:58:16 -04:00
Lex Berezhny
6dc57fc02c revert version 2022-08-11 10:20:58 -04:00
Lex Berezhny
10df0c1fba disable Hotbit and UPBit exchange rate feeds 2022-08-11 10:19:54 -04:00
Lex Berezhny
ec751e5add v0.110.0 2022-08-10 13:52:46 -04:00
Lex Berezhny
3e3974f813 lint 2022-08-08 14:55:44 -04:00
Lex Berezhny
ec82486e15 removed go hub dependency 2022-08-08 14:55:44 -04:00
Lex Berezhny
e16f6b07b8 revert release 2022-08-08 13:02:12 -04:00
Lex Berezhny
9a842c273b v0.110.0 2022-08-08 08:46:32 -04:00
Jonathan Moody
40f7d3ee4b Stabilize test_streaming.py by scaning the data_dir, not the parent of data_dir 2022-08-01 17:37:06 -04:00
Lex Berezhny
1dc2f0458b fix lint 2022-08-01 10:04:24 -04:00
Lex Berezhny
3924b28cc3 raise not implemented error when importing unencrypted wallet 2022-08-01 10:04:24 -04:00
Lex Berezhny
020487b6a0 account merge bug fix from upstream 2022-08-01 10:04:24 -04:00
zeppi
14037c9b2f help string edits 2022-08-01 10:04:24 -04:00
zeppi
0cb37a5c4b linting 2022-08-01 10:04:24 -04:00
zeppi
fa5f3e7e55 change api for data first, password optional, return (str) 2022-08-01 10:04:24 -04:00
zeppi
30aa0724ec newline end of test file 2022-08-01 10:04:24 -04:00
zeppi
059890e4e5 wallet import export feature 2022-08-01 10:04:24 -04:00
Jonathan Moody
9654d4f003 Obtain "amount" from new_txo.amount when calling save_supports(). 2022-08-01 09:10:49 -04:00
Jonathan Moody
956b52a2c1 Refactor _old_get_temp_claim_info(), eliminating "bid" arg. Obtain the value from txo.amount. 2022-08-01 09:10:49 -04:00
Lex Berezhny
2e975c8b61 lint 2022-07-26 22:18:29 -04:00
Lex Berezhny
656e299100 migrate key addresses on changed accounts after sync apply 2022-07-26 22:18:29 -04:00
Jack Robison
352e45b6b7 update pinned hub version 2022-07-25 10:12:46 -04:00
Jack Robison
a9a1076362 improve test_es_sync_utility 2022-07-25 10:12:46 -04:00
Jack Robison
6d370b0a12 dont skip test_setting_stream_fields 2022-07-25 10:12:46 -04:00
Jack Robison
c9fac27b66 test resolving different streams for a channel using short urls 2022-07-25 10:12:46 -04:00
Jack Robison
59bc0b9682 update censored error 2022-07-25 10:12:46 -04:00
Lex Berezhny
ba60aeeebc migrate certificates after importing new account 2022-07-18 10:36:21 -04:00
Jonathan Moody
dc427ecf6c Correct collection_update, account_fund docstrings. Regenerate api.json using generate_json_api.py. 2022-07-07 21:33:43 -04:00
Victor Shyba
87b4404767
Merge pull request #3624 from moodyjon/test_fix_exch_rate1
Fixes for intermittent test failures: test_exchange_rate_manager(), test_basic_claim_search()
2022-07-01 17:29:44 -03:00
Jonathan Moody
ba9ac489c3 Relax range in test_exchange_rate_manager.py. (again) 2022-06-19 19:17:48 -04:00
Jonathan Moody
7049629ad7 Relax range in test_exchange_rate_manager.py. 2022-06-19 19:17:40 -04:00
Jonathan Moody
3ae4aeea47 Search for longer prefix of sd_hash to give better chance of unique results. 2022-06-19 19:14:54 -04:00
Lex Berezhny
8becf1f69f v0.109.0 2022-06-08 12:40:35 -04:00
Victor Shyba
582f79ba1c do not consider pending blobs on disk space query 2022-06-08 12:25:38 -04:00
Lex Berezhny
3c28d869f4
Merge pull request #3620 from lbryio/repost_title_tags
reposts can have title, description and tags
2022-06-08 12:24:28 -04:00
Lex Berezhny
fe61b90610 reposts can have title, description and tags 2022-06-08 10:35:22 -04:00
Lex Berezhny
c04fbb2908
Merge pull request #3614 from lbryio/grins-tracker
add tracker.lbry.grin.io
2022-06-06 13:13:01 -04:00
Alex Grintsvayg
571e71b28e
add tracker.lbry.grin.io 2022-06-06 11:29:09 -04:00
Lex Berezhny
39fcfcccfb
Merge pull request #3608 from lbryio/fix_ci
upgraded SDK to use the new LBRY hub project
2022-06-06 09:01:57 -04:00
Jack Robison
2313d30996
fix reconnect test 2022-05-27 11:59:18 -04:00
Jack Robison
ac7e94c6ed
pylint 2022-05-27 09:59:11 -04:00
Jack Robison
a391fe9fc7
scribe -> hub 2022-05-27 09:58:13 -04:00
Jack Robison
ea8adc5367
update scribe env and fix tests 2022-05-27 09:58:13 -04:00
Victor Shyba
0ea8ba72dd
Env->ServerEnv from scribe changes 2022-05-26 14:28:33 -04:00
Victor Shyba
7a8d5da0e8
Merge pull request #3613 from lbryio/fix_ci_lbcd_urls
tests: fix ci lbcd/lbcwallet urls
2022-05-26 10:21:02 -03:00
Victor Shyba
da30f003e8 update lbcwallet url 2022-05-25 12:17:57 -03:00
Victor Shyba
6257948ad7 update lbcd url 2022-05-25 12:17:28 -03:00
Victor Shyba
a7f606d62c change pip upgrade due windows error 2022-05-23 16:28:36 -03:00
Victor Shyba
1d95eb1549
Merge pull request #3599 from moodyjon/async-for-pr3504
Tighten up IterativeFinder async close behavior (DHT iterator continues after consumer breaks out of it)
2022-05-23 11:12:40 -03:00
Jonathan Moody
e5e9873f79 Simplify by eliminating AsyncGenerator base and generator function. Remove any new places enforcing max_results. 2022-05-20 17:23:39 -04:00
Jonathan Moody
530f9c72ea Fix lint error lbry/utils.py 2022-05-20 17:23:39 -04:00
Jonathan Moody
fad84c771c Support official contextlib.aclosing() when it's available. 2022-05-20 17:23:39 -04:00
Jonathan Moody
fe07aac79c Define and use lbry.utils.aclosing() in lieu of official contextlib.aclosing(). 2022-05-20 17:23:39 -04:00
Jonathan Moody
91a6eae831 Fix lint issue in iterative_find.py. 2022-05-20 17:23:39 -04:00
Jonathan Moody
5852fcd287 Don't wait on running_tasks after cancel(). Sometimes a CancelledError exception is received, which is unhelpful, and complicates shutting down the generator. 2022-05-20 17:23:39 -04:00
Jonathan Moody
4767bb9dee Wrap "async for" over IterativeXXXFinder in try/finally ensuring aclose(). 2022-05-20 17:23:39 -04:00
Jonathan Moody
82d7f81f41 Correct call to _aclose() in response to TransportNotConnected. 2022-05-20 17:23:39 -04:00
Jonathan Moody
b036961954 Tighten up IterativeFinder logic to respect max_records better, and wait after task cancel().
Also make IterativeFinder a proper AsyncGenerator. This gives it an offically recognized aclose() method and could help with clean finalization.
2022-05-20 17:23:39 -04:00
Victor Shyba
5c708e1c6f
Merge pull request #3611 from lbryio/fix_hub_url
tests: fix hub url
2022-05-20 18:19:39 -03:00
Victor Shyba
9436600267 tests: bump exchange rate manager test 2022-05-20 17:25:02 -03:00
Victor Shyba
4ab29c4d5f tests: fix hub url 2022-05-20 16:50:09 -03:00
Alex Grin
6944c4a7c4
Update LICENSE 2022-05-17 12:16:00 -04:00
Victor Shyba
2735484fae
Merge pull request #3576 from lbryio/trackers
Add support for announcing and querying LBRY streams over BEP15 (BitTorrent Trackers)
2022-05-13 17:56:20 -03:00
Victor Shyba
03b0d5e250 tracker client: extract default timeout and concurreny. Bump concurrency to 100 2022-05-11 21:13:30 -03:00
Victor Shyba
629812337b changes from review 2022-05-11 21:13:30 -03:00
Victor Shyba
e54cc8850c return KademliaPeers directly into the queue instead of exposing Announcement abstraction 2022-05-11 21:13:30 -03:00
Victor Shyba
7cba51ca7d update tests, query with port 0, filter bad ports earlier, make unit tests more reliable 2022-05-11 21:13:30 -03:00
Victor Shyba
3dc145fe68 make peer list query trackers 2022-05-11 21:13:30 -03:00
Victor Shyba
7d560df9fd use same arg name as overriden datagram_received (linting) 2022-05-11 21:13:30 -03:00
Victor Shyba
b3f894e480 add integration test for tracker discovery 2022-05-11 21:13:30 -03:00
Victor Shyba
235cc5dc05 results are indexed by ip, setdefault after resolve 2022-05-11 21:13:30 -03:00
Victor Shyba
c276053301 move server implementation to tracker module 2022-05-11 21:13:30 -03:00
Victor Shyba
2e85e29ef1 peer id PREFIX is a constant 2022-05-11 21:13:30 -03:00
Victor Shyba
1169a02c8b make client server updatable from conf 2022-05-11 21:13:30 -03:00
Victor Shyba
a7cea4082e tracker:log DNS errors as warning instead of trace 2022-05-11 21:13:30 -03:00
Victor Shyba
7e6ea97499 make peer id according to BEP20 2022-05-11 21:13:30 -03:00
Victor Shyba
3c46cc4fdd expire connection id quicker as some trackers have it set low 2022-05-11 21:13:30 -03:00
Victor Shyba
6e5c7a1927 use cache_concurrent to avoid requesting the same connection_id multiple times 2022-05-11 21:13:30 -03:00
Victor Shyba
4e09b35012 remove unused import and dead code 2022-05-11 21:13:30 -03:00
Victor Shyba
16a2023bbd stop tasks before removing transport 2022-05-11 21:13:30 -03:00
Victor Shyba
99fc7178c1 better way to batch announce + handle different intervals for different trackers 2022-05-11 21:13:30 -03:00
Victor Shyba
d4aca89a48 handle multiple results from multiple trackers 2022-05-11 21:13:30 -03:00
Victor Shyba
2918d8c7b4 tracker component is running only if the task is alive 2022-05-11 21:13:30 -03:00
Victor Shyba
407c570f8b tests: lower timeout, add test with bad and good mixed 2022-05-11 21:13:30 -03:00
Victor Shyba
e299a9c159 tests: multiple trackers, simple case 2022-05-11 21:13:30 -03:00
Victor Shyba
cc4a578578 tests: add support for multiple trackers 2022-05-11 21:13:30 -03:00
Victor Shyba
0e4f1eae5b reduce timeout to 10, fix lints 2022-05-11 21:13:30 -03:00
Victor Shyba
eccf0e6234 fix reusing result interval from failed expired attempt 2022-05-11 21:13:30 -03:00
Victor Shyba
a3da041412 fix exceptions on shutdown, stop using cancel_tasks 2022-05-11 21:13:30 -03:00
Victor Shyba
2f1617eee4 less verbose on timeouts, dont count timeouts, fix stop 2022-05-11 21:13:30 -03:00
Victor Shyba
05124d41ae only log when really announcing, stop counting cached ones 2022-05-11 21:13:30 -03:00
Victor Shyba
42fd1c962e stop tracker tasks on shutdown 2022-05-11 21:13:30 -03:00
Victor Shyba
47e432b4bb make it less verbose, only log after all events are fired 2022-05-11 21:13:30 -03:00
Victor Shyba
61c99abcf1 avoid readding the same hash when tracker is busy with too many files 2022-05-11 21:13:30 -03:00
Victor Shyba
28fdd62945 move concurreny control to lower layer 2022-05-11 21:13:30 -03:00
Victor Shyba
3855db6c66 pause announcer for 1 minute each round 2022-05-11 21:13:30 -03:00
Victor Shyba
30acde0afc at most 10 announces concurrently 2022-05-11 21:13:30 -03:00
Victor Shyba
2d9c5742c7 cache results, save interval on tracker 2022-05-11 21:13:30 -03:00
Victor Shyba
43e50f7f04 fix subscribe_hash 2022-05-11 21:13:30 -03:00
Victor Shyba
888e9918a6 improve timeout handling 2022-05-11 21:13:30 -03:00
Victor Shyba
9e9a64d989 evented system for tracker announcements 2022-05-11 21:13:30 -03:00
Victor Shyba
7acaecaed2 managed_stream: remove unused imports 2022-05-11 21:13:30 -03:00
Victor Shyba
24eb189b7f skip component on test cli 2022-05-11 21:13:30 -03:00
Victor Shyba
2344aca146 fix component property 2022-05-11 21:13:30 -03:00
Victor Shyba
758f9deafe fix unit tests 2022-05-11 21:13:30 -03:00
Victor Shyba
7b425eb2ac add tracker announcer component 2022-05-11 21:13:30 -03:00
Victor Shyba
30e8728f7f use tracker on download 2022-05-11 21:13:30 -03:00
Victor Shyba
3989eef84b return whole announcement so the caller knows the interval 2022-05-11 21:13:30 -03:00
Victor Shyba
dc6f8c4fc4 add arg to announce stopped, removing the announcement 2022-05-11 21:13:30 -03:00
Victor Shyba
2df8a1d99d make a helper function to announce 2022-05-11 21:13:30 -03:00
Victor Shyba
4ea858fdd3 add new conf: tracker_servers 2022-05-11 21:13:30 -03:00
Victor Shyba
006391dd26 move udp server to test file, add link to BEP15 2022-05-11 21:13:29 -03:00
Victor Shyba
4a0bf8a702 add torrent udp tracker client, server and tests 2022-05-11 21:13:29 -03:00
Victor Shyba
d0e715feb9
Merge pull request #3609 from lbryio/pin_scribe
CI: pin scribe, fix exchange rate manager test
2022-05-11 21:13:00 -03:00
Victor Shyba
fd73412f12 test_exchange_rate_manager: bump value 2022-05-11 20:28:06 -03:00
Victor Shyba
3819552861 try usedevelop=true 2022-05-11 20:14:55 -03:00
Victor Shyba
ca6fd5b7b9 fix scribe pinning 2022-05-11 20:14:44 -03:00
Lex Berezhny
b8867cd18c release.py script changed to use gh auth login for authentication 2022-04-10 23:28:16 -04:00
Lex Berezhny
8209eafc6b v0.108.0 2022-04-10 23:25:15 -04:00
Lex Berezhny
858e72a555
Merge pull request #3595 from lbryio/default_feer_per_name_char
pin scribe to specific version
2022-04-08 13:49:11 -04:00
Lex Berezhny
d3880fffa0 pin scribe to specific version 2022-04-08 13:48:30 -04:00
Lex Berezhny
0a51898722
Merge pull request #3593 from lbryio/default_feer_per_name_char
set the default per character fee for claims to zero
2022-04-08 13:46:54 -04:00
Lex Berezhny
63cef81015 fix scribe server version test 2022-04-08 13:22:51 -04:00
Lex Berezhny
9279865078 add sleep to transaction show test per jack suggestion 2022-04-08 12:59:25 -04:00
Lex Berezhny
fba7fc7aba fix scribe server version test 2022-04-08 12:53:19 -04:00
Lex Berezhny
a3d9d5bce7 fix transaction unit test 2022-04-08 11:05:45 -04:00
Lex Berezhny
23ecbc8ebe set the default per character fee for claims to zero 2022-04-08 10:58:02 -04:00
Lex Berezhny
42b2dbd92e
Merge pull request #3572 from orblivion/json-schema
Add wallet json-schema, validate in one test.
2022-04-08 10:56:58 -04:00
Lex Berezhny
37eb55375a only install jsonschema during testing 2022-04-08 10:56:18 -04:00
Lex Berezhny
94bf357817 cleanup paths 2022-04-08 10:56:18 -04:00
Daniel Krol
eca69391ef Add wallet json-schema, validate in one test. 2022-04-08 10:56:18 -04:00
Lex Berezhny
d0c5b32a90
Merge pull request #3575 from lbryio/spend_time_locked
added `account_deposit` command which is able to deposit time locked transaction into wallet
2022-04-08 10:52:08 -04:00
Lex Berezhny
84ef52cf4d fix redeem scripthash test 2022-04-08 10:11:11 -04:00
Lex Berezhny
8fb14bf713 remove command not available in lbcd 2022-04-08 09:59:22 -04:00
Lex Berezhny
16eb50a291 working jsonrpc_account_deposit 2022-04-08 09:57:15 -04:00
Lex Berezhny
dd503fbb82 set locktime from script 2022-04-08 09:57:15 -04:00
Lex Berezhny
ae79314869 wip 2022-04-08 09:57:15 -04:00
Lex Berezhny
0cbc514a8e account_deposit command added which accepts time locked TXs 2022-04-08 09:57:15 -04:00
Lex Berezhny
5777f3e15c wip 2022-04-08 09:57:15 -04:00
Lex Berezhny
8cdcd770c0
Merge pull request #3590 from lbryio/fix-address_list-pagination
fix pagination with `address_list` command
2022-04-06 09:52:41 -04:00
Lex Berezhny
2d20458bc2 re-use existing constraints cleanup function 2022-04-06 09:09:39 -04:00
zeppi
2bd2088248 bugfix 2022-04-06 09:09:39 -04:00
zeppi
5818270803 fix address_list pagination 2022-04-06 09:09:39 -04:00
Victor Shyba
79a5f0e375 lint 2022-04-05 00:35:48 -03:00
Victor Shyba
c830784f65
Merge pull request #3586 from AlessandroSpallina/master
fix #3530 added error log when tcp port is already in use
2022-04-05 00:04:59 -03:00
Victor Shyba
3fc538104d v0.107.2 2022-03-31 17:19:58 -03:00
AlessandroSpallina
96490fdb15
Merge branch 'master' into master 2022-03-29 13:50:57 +02:00
Victor Shyba
5a0c225c6f v0.107.0 2022-03-28 15:56:06 -03:00
Lex Berezhny
c3e524cb8b
Merge pull request #3588 from lbryio/scribe
move `lbry.wallet.server` to new project called `scribe`, switch from using `lbrycrd` to `lbcd` in integration tests
2022-03-28 00:14:54 -04:00
Jack Robison
9faf6e46ca move lbry.wallet.server to new project called scribe
switch from using lbrycrd to lbcd
2022-03-27 23:33:26 -04:00
Victor Shyba
e89acac235
Merge pull request #3585 from lbryio/fix_blob_db_queries
Fixes bugs on disk space management and stream recovery
2022-03-24 21:01:14 -03:00
Victor Shyba
200761ff13 make added_on a required parameter on BlobInfo, fix callers 2022-03-24 19:51:48 -03:00
Victor Shyba
cb78e95e3d add missing space on query, typo 2022-03-23 13:40:01 -03:00
AlessandroSpallina
f01cf98d62 fix #3530 added error log when tcp port is already in use 2022-03-22 17:17:41 +01:00
Victor Shyba
c9c2495611 if a blob file exists but is pending on db, fix on startup 2022-03-21 21:58:36 -03:00
Victor Shyba
aac72fa512 fix bug where recovery doesnt update blob status 2022-03-21 21:33:33 -03:00
Victor Shyba
c5e2f19dde fix bug where added_on is always 0 for downloads 2022-03-21 04:38:51 -03:00
Victor Shyba
34bd9e5cb4 exclude sd blobs from calculation and make them be picked last on removal 2022-03-21 04:26:27 -03:00
Lex Berezhny
ad489ed606
Merge pull request #3581 from lbryio/deterministic_channel_keys_post_unlock
eagerly load deterministic channel keys immediately after wallet is unlocked
2022-03-14 12:36:04 -04:00
Lex Berezhny
bb541901d9 fix tests 2022-03-13 21:30:38 -04:00
Lex Berezhny
ca4ba19a5e fixes #3577 2022-03-13 20:42:34 -04:00
Victor Shyba
f05943ff79 implement announcer as a consumer task on gather 2022-03-02 13:00:34 -03:00
Victor Shyba
7ded8a1333 make active an explicit ordered dict 2022-03-02 13:00:34 -03:00
Victor Shyba
c2478d4add remove unused search rounds 2022-03-02 13:00:34 -03:00
Victor Shyba
f69747bc89 timeout is now supported on dht tests 2022-03-02 13:00:34 -03:00
Victor Shyba
441cc950aa fix and enable test_blob_announcer 2022-03-02 13:00:34 -03:00
Victor Shyba
a76a0ac8c4 simplify dht mock and restore clock after accelerating 2022-03-02 13:00:34 -03:00
Victor Shyba
8b1009161a better representation of kademliapeer on debug logs 2022-03-02 13:00:34 -03:00
Victor Shyba
868a620e91 add a way to wait announcements to finish so tests are reliable 2022-03-02 13:00:34 -03:00
Victor Shyba
a0e34b0bc8 make timeout handler immune to asyncio time tricks 2022-03-02 13:00:34 -03:00
Victor Shyba
612dbcb2f3 allow running some extra probes for k replacements 2022-03-02 13:00:34 -03:00
Victor Shyba
b3614d965d remove all references to bottoming out 2022-03-02 13:00:34 -03:00
Victor Shyba
5d7137255e no stop condition, let it exhaust 2022-03-02 13:00:34 -03:00
Victor Shyba
6ff867ef55 bottoming out is now warning and no results for peer search 2022-03-02 13:00:34 -03:00
Victor Shyba
c14915df29 don't probe peers too far from the top closest 2022-03-02 13:00:34 -03:00
Victor Shyba
7d4966e2ae use a dict for the active queue 2022-03-02 13:00:34 -03:00
Victor Shyba
3876e0317d log bottom out of peer search in debug, show short key id for find value 2022-03-02 13:00:34 -03:00
Victor Shyba
0b2b10f759 bump bottom out limit of peer search so people can use 100 concurrent announcers 2022-03-02 13:00:34 -03:00
Victor Shyba
9a79b33664 wait until k peers are ready. do not double add peers 2022-03-02 13:00:34 -03:00
Victor Shyba
af1a6edd15 only return good (contacted) peers 2022-03-02 13:00:34 -03:00
Victor Shyba
b78929f4d5 reset closest peer on failure 2022-03-02 13:00:34 -03:00
Victor Shyba
fb6e342043 add peers from shortlist regardless, but check from other nodes 2022-03-02 13:00:34 -03:00
Victor Shyba
0faa2d35da bump split index to 2 2022-03-02 13:00:34 -03:00
Victor Shyba
511e57c231 fix distance sorting and improve logging 2022-03-02 13:00:34 -03:00
Victor Shyba
d762d675c4 closest peer is only ready when it was contacted and isn't known to be bad 2022-03-02 13:00:34 -03:00
Victor Shyba
3fdadee87c dont probe and ignore bad peers 2022-03-02 13:00:34 -03:00
Victor Shyba
1aa4d9d585 simplify, genaralize to any size and fix tests 2022-02-28 13:06:51 -03:00
Victor Shyba
8019f4bdb3 stop after finding what to download 2022-02-28 13:06:51 -03:00
Victor Shyba
ca65c1ebc5 replace duplicated code 2022-02-28 13:06:51 -03:00
Victor Shyba
f0e47aae86 add get_colliding_prefix_bits, docs and tests 2022-02-28 13:06:51 -03:00
Victor Shyba
dc7cd545ba extract method and avoid using hash builtin name 2022-02-28 13:06:51 -03:00
Victor Shyba
76bd59d82e extract min_prefix_colliding_bits to a contanst 2022-02-28 13:06:51 -03:00
Victor Shyba
461687ffb4 check that the stored blob is at least 1 prefix byte close to peer id 2022-02-28 13:06:51 -03:00
Victor Shyba
dd5b9ca81b add migrator to set head blobs should_announce=0 2022-02-20 22:33:57 -03:00
Victor Shyba
89ed04f8a7 fix test_announces 2022-02-20 22:33:57 -03:00
Victor Shyba
ec0d9f06c5 do not search for the head blob 2022-02-20 22:33:57 -03:00
Victor Shyba
03b59ac6fc dont set head blob to announce on save 2022-02-20 22:33:57 -03:00
Victor Shyba
43ac3336d7 break tie by length 2022-02-20 22:24:04 -03:00
Victor Shyba
d12c78db74 fix and test case for blob_clean after disabling network storage 2022-02-20 22:24:04 -03:00
Jack Robison
bfaf1b0957
Merge pull request #3564 from lbryio/fix_downloader_losing_peers
fix handling re-adding lost peers during download
2022-02-16 11:55:22 -05:00
Victor Shyba
bb60c385d5 put back all the peers, get rid of re_add 2022-02-08 21:41:52 -03:00
Alex Grin
c96d1d9c32
Merge pull request #3537 from lbryio/repost_update 2022-02-08 12:20:20 -05:00
Alex Grintsvayg
7c7a0d4bdf
let stream_update work on non-stream claims 2022-02-08 09:28:17 -05:00
Lex Berezhny
cc829a7bf4
Merge pull request #3558 from lbryio/jeffreypicard-patch-1
Update __init__.py
2022-02-04 12:36:01 -05:00
Jeffrey Picard
e0ea6383e2
Update __init__.py
Update go hub binary to fix es sync test.
2022-02-04 12:17:19 -05:00
Lex Berezhny
bcec5dc2ae
Merge pull request #3556 from lbryio/txo_dust_prevention
prevent creation of change which is below the dust threshold of 1000 dewies
2022-02-04 12:08:16 -05:00
Lex Berezhny
cba9c16a06 fix 2022-02-04 12:07:41 -05:00
Lex Berezhny
dd68fb077b prevent creation of change which is below the dust threshold of 1000 dewies 2022-02-04 12:07:41 -05:00
Jack Robison
c2294e97db
Merge pull request #3552 from lbryio/bump_dht_cache
Increase DHT peer manager cache size to 16384
2022-02-04 11:59:19 -05:00
Victor Shyba
c0f512ace7 bump DHT peer manager cache to 16384 2022-02-02 16:54:42 -03:00
Lex Berezhny
3305eb67c6
Merge pull request #3548 from lbryio/announce_metrics
Add optional Prometheus metrics for DHT announcements
2022-02-02 11:06:48 -05:00
Victor Shyba
c9d637b4da add gauge for queue size 2022-02-02 11:56:42 -03:00
Victor Shyba
ae3e8fadf5 count announcements and how many peers we were able to announce to 2022-02-02 11:56:42 -03:00
Lex Berezhny
a1abd94387
Merge pull request #3542 from eug3nix/gh_3481_file_type_detection
file type detection now looks inside the file to determine the type, in addition to using the file extension
2022-01-31 10:29:47 -05:00
Eugene Dubinin
9b463a8cab adds tests for guess_media_type
removes unnecessary comments
2022-01-29 20:49:42 +02:00
Eugene Dubinin
babc54a240 adjusts code style 2022-01-29 15:25:17 +02:00
Eugene Dubinin
5836a93b21 fixes KeyError on missing synonyms 2022-01-29 15:25:17 +02:00
Eugene Dubinin
557348e345 detect media_type from the file contents 2022-01-29 15:25:17 +02:00
Lex Berezhny
9adfec6b00
Merge pull request #3549 from lbryio/wallet_lock_w_deterministic_channels
wallet locking/unlocking no longer breaks deterministic channel keys
2022-01-26 11:17:55 -05:00
Lex Berezhny
3a496902f8 wallet locking/unlocking no longer breaks deterministic channel keys 2022-01-24 09:45:08 -05:00
Lex Berezhny
b5ead91746
Merge pull request #3534 from lbryio/normalize_signatures
drop dependency on cryptography library in wallet module
2022-01-17 13:38:20 -05:00
Lex Berezhny
302461b446 updated based on code review 2022-01-17 11:08:28 -05:00
Lex Berezhny
ac201c718e drop dependency on cryptography library in wallet module 2022-01-17 10:43:59 -05:00
Jack Robison
f78e3825ca
Merge pull request #3500 from lbryio/fix_script
Add Prometheus metrics for DHT internals
2022-01-14 12:46:28 -05:00
Victor Shyba
0618053bd4 remove request_flight metric 2022-01-12 12:41:04 -03:00
Victor Shyba
8e6fa3490c disable CSV endpoints by default 2022-01-12 12:39:23 -03:00
Victor Shyba
8a1a1a4000 remove estimation endpoints as that is done over prometheus metrics now 2022-01-12 12:39:23 -03:00
Victor Shyba
fd9dcbf9a8 add granular metric for stored blob prefix, for network announcements calculation 2022-01-12 12:39:23 -03:00
Victor Shyba
beb8583436 change colliding bits metric to gauge 2022-01-12 12:39:23 -03:00
Victor Shyba
b44e2c0b38 count bit collisions between 8 and 16 2022-01-12 12:39:23 -03:00
Victor Shyba
06e94640b5 add counter for peers with colliding bytes 2022-01-12 12:39:23 -03:00
Victor Shyba
ff36bdc802 add requests in flight and error 2022-01-12 12:39:23 -03:00
Victor Shyba
46f576de46 add request received 2022-01-12 12:39:23 -03:00
Victor Shyba
7b09c34fce add request_sent and request_time metric on dht 2022-01-12 12:39:23 -03:00
Victor Shyba
a22f50aa84 add storing_peers and peer_manager_keys 2022-01-12 12:39:23 -03:00
Victor Shyba
2d9130b4e0 prometheus: move blobs_stored and peers to SDK. add buckets_in_routing_table 2022-01-12 12:39:23 -03:00
Victor Shyba
470ee72462 add passive estimation to prometheus 2022-01-12 12:39:23 -03:00
Victor Shyba
add147b409 fix missing async 2022-01-12 12:39:23 -03:00
Victor Shyba
371df6e6c2 keep same node id between runs 2022-01-12 12:39:23 -03:00
Victor Shyba
7ed5fe8f66 add semaphore on active estimation to avoid abuse 2022-01-12 12:39:23 -03:00
Victor Shyba
a6ca7a6f38 same api across different estimation methods 2022-01-12 12:39:23 -03:00
Victor Shyba
1c857b8dd8 be explicit about ignoring params 2022-01-12 12:39:23 -03:00
Victor Shyba
87ff3f95ff better endpoint names, small docs 2022-01-12 12:39:23 -03:00
Victor Shyba
5cb4c06d0c add prefix_neighbors_count to routing table debug api 2022-01-12 12:39:23 -03:00
Jack Robison
e7d9079389 improve script 2022-01-12 12:39:23 -03:00
Victor Shyba
9cdcff0e1e first attempt at crawling 2022-01-12 12:39:23 -03:00
Lex Berezhny
a4dce8cf9f
Merge pull request #3535 from vertbyqb/hexdata-string
convert hexdata argument to a string before signing in `channel_sign` command
2022-01-10 09:48:41 -05:00
Lex Berezhny
aaa11c02bf added integration test 2022-01-10 08:46:10 -05:00
vertbyqb
d2ebbf5db6 jsonrpc_channel_sign - Convert hexdata to a string before signing
Fixes #3533
2022-01-10 08:46:10 -05:00
Jack Robison
e6efc1ad4a
Merge pull request #3538 from lbryio/dht_memory
Unify and fix DHT memory caches for peer manager
2022-01-07 11:29:44 -05:00
Victor Shyba
a8523996a9 extract cache values, increase peer cache to 2048 2022-01-07 12:58:52 -03:00
Victor Shyba
f586de2bbe DHT bugfix: failures tracking should be bound to 2048 LRU cache size 2022-01-07 12:46:00 -03:00
Victor Shyba
7df02303b2 fix missing docopt argument 2022-01-05 17:10:31 -03:00
Victor Shyba
f89c75e642 bump hub version to latest supporting sd_hash search 2022-01-05 17:10:31 -03:00
Victor Shyba
d2c1961101 update hub protobuf including sd_hash field 2022-01-05 17:10:31 -03:00
Victor Shyba
2a4c5a48bf increase indexed sd_hash prefix to 4 chars 2022-01-05 17:10:31 -03:00
Victor Shyba
5f5f39a4aa enable and test prefix search for sd hash 2022-01-05 17:10:31 -03:00
Victor Shyba
df54cc04af sync and search sd_hash 2022-01-05 17:10:31 -03:00
Victor Shyba
0439616480 add test 2022-01-05 17:10:31 -03:00
Victor Shyba
19fa274227 add sd hash to API 2022-01-05 17:10:31 -03:00
Lex Berezhny
8076000c27
Merge pull request #3450 from lbryio/deterministic_channel_keys
deterministic channel keys (requires wallet server re-sync)
2021-12-23 15:38:15 -05:00
Lex Berezhny
c80b30f070 test another signed claim by ytsync 2021-12-22 18:29:46 -05:00
Lex Berezhny
486d5c48b0 takeover tests fix 2021-12-22 18:29:46 -05:00
Lex Berezhny
4822792ee2 create nondetermnistic channel in test to replicate old test behavior 2021-12-22 18:29:46 -05:00
Lex Berezhny
569f1d42b1 fix tests 2021-12-22 18:29:46 -05:00
Lex Berezhny
23c10faff5 lint 2021-12-22 18:29:46 -05:00
Lex Berezhny
1eaa195363 reduced crypto dependency in wallet to coincurve 2021-12-22 18:29:46 -05:00
Lex Berezhny
fb57cfa5d8 moved imports for lint 2021-12-22 18:29:46 -05:00
Lex Berezhny
d33086c8f7 deleted extraneous test 2021-12-22 18:29:46 -05:00
Lex Berezhny
d815a6f02c use ecdsa for signing/veryfing instead of coincurve due to compatibility issues 2021-12-22 18:29:46 -05:00
Lex Berezhny
8216f4a873 work in progress 2021-12-22 18:29:46 -05:00
Lex Berezhny
e4cc4521d9 channel key generation no longer arbitrarily bounded 2021-12-22 18:29:46 -05:00
Lex Berezhny
6bd9b3744d progress, channel keys generate deterministically now 2021-12-22 18:29:46 -05:00
Lex Berezhny
f741b00768 progress on deterministic channel keys 2021-12-22 18:29:46 -05:00
Lex Berezhny
5eb95d7dd4
Merge pull request #3529 from lbryio/change_default_coin_selection_strategy
changes default coin selection strategy from standard to prefer_confirmed
2021-12-22 11:28:22 -05:00
Lex Berezhny
e5268f43e7 changes default coin selection strategy from standard to prefer_confirmed 2021-12-21 10:22:09 -05:00
Victor Shyba
54d6fb9da4 do not limit DHT results by K, respect max_results 2021-12-09 14:34:55 -03:00
Victor Shyba
3d5c9cc1c2 clarify DHT debug logging on key and operation 2021-12-09 14:32:30 -03:00
Alex Grin
442326f1d8
Merge pull request #3499 from lbryio/multiple-release-time-constraints
@jeffreypicard assures me these timeouts are safe to ignore
2021-12-06 11:43:46 -05:00
Jeffrey Picard
d66f46e07b Switch RangeField back to ints 2021-12-03 18:12:38 -05:00
Jeffrey Picard
757b53443d Try forcing tox reset 2021-12-03 17:42:56 -05:00
Jeffrey Picard
3436965b33 Debugging 2021-12-03 17:22:52 -05:00
Jeffrey Picard
df71132957
Update es version in workflow 2021-12-03 13:03:00 -05:00
Jeffrey Picard
1b322dc404
Update protobufs, go hub shim, and claim test. 2021-12-03 13:03:00 -05:00
Jack Robison
58341f4ff1
remove unused ES fields 2021-12-03 13:03:00 -05:00
Jack Robison
0d3ca80008
support lists of constraints for all range fields 2021-12-03 13:03:00 -05:00
Lex Berezhny
63437712cd
Merge pull request #3490 from ghost/integration_test_setup_cleanup_timeouts
added timeout of async operations to integration test setup/teardown
2021-12-02 19:52:44 -05:00
Jack Robison
26d0e87f46 v0.106.0 2021-12-02 17:17:00 -05:00
Jack Robison
2cad4fa1ce update json docs 2021-12-02 14:51:52 -05:00
Jack Robison
7bb293e5d6 update claim_search doc
backward compatibility for `trending_mixed`, `trending_local`, `trending_global`, and `trending_group` args to `claim_search`
2021-12-02 14:51:52 -05:00
Lex Berezhny
e4777f9314
Merge branch 'master' into integration_test_setup_cleanup_timeouts 2021-12-01 22:08:18 -05:00
Jack Robison
3508f562a7
update json docs 2021-12-01 18:47:03 -05:00
Jack Robison
1aa66c6038
update header checkpoints 2021-12-01 18:46:24 -05:00
Victor Shyba
e7458edb72 test case for stream_type search on claims missing source + fix 2021-12-01 18:42:47 -05:00
Lex Berezhny
7f97013703
Merge pull request #3497 from lbryio/fee_per_name_env_var
fee per name env var
2021-12-01 11:26:00 -05:00
Lex Berezhny
9e43060d41 fee per name env var 2021-12-01 10:22:34 -05:00
FemtosecondLaser
d69486fb6e returned conditional check in add_timeout() as it was making test_node.py tests unhappy 2021-11-30 01:01:35 +00:00
FemtosecondLaser
d4ebfdbc3c removed conditional check in add_timeout() 2021-11-29 22:56:50 +00:00
FemtosecondLaser
e00c3db71a
Merge branch 'master' into integration_test_setup_cleanup_timeouts 2021-11-29 21:50:05 +00:00
Victor Shyba
11c3ea0b87 fix typo from arg name 2021-11-24 13:05:43 -03:00
Jack Robison
7531401623
keep touched_or_deleted records 2021-11-21 13:52:03 -05:00
FemtosecondLaser
e6c1dc251e changed addTimeout to add_timeout for lint compliance 2021-11-20 00:47:46 +00:00
FemtosecondLaser
dca7977051 added timeout of async operations to integration test setup/teardown 2021-11-20 00:22:25 +00:00
Victor Shyba
d19e07d661 add blob endpoint for listing announced blobs 2021-11-17 13:27:19 -03:00
Victor Shyba
751ff6e21f add /peers.csv to monitoring endpoint 2021-11-17 13:27:19 -03:00
Brendon J. Brewer
3f6fe995b8 Rename trending 2021-11-16 10:59:10 -05:00
Jack Robison
1e00fb369d fix missing es notification for support amount changing 2021-11-15 00:58:18 -05:00
Jack Robison
54b522383a improve tests 2021-11-15 00:58:18 -05:00
Jack Robison
90a7de3b5c improve resolve tests 2021-11-15 00:58:18 -05:00
Jack Robison
3fe1582432 fix duplicate trending notification to ES 2021-11-15 00:58:18 -05:00
Jack Robison
85eddd2100 fix effective amount for resolve/ES being off while claims/supports are unactivated 2021-11-15 00:58:18 -05:00
Jack Robison
f5f8775c59 fix test_colliding_short_id 2021-11-10 13:02:28 -03:00
Jack Robison
0ca98678f7 update default tcp/blob port to be the same as the default udp/dht port (4444) 2021-11-10 13:02:28 -03:00
Victor Shyba
a19060c08d log unexpected errors, rename task/loop 2021-11-09 14:27:06 -05:00
Victor Shyba
fa2ad88cc4 clear cache on test assertions 2021-11-09 14:27:06 -05:00
Victor Shyba
63cbcd0956 make sure the downloader always stops gracefully 2021-11-09 14:27:06 -05:00
Victor Shyba
d6d0ebf8f4 cache space stats from running components so status is instant 2021-11-09 14:27:06 -05:00
Victor Shyba
0d810d92ca add index for blob table so size summaries are faster 2021-11-09 14:27:06 -05:00
Victor Shyba
1ff914a6f4 download from stored announcements and dont reannounce 2021-11-09 14:27:06 -05:00
Victor Shyba
5959b1be72 improve disk space manager status, include more info and unify space queries 2021-11-09 14:27:06 -05:00
Victor Shyba
d12a214c05 normal_blobs->stream_blobs, proactive->background 2021-11-09 14:27:06 -05:00
Victor Shyba
3a83052f2e fix free space calculation, test it and give a margin of 10mb before starting so it doesnt insist when full 2021-11-09 14:27:06 -05:00
Victor Shyba
510b44ca92 move more logic out of the downloader component 2021-11-09 14:27:06 -05:00
Victor Shyba
15edb6756d extract background downloader to its own class 2021-11-09 14:27:06 -05:00
Victor Shyba
fbfd02b08b add analytics event for network disk space 2021-11-09 14:27:06 -05:00
Victor Shyba
b39c26fc86 announce orphan blobs manually, as that was done when save stream 2021-11-09 14:27:06 -05:00
Victor Shyba
95b2c8d175 cleanup background downloader blobs from conf 2021-11-09 14:27:06 -05:00
Victor Shyba
d52748b09f separated network seeding space metrics 2021-11-09 14:27:06 -05:00
Victor Shyba
34d18a3a9a don't save streams for network blobs and bypass disk space manager 2021-11-09 14:27:06 -05:00
Victor Shyba
3b27d6a9b5 add conf for network seeding space limit 2021-11-09 14:27:06 -05:00
Victor Shyba
703c391f99 schedule the download task instead 2021-11-09 14:27:06 -05:00
Victor Shyba
4f1dc29df1 fix unit tests from component dependency chain changes 2021-11-09 14:27:06 -05:00
Victor Shyba
13667df374 download from DHT 2021-11-09 14:27:06 -05:00
Victor Shyba
8800d6985f drop channel support, prepare to hook into DHT 2021-11-09 14:27:06 -05:00
Victor Shyba
364b8f2605 handle case where something that isn't a sd blob gets hit 2021-11-09 14:27:06 -05:00
Victor Shyba
67b9ea9deb no api yet 2021-11-09 14:27:06 -05:00
Victor Shyba
b78f2336a7 download only blobs 2021-11-09 14:27:06 -05:00
Victor Shyba
c7ba637c7d fix tests 2021-11-09 14:27:06 -05:00
Victor Shyba
23a5ce3df7 fix exception arguments 2021-11-09 14:27:06 -05:00
Victor Shyba
8f88e28e50 test add/remove/list subscriptions 2021-11-09 14:27:06 -05:00
Victor Shyba
9cf6139557 fix and test main api 2021-11-09 14:27:06 -05:00
Victor Shyba
d556065a8b download all blobs and check that on tests 2021-11-09 14:27:06 -05:00
Victor Shyba
951716f7dc create downloader component and initial tests 2021-11-09 14:27:06 -05:00
Victor Shyba
1ddc7ddda3 with the fix we no longer need to restart the stream 2021-11-08 10:50:47 -05:00
Victor Shyba
903ed9f3dc fix tests by checking there are actual blobs being deleted 2021-11-08 10:50:47 -05:00
Victor Shyba
c42b76dcb8 dont lose results on duplicates, just warn 2021-11-08 10:50:47 -05:00
Victor Shyba
a73582d9ae remove tried_for_this_blob so banned peers are retried for same blob 2021-11-08 10:50:47 -05:00
Cristian Vicas
42c4fc7557 Bug [#2070] where blob_get RPC timed out.
Both stream.downloader and blob_exchange.downloader paths are adding the fixed_peers list to the DHT node.
Tested jsonrpc_blob_get daemon call.

Bug [#2070] where blob_get RPC timed out.

Both stream.downloader and blob_exchange.downloader paths are adding the fixed_peers list to the DHT node.
Tested jsonrpc_blob_get daemon call.
2021-11-08 10:49:48 -05:00
Jack Robison
ddbbb6f1dd
use mempool cache in transaction_get_batch 2021-10-27 20:19:08 -04:00
Lex Berezhny
ff21a92330
Merge pull request #3457 from FemtosecondLaser/feature/3270-check-default-download-dir-writable
Modified ensure_directory_exists() to check if the directory is writable by the process.
2021-10-27 11:00:13 -04:00
FemtosecondLaser
07f76f7ad1 Added an integration test covering the following scenario:
On start, if download dir is non-writable - daemon terminates with a helpful message.
2021-10-26 11:17:52 +01:00
Jack Robison
c90ccffd7b
Update docker-compose-wallet-server.yml 2021-10-25 14:20:39 -04:00
Jack Robison
a00d5f18af
add script to setup docker volumes from snapshots 2021-10-24 16:25:34 -04:00
Jack Robison
1e391d211b
fix attempting to update trending on abandoned claims 2021-10-23 18:39:04 -04:00
FemtosecondLaser
d87f9672fa Improved the readability of the tests. 2021-10-23 13:12:49 +01:00
FemtosecondLaser
2b5838aa01 Changed the tests to execute against a real file system instead of a fake one. 2021-10-23 02:52:58 +01:00
Jack Robison
e10486d6ec
update docs 2021-10-22 16:51:59 -04:00
Jack Robison
1a74d6604d skip loading tx/claim caches in the elastic sync script when not needed 2021-10-22 15:10:35 -04:00
Alex Grin
6d118536b6
Merge pull request #3460 from lbryio/dht_seed_script_metrics 2021-10-22 12:44:16 -04:00
Alex Grin
ca4d758db9
Merge branch 'master' into dht_seed_script_metrics 2021-10-22 11:54:19 -04:00
Victor Shyba
dc18c26aa4 add optional prometheus to dht_node script 2021-10-22 03:39:46 -03:00
Jack Robison
48505c2968 update trending with help from @eggplantbren 2021-10-21 00:17:12 -04:00
Jack Robison
a98ea1e66a update sync script to handle ES falling behind leveldb on shutdown 2021-10-20 23:41:11 -04:00
Jack Robison
3dec697816 logging 2021-10-20 23:41:11 -04:00
Jack Robison
88fd41e597 update docker 2021-10-20 23:41:11 -04:00
Jack Robison
b05d071a1c update Env to accept parameters from cli args 2021-10-20 23:41:11 -04:00
Jack Robison
a27d3b9689 set default CACHE_MB to 1024mb and the default QUERY_TIMEOUT_MS to 10s 2021-10-20 23:41:11 -04:00
Jack Robison
1facc0cd01 remove unused hub env settings 2021-10-20 23:41:11 -04:00
FemtosecondLaser
837f91d830 renamed the test class to be more specific about the sut 2021-10-21 00:31:02 +01:00
FemtosecondLaser
9c5f5aefb0 removed redundant tests
renamed a test to be more specific about the kind of the precondition
2021-10-21 00:27:31 +01:00
FemtosecondLaser
6b8d4a444b Modified ensure_directory_exists() to check if the directory is writable by the process. 2021-10-20 15:26:16 +01:00
Jack Robison
6bef09a3b1 update lbry-hub-elastic-sync to support resyncing recent blocks 2021-10-19 15:53:20 -04:00
Jack Robison
e35319e5a2 add CACHE_ALL_CLAIM_TXOS hub setting 2021-10-19 15:53:20 -04:00
Jack Robison
0e548b3812 remove dead code 2021-10-19 15:53:20 -04:00
Jack Robison
bfac02ccab add CACHE_ALL_TX_HASHES setting to optionally use more memory to save i/o 2021-10-19 15:53:20 -04:00
Jack Robison
7ea1a2b361 sleeps 2021-10-19 15:53:20 -04:00
Jack Robison
99df418f1d improve resolve caching 2021-10-19 15:53:20 -04:00
Jack Robison
6416d8ce9c threadpools for block processor and es sync reader 2021-10-19 15:53:20 -04:00
Jack Robison
22b43a2b01 doc strings 2021-10-19 15:53:20 -04:00
Jack Robison
05e5d24c5e improve claims_producer performance 2021-10-19 15:53:20 -04:00
Jack Robison
eabcc30367 resolve lru cache 2021-10-19 15:53:20 -04:00
Jack Robison
f5e0ef5223 add block_txs index 2021-10-19 15:53:20 -04:00
Jack Robison
f46d9330b0 smaller caches 2021-10-19 15:53:20 -04:00
Jack Robison
b62a0b4607 Update daemon.py
docstring
2021-10-15 09:40:15 -04:00
Cristian Vicas
1f044321fb Updated documentation for RPC calls: status, blob_list. 2021-10-15 09:40:15 -04:00
Jack Robison
a841d49483
Merge branch 'belikor-fix-wrong-url' 2021-10-15 09:00:59 -04:00
belikor
9509acc490
file_manager: raise new InvalidStreamURLError if the URL is invalid
When using `lbrynet get URL`, if the URL is not a valid URL
the function `url.URL.parse` will raise a `ValueError` exception
which will produce a whole backtrace.

For example, this is the case if we provide a channel name
with a forward slash but without a stream name.
```
lbrynet get @Non-existing/
```

```
Traceback (most recent call last):
  File "/opt/git/lbry-sdk/lbry/file/file_manager.py", line 84, in download_from_uri
    if not URL.parse(uri).has_stream:
  File "/opt/git/lbry-sdk/lbry/schema/url.py", line 114, in parse
    raise ValueError('Invalid LBRY URL')
ValueError: Invalid LBRY URL
WARNING  lbry.extras.daemon.daemon:1110: Error downloading Non-existing/: Invalid LBRY URL
```

Now we raise a new `InvalidStreamURLError` which can be trapped in the upper functions
that use `url.URL.parse` such as `FileManager.download_from_uri`.
If we do this the traceback won't be shown.
```
WARNING  lbry.file.file_manager:252:
Failed to download Non-existing/: Invalid LBRY stream URL: '@Non-existing/'
WARNING  lbry.extras.daemon.daemon:1110:
Error downloading Non-existing/: Invalid LBRY stream URL: '@Non-existing/'
```

This handles the case when trying to download only "channel" parts
without the claim part.
```
lbrynet get @Non-existing
lbrynet get @Non-existing/
lbrynet get Non-existing/
```
2021-10-15 08:59:37 -04:00
Jack Robison
02d356ef12
Merge pull request #3443 from lbryio/fix-resolve-reposted-channel
Fix including channels for reposted claims when resolving a repost
2021-10-08 16:51:40 -04:00
Jack Robison
d3516f299e
clear es attributes during initial sync 2021-10-08 16:34:48 -04:00
Jack Robison
79630767c2
fix setting references on txos in extra_txos 2021-10-08 16:34:15 -04:00
Jack Robison
084a76d075
fix reposted channel being missing from resolve result
-improve names of the resolve related methods in `LevelDB`
2021-10-07 15:09:13 -04:00
Jack Robison
bc6822e397
Merge pull request #3205 from lbryio/leveldb-resolve
drop sqlite in the hub and make resolve handle reorgs
2021-10-07 02:07:48 -04:00
Jack Robison
43432a9e48
fix compactify script 2021-10-07 00:37:55 -04:00
Jack Robison
d64a5bc12f
fix test 2021-10-06 23:53:17 -04:00
Jack Robison
b2922d18e2
move test_transaction_commands, test_internal_transaction_api , and test_transactions into their own runner
-move test_resolve_command to its own runner
2021-10-06 23:53:17 -04:00
Jack Robison
ccf03fc07b
only save undo info for blocks within reorg limit 2021-10-06 12:07:42 -04:00
Jack Robison
a7c45da10c
fix channel count 2021-10-06 00:02:16 -04:00
Jack Robison
e03f01e24a
try to fix test_sqlite_coin_chooser 2021-10-05 19:36:49 -04:00
Jack Robison
0939589557
move test_claim_commands and test_resolve_command into new directory 2021-10-05 17:51:43 -04:00
Jack Robison
8167af9b4a
sort touched or deleted claim hashes 2021-10-05 16:44:49 -04:00
Jack Robison
4cf76123e5
block processor db refactoring
-access db through HubDB class, don't use plyvel.DB directly
-add channel count and support amount prefixes
2021-10-05 16:44:49 -04:00
Jack Robison
01ee4b23e6
fix and add test for abandoning a controlling in the same block a new claim is made 2021-10-05 16:44:49 -04:00
Jack Robison
b198f79214
fix test_sqlite_coin_chooser 2021-10-05 16:44:49 -04:00
Jack Robison
09db868a28
fix ES index name so it stays the same within a test case 2021-10-05 16:44:49 -04:00
Jack Robison
33e8ef75ff
fix bug with early takeover by an update 2021-10-05 16:44:49 -04:00
Jack Robison
11dcb16b14
fix test 2021-10-05 16:44:49 -04:00
Jack Robison
86f21da28b
fix activating non existent claim 2021-10-05 16:44:49 -04:00
Jack Robison
89cd6a9aa4
add tests for takeovers from amount changes in updates before/on/after activation 2021-10-05 16:44:49 -04:00
Jack Robison
18e1256037
batch address history notifications 2021-10-05 16:44:49 -04:00
Jack Robison
02cf478d91
improve leveldb caching 2021-10-05 16:44:49 -04:00
Jack Robison
6ec70192fe
refactor reload_blocking_filtering_streams 2021-10-05 16:44:49 -04:00
Jack Robison
8c75098a9a
fix filtering error upon abandon 2021-10-05 16:44:49 -04:00
Jack Robison
72500f6948
faster read_claim_txos 2021-10-05 16:44:49 -04:00
Jack Robison
37ec9ab464
remove unused executor 2021-10-05 16:44:49 -04:00
Victor Shyba
82fe2a4c8d
fix blocking and filtering 2021-10-05 16:44:49 -04:00
Jack Robison
aa50e6ee66
fix test 2021-10-05 16:44:49 -04:00
Jack Robison
91a07cfaee
fix logging number of notified sessions 2021-10-05 16:44:49 -04:00
Jack Robison
709f5e9a65
fix update that initiates takeover not being delayed 2021-10-05 16:44:49 -04:00
Jack Robison
b2f9ef21cc
use hub binary from https://github.com/lbryio/hub/pull/13 2021-10-05 16:44:49 -04:00
Jack Robison
be6b72edcd
handle invalid release time 2021-10-05 16:44:49 -04:00
Jack Robison
ece2d1e78a
name and normalized -> claim_name and normalized_name
-update generated pb files
2021-10-05 16:44:49 -04:00
Jack Robison
1ee1a5f2a1
fix es sync.py 2021-10-05 16:44:49 -04:00
Jack Robison
a567326853
fix all_claims_producer 2021-10-05 16:44:49 -04:00
Jack Robison
6231861dd6
merge conflicts 2021-10-05 16:44:49 -04:00
Jack Robison
1ff7b77ee0
claim search fixes 2021-10-05 16:44:49 -04:00
Jack Robison
9365708bb2
fix release_time and creation_timestamp 2021-10-05 16:44:49 -04:00
Jack Robison
d23a0a8589
delete unused code 2021-10-05 16:44:49 -04:00
Jack Robison
701b39b043
test_spec_example 2021-10-05 16:44:49 -04:00
Jack Robison
58ad1f3876
non blocking claim producer 2021-10-05 16:44:49 -04:00
Jack Robison
2138e7ea33
fix tests 2021-10-05 16:44:49 -04:00
Jack Robison
32f8c9e59f
renormalization 2021-10-05 16:44:49 -04:00
Jack Robison
57028eab39
add trending integration test 2021-10-05 16:44:49 -04:00
Jack Robison
3a16edd8a6
fix trending overflow 2021-10-05 16:44:49 -04:00
Jack Robison
165f3bb270
refactor trending 2021-10-05 16:44:49 -04:00
Jack Robison
0ba75153f3
trending fixes 2021-10-05 16:44:49 -04:00
Jack Robison
db2789990f
make app backward compatible with trending_score
-update trending decay function to zero out low trending score values faster
2021-10-05 16:44:49 -04:00
Jack Robison
acaf299bcb
log time to update and decay trending in elasticsearch 2021-10-05 16:44:49 -04:00
Jack Robison
1940301824
skip integrity errors for trending spikes 2021-10-05 16:44:49 -04:00
Jack Robison
34576e880d
update trending in elasticsearch
-add TrendingPrefixSpike to leveldb
-expose `TRENDING_HALF_LIFE`, `TRENDING_WHALE_HALF_LIFE` and `TRENDING_WHALE_THRESHOLD` hub settings
2021-10-05 16:44:49 -04:00
Brendon J. Brewer
65c0668d40
constants 2021-10-05 16:44:49 -04:00
Brendon J. Brewer
53bd2bcbfe
Put trending score into ES 2021-10-05 16:44:49 -04:00
Brendon J. Brewer
388724fccb
Mark claims as touched 2021-10-05 16:44:49 -04:00
Jack Robison
231eabb013
fix non normalized canonical urls 2021-10-05 16:44:49 -04:00
Jack Robison
54903fc2ea
handle unicode error for unnormalized names 2021-10-05 16:44:49 -04:00
Jack Robison
3a1baf0700
prefix db 2021-10-05 16:44:49 -04:00
Brendon J. Brewer
0c0e36b6f8
trending 2021-10-05 16:44:49 -04:00
Jack Robison
234c03db09
fix claims not having non-normalized names 2021-10-05 16:44:49 -04:00
Jack Robison
59db5e7889
update test 2021-10-05 16:44:49 -04:00
Jack Robison
28aa7da349
merge conflicts 2021-10-05 16:44:49 -04:00
Jack Robison
c51e344b87
fix missing fields in reposts 2021-10-05 16:44:49 -04:00
Jack Robison
54461dfa75
fix merge conflicts and simplify extract_doc 2021-10-05 16:44:49 -04:00
Jack Robison
2d48e93f74
fix bulk es sync 2021-10-05 16:44:49 -04:00
Jack Robison
af22646322
fix tests 2021-10-05 16:44:49 -04:00
Jack Robison
722b42a93e
fix tests 2021-10-05 16:44:49 -04:00
Jack Robison
8f9e7f77a7
handle invalid claim update 2021-10-05 16:44:49 -04:00
Jack Robison
09bb1ba494
fix keeping claim_hash_to_txo and txo_to_claim in sync 2021-10-05 16:44:49 -04:00
Victor Shyba
d4137428ff
implement blocking and filtering 2021-10-05 16:44:49 -04:00
Jack Robison
b4d6c4f5b7
fix _get_pending_claim_name 2021-10-05 16:44:49 -04:00
Jack Robison
ffbe59ece5
fix applying expiration fork 2021-10-05 16:44:49 -04:00
Jack Robison
fab9c90ccb
update iterators to use pack_partial_key 2021-10-05 16:44:49 -04:00
Jack Robison
fb1a774bc4
delete lbry/wallet/server/storage.py
-expose leveldb lru cache size as `CACHE_MB` hub param
2021-10-05 16:44:49 -04:00
Jack Robison
98bc7d1e0e
remove dead code 2021-10-05 16:44:49 -04:00
Jack Robison
f7622f24b2
non blocking mempool loop 2021-10-05 16:44:49 -04:00
Jack Robison
f0a195a6d4
faster es sync 2021-10-05 16:44:49 -04:00
Jack Robison
180ba27d84
run advance_block in threadpool 2021-10-05 16:44:49 -04:00
Jack Robison
f944671f86
use claim_to_txo cache 2021-10-05 16:44:49 -04:00
Jack Robison
def2903f7d
faster _cached_get_active_amount for claims
-remove dead code
2021-10-05 16:44:49 -04:00
Jack Robison
0273a4e839
fix claim search by fee for claims without fees 2021-10-05 16:44:49 -04:00
Jack Robison
f8d2f02c5d
clear claim_to_txo cache before reading 2021-10-05 16:44:49 -04:00
Jack Robison
25147d8897
handle claims that dont exist in ES sync 2021-10-05 16:44:49 -04:00
Jack Robison
0fb6f05fba
in memory claim_to_txo and txo_to_claim dictionaries 2021-10-05 16:44:49 -04:00
Jack Robison
4e4e899356
fix spend_utxo 2021-10-05 16:44:49 -04:00
Jack Robison
5a01dbf269
split flush from advance_block 2021-10-05 16:44:49 -04:00
Jack Robison
30b923b283
rename extend_ops 2021-10-05 16:44:49 -04:00
Jack Robison
73ba381d20
faster spend_utxo 2021-10-05 16:44:49 -04:00
Jack Robison
1a5912877e
faster get_future_activated 2021-10-05 16:44:49 -04:00
Jack Robison
813e506b68
threadpool 2021-10-05 16:44:49 -04:00
Jack Robison
077ca987f7
cleanup 2021-10-05 16:44:49 -04:00
Jack Robison
c632a7a6a5
fix getting block hash during reorg 2021-10-05 16:44:49 -04:00
Jack Robison
e33e767510
fix test 2021-10-05 16:44:49 -04:00
Jack Robison
ac82617aa9
fix spends in address histories 2021-10-05 16:44:49 -04:00
Jack Robison
a35dfd1fd1
faster es sync 2021-10-05 16:44:49 -04:00
Jack Robison
c28aae9913
fix expiring channels 2021-10-05 16:44:49 -04:00
Jack Robison
c26a99e65c
fix abandoning signed claims in the same tx as their channel
-fix canonical/short url in es
2021-10-05 16:44:49 -04:00
Jack Robison
ca57dcfc2f
handle failure to generate a short id 2021-10-05 16:44:49 -04:00
Jack Robison
df5662dd69
fix resolve by short id 2021-10-05 16:44:49 -04:00
Jack Robison
8927a4889e
tests 2021-10-05 16:44:49 -04:00
Jack Robison
1ac7831f3c
move MemPool into BlockProcessor 2021-10-05 16:44:49 -04:00
Jack Robison
292d272a94
combine MemPool and Notifications classes 2021-10-05 16:44:49 -04:00
Jack Robison
a6ee8dc66e
fix touched hashXs notifications 2021-10-05 16:44:49 -04:00
Jack Robison
496f89f184
reorg claims in the search index 2021-10-05 16:44:49 -04:00
Jack Robison
7a56eff1ac
small fixes 2021-10-05 16:44:49 -04:00
Jack Robison
07e182aa16
rename 2021-10-05 16:44:49 -04:00
Jack Robison
7de06aa1e0
delete stale code 2021-10-05 16:44:49 -04:00
Jack Robison
3955b64405
simplify advance and reorg 2021-10-05 16:44:49 -04:00
Jack Robison
2bb55d681d
update limited_history 2021-10-05 16:44:49 -04:00
Jack Robison
f94e6ac527
update lookup_utxos 2021-10-05 16:44:49 -04:00
Jack Robison
b344f17b86
update RevertableOpStack 2021-10-05 16:44:49 -04:00
Jack Robison
677b8cb633
add remaining db prefixes 2021-10-05 16:44:49 -04:00
Jack Robison
6f3342e09e
update plyvel to 1.3.0
https://github.com/lbryio/lbry-sdk/pull/3205#issuecomment-877564489
2021-10-05 16:44:49 -04:00
Jack Robison
a1ddd762e0
cleanup 2021-10-05 16:44:49 -04:00
Jack Robison
68474e4057
skip es sync during initial hub sync, halt the hub upon finishing initial sync 2021-10-05 16:44:49 -04:00
Jack Robison
a84b9ee396
fix es sync 2021-10-05 16:44:49 -04:00
Jack Robison
b9c2ee745a
fix non localhost elasticsearch 2021-10-05 16:44:49 -04:00
Jack Robison
c91a47fcaa
improve channel invalidation test 2021-10-05 16:44:49 -04:00
Jack Robison
615e489d8d
fix stream_update --clear_channel flag 2021-10-05 16:44:49 -04:00
Jack Robison
c68f9f6f16
fix signed claim invalidation corner cases 2021-10-05 16:44:49 -04:00
Jack Robison
229cb85a6a
extra deletes
-the channel_to_claim/claim_to_channel entries already get deleted when the claim txo is spent
2021-10-05 16:44:49 -04:00
Jack Robison
e5c22fa665
fix has_no_source for reposts 2021-10-05 16:44:49 -04:00
Jack Robison
8bcfff05d7
update channel_to_claim and claim_to_channel at the same time 2021-10-05 16:44:49 -04:00
Jack Robison
6416ee8151
typing and fix error string 2021-10-05 16:44:49 -04:00
Jack Robison
f8eceb48e6
update staged txo_to_claim after invalidating channel sig
-fixes abandon of claim with invalidated signature and an update in same block
2021-10-05 16:44:49 -04:00
Jack Robison
310c483bfa
missing channel_to_claim delete 2021-10-05 16:44:49 -04:00
Jack Robison
a8f20361aa
fix RepostKey 2021-10-05 16:44:49 -04:00
Jack Robison
290be69d99
typing 2021-10-05 16:44:49 -04:00
Jack Robison
3b96bd7ea0
fix 2021-10-05 16:44:49 -04:00
Jack Robison
dc2f22f5fa
cleanup 2021-10-05 16:44:49 -04:00
Jack Robison
821be29f41
rename effective_amount prefix 2021-10-05 16:44:49 -04:00
Jack Robison
52ff1a12ff
fix undeleted claim_to_channel record 2021-10-05 16:44:49 -04:00
Jack Robison
814699ef11
cleanup 2021-10-05 16:44:49 -04:00
Jack Robison
0c30838b25
fix mismatch in claim_to_txo<->txo_to_claim 2021-10-05 16:44:49 -04:00
Jack Robison
cf66c2a1ee
rename things
-fix effective amount integrity error
2021-10-05 16:44:49 -04:00
Jack Robison
2ee419ffca
fix 2021-10-05 16:44:49 -04:00
Jack Robison
bfb9d696d7
pretty print 2021-10-05 16:44:49 -04:00
Jack Robison
bb2a34dd6b
fix duplicate activate 2021-10-05 16:44:49 -04:00
Jack Robison
ed652c0c56
fix updating resolve by effective amount after abandoning support 2021-10-05 16:44:49 -04:00
Jack Robison
1dc961d6eb
use RevertableOpStack in _get_takeover_ops 2021-10-05 16:44:49 -04:00
Jack Robison
d119fcfc98
remove debug prints 2021-10-05 16:44:49 -04:00
Jack Robison
4d3573724a
add RevertableOpStack to verify consistency of ops as they're staged 2021-10-05 16:44:49 -04:00
Jack Robison
8b37a66075
fix fee amount overflow in es 2021-10-05 16:44:49 -04:00
Jack Robison
ba4f32075a
faster claim producer
-make batches of claim txos from the iterator, and sort by tx hash before fetching to maximize cache and read ahead hits
2021-10-05 16:44:49 -04:00
Jack Robison
218be22576
imports 2021-10-05 16:44:49 -04:00
Jack Robison
7688293716
close db in sync script 2021-10-05 16:44:49 -04:00
Jack Robison
458f8533c4
try default block size 2021-10-05 16:44:49 -04:00
Jack Robison
34502752fc
update elastic sync 2021-10-05 16:44:49 -04:00
Jack Robison
d6758fd823
invalidate channel signatures upon channel abandon 2021-10-05 16:44:49 -04:00
Jack Robison
65700e790e
_prepare_claim_for_sync generators 2021-10-05 16:44:49 -04:00
Jack Robison
7c34e4bb96
logging 2021-10-05 16:44:49 -04:00
Jack Robison
d0d6e3563b
use default sync=False during write_batch 2021-10-05 16:44:49 -04:00
Jack Robison
a2619f8c78
genesis_bytes attribute 2021-10-05 16:44:49 -04:00
Jack Robison
42d07fd2f0
fix 2021-10-05 16:44:49 -04:00
Jack Robison
8bea10960f
disable es (revert) 2021-10-05 16:44:49 -04:00
Jack Robison
9cbb19c304
_cached_get_active_amount 2021-10-05 16:44:49 -04:00
Jack Robison
1b94dfd712
fix removing unactivated support 2021-10-05 16:44:49 -04:00
Jack Robison
9f3604d739
debug 2021-10-05 16:44:49 -04:00
Jack Robison
4a1b2be269
leveldb tuning 2021-10-05 16:44:49 -04:00
Jack Robison
962dc1b55b
debug 2021-10-05 16:44:49 -04:00
Jack Robison
07c86502f6
refactor ClaimToTXO prefix 2021-10-05 16:44:49 -04:00
Jack Robison
adb188e5d0
filter abandoned claims from those considered for early activation 2021-10-05 16:44:49 -04:00
Jack Robison
ce031dc6b8
only do early takeover on a larger amount (fix case where they're equal) 2021-10-05 16:44:49 -04:00
Jack Robison
18b5f03247
filter supported claim hashes for claims that dont exist from early takeover/activations 2021-10-05 16:44:49 -04:00
Jack Robison
8a555ecf1c
remove extra open functions 2021-10-05 16:44:49 -04:00
Jack Robison
1b325b9acd
fix flush id 2021-10-05 16:44:49 -04:00
Jack Robison
1bdaddb319
fix clearing pending_support caches upon abandon 2021-10-05 16:44:49 -04:00
Jack Robison
7896e177ef
fix putting spent unactivated supports in removed_active_support 2021-10-05 16:44:49 -04:00
Jack Robison
ce8e659008
fix syncing claim to es where channel is in the same block 2021-10-05 16:44:49 -04:00
Jack Robison
27be5deeb2
ignore activation for headless supports 2021-10-05 16:44:49 -04:00
Jack Robison
515f270c3a
faster get_future_activated 2021-10-05 16:44:49 -04:00
Jack Robison
ffff3bd334
debugging 2021-10-05 16:44:49 -04:00
Jack Robison
f493f13b25
prints 2021-10-05 16:44:49 -04:00
Jack Robison
e605c14b13
flush count 2021-10-05 16:44:49 -04:00
Jack Robison
338488f16d
tests 2021-10-05 16:44:49 -04:00
Jack Robison
2abc67c3e8
reposts 2021-10-05 16:44:49 -04:00
Jack Robison
eb1ba143ec
fix updating the ES search index
-update search index to use ResolveResult tuples
2021-10-05 16:44:49 -04:00
Jack Robison
6f5bca0f67
bid ordered resolve, feed ES claim data from block processor 2021-10-05 16:44:49 -04:00
Jack Robison
407cd8dd4b
fix duplicate update op for early activating claim 2021-10-05 16:44:49 -04:00
Jack Robison
62a4f0fc04
fix early takeovers by not-yet activated claims 2021-10-05 16:44:49 -04:00
Jack Robison
77cde411f1
test_early_takeover_abandoned_controlling_support 2021-10-05 16:44:49 -04:00
Jack Robison
3eb9d23108
require previous_winning arg for get_takeover_name_ops 2021-10-05 16:44:49 -04:00
Jack Robison
410d4aeb21
fix takeover edge case
if a claim with a higher value than that of a claim taking over a name exists but isn't yet activated, activate it early and have it take over the name
2021-10-05 16:44:49 -04:00
Jack Robison
0a28d216fd
comments 2021-10-05 16:44:49 -04:00
Jack Robison
b69faf6920
bid ordered resolve (WIP) 2021-10-05 16:44:49 -04:00
Jack Robison
efb92ea37a
fix udp ping test 2021-10-05 16:44:49 -04:00
Jack Robison
e77f9981df
DBError 2021-10-05 16:44:49 -04:00
Jack Robison
d27c2cc1e9
remove unused COIN file 2021-10-05 16:44:49 -04:00
Jack Robison
586b19675e
claim takeovers 2021-10-05 16:44:49 -04:00
Jack Robison
f2907536b4
move get_expiration_height and claimtrie constants to Coin class 2021-10-05 16:44:49 -04:00
Jack Robison
4aa4e35d1c
tests 2021-10-05 16:44:49 -04:00
Jack Robison
9a11ac06bf
claim activations and takeovers (WIP) 2021-10-05 16:44:49 -04:00
Jack Robison
aa3b18f848
advance_blocks -> advance_block 2021-10-05 16:44:49 -04:00
Jack Robison
103bdc151f
dead code 2021-10-05 16:44:49 -04:00
Jack Robison
6d4c1cd879
LBRYBlockProcessor -> BlockProcessor
- temporarily disable claim_search
2021-10-05 16:44:49 -04:00
Jack Robison
cacbe30871
rebase 2021-10-05 16:44:49 -04:00
Jack Robison
bfeeacb230
tests 2021-10-05 16:44:49 -04:00
Jack Robison
04bb7b4919
add wrapper for getnamesintrie
-used for verifying db state against lbrycrd
2021-10-05 16:44:49 -04:00
Jack Robison
b7df277a5c
db state struct
-remove dead code
2021-10-05 16:44:49 -04:00
Jack Robison
c681041b48
claim expiration 2021-10-05 16:44:49 -04:00
Jack Robison
923834c784
get_claim_by_claim_id 2021-10-05 16:44:49 -04:00
Jack Robison
588edf98be
claims db
-move all leveldb prefixes to DB_PREFIXES enum
-add serializable RevertableOp interface for key/value puts and deletes
-resolve urls from leveldb
2021-10-05 16:44:49 -04:00
Jack Robison
28c603ad5f
transaction_num_mapping 2021-10-05 16:44:49 -04:00
Jack Robison
6988a47e02
disable sqlite in block processor 2021-10-05 16:44:49 -04:00
Jack Robison
2c8ceb1217
named tuples 2021-10-05 16:44:49 -04:00
Jack Robison
ccac4ffa24
consolidate flush_backup 2021-10-05 16:44:49 -04:00
Jack Robison
4258cef9bd
remove lbry.wallet.server.history 2021-10-05 16:44:49 -04:00
Jack Robison
62cc6dfe76
consolidate leveldb block advance/reorg
-move methods from History to LevelDB
2021-10-05 16:44:49 -04:00
Jack Robison
9f224a971b
atomic flush_dbs 2021-10-05 16:44:49 -04:00
Jack Robison
cf5dba9157
combine leveldb databases 2021-10-05 16:44:49 -04:00
Jack Robison
23035b9aa0
Merkle staticmethods 2021-10-05 16:44:49 -04:00
Lex Berezhny
84908ec8ec v0.105.0 2021-10-05 11:29:39 -04:00
Victor Shyba
dade49743b fix file reflect and add test 2021-10-04 19:26:05 -03:00
Lex Berezhny
f29bf35c2a
Merge pull request #3438 from lbryio/disk_space_metrics
metrics reported now include disk space consumed by blobs and what the disk usage limit, if any, is set to
2021-10-03 20:01:50 -04:00
Lex Berezhny
dfa6701c43 disk space metrics 2021-10-03 19:33:18 -04:00
Victor Shyba
763ca69a73 dht: use bytes hex/fromhex instead of binascii 2021-09-30 13:26:33 -03:00
Victor Shyba
6bf3b152bf add grin to dht known list 2021-09-30 13:26:33 -03:00
Victor Shyba
aa19f85996 add madiator to known dht nodes 2021-09-30 13:26:33 -03:00
Victor Shyba
156d89567e add option to set bootstrap_node 2021-09-30 13:26:33 -03:00
Victor Shyba
ecc71baf61 add dockerfile for dht node 2021-09-30 13:26:33 -03:00
Victor Shyba
90c743d963 configure where to save peers 2021-09-30 13:26:33 -03:00
Victor Shyba
b926293fa7 define arg types 2021-09-30 13:26:33 -03:00
Victor Shyba
71a19191f8 add dht seed node script 2021-09-30 13:26:33 -03:00
Victor Shyba
38a0f20a33 fix conflict with imported function 2021-09-30 13:24:17 -03:00
Victor Shyba
c35192108c errors for empyt and misssing file on publish 2021-09-30 13:24:17 -03:00
Victor Shyba
245b564f13 generalize stream empty to argument empty 2021-09-30 13:24:17 -03:00
Victor Shyba
0d8d1ea4f3 empty stream name error for user input 2021-09-30 13:24:17 -03:00
Victor Shyba
27a427a363 error for missing channel private key 2021-09-30 13:24:17 -03:00
Victor Shyba
2ff028a694 error for already purchased claims 2021-09-30 13:24:17 -03:00
Lex Berezhny
c211338218
Merge pull request #3434 from belikor/fix-documentation
fix typo in `file list` arguments list
2021-09-27 11:08:10 -04:00
belikor
8ac89af8bd api.json: correct the error in the generated documentation
From `"name": "blobs_in_stream<blobs_in_stream>"`
to `"name": "blobs_in_stream"`.
2021-09-23 21:01:17 -05:00
belikor
bbbaf59591 daemon: fix documentation in the file_list docstring
This is necessary to produce the `docs/api.json`
(through `scripts/generate_json_api.py`)
with correct information, and to be able to parse this file later on
by other tools.
2021-09-23 21:00:31 -05:00
Lex Berezhny
169419896f v0.104.0 2021-09-22 18:39:01 -04:00
Lex Berezhny
0543dca502 re-enable coveralls 2021-09-22 18:15:13 -04:00
Lex Berezhny
cc6011d57a ubuntu 16.04 is deprecated on github actions, upgrading to 18.04 2021-09-22 18:14:15 -04:00
Lex Berezhny
fc4407ef7e revert release 2021-09-22 18:11:41 -04:00
Lex Berezhny
03735a125f v0.104.0 2021-09-22 14:02:52 -04:00
Lex Berezhny
5baeda9ff1
Merge pull request #3417 from lbryio/preserve_own_blobs
use database to track blob disk space use and preserve own blobs
2021-09-20 11:32:59 -04:00
Lex Berezhny
9b9794b5e0 default is_mine to true during migration 2021-09-20 09:23:42 -04:00
Lex Berezhny
0697d60a48 coveralls still down, will have to merged with coveralls off 2021-09-20 09:01:35 -04:00
Lex Berezhny
cfe6c82a31 tests 2021-09-19 21:38:09 -04:00
Lex Berezhny
3e30228d95 lint 2021-09-15 10:49:03 -04:00
Lex Berezhny
7264b53e5f during disk clean your own sd blob is now kept and file status of deleted files is set to stopped 2021-09-15 10:37:08 -04:00
Lex Berezhny
60836d8523 db migration and other fixes 2021-09-15 09:10:06 -04:00
Lex Berezhny
ef89c2e47a use databse to track blob disk space use and preserve own blobs 2021-09-15 09:10:06 -04:00
Lex Berezhny
2d9e3e1847 v0.103.0 2021-09-14 23:25:32 -04:00
Lex Berezhny
30136a9697 omit just node.py 2021-09-14 23:06:00 -04:00
Lex Berezhny
db7ccd66d3 coverage omit fix 2021-09-14 22:38:39 -04:00
Lex Berezhny
cfe6483102 omit coverage inside tox 2021-09-14 22:20:09 -04:00
Alex Grin
561566e723
Merge pull request #3421 from lbryio/vault_temp
avoid [''] on peers list
2021-09-13 16:14:43 -04:00
Victor Shyba
c2dcc4c898 avoid [''] on peers list 2021-09-13 15:57:21 -03:00
Lex Berezhny
d09bfdc4ff omit orchstr8 stuff since it doesnt always run the same way on every test run 2021-09-12 11:45:52 -04:00
Victor Shyba
358ef4536f add ConflictingInputValueError for claim_id+claim_ids 2021-09-10 18:57:20 -03:00
Victor Shyba
5061a35e66 remove ignored output from hub node 2021-09-10 18:57:20 -03:00
Victor Shyba
cd9a1e8c9e default to legacy search for this release 2021-09-10 18:57:20 -03:00
Victor Shyba
646902e75e only duplicate blockchain CI step 2021-09-10 18:57:20 -03:00
Victor Shyba
40d26cb868 fix error msg to match Go msg 2021-09-10 18:57:20 -03:00
Victor Shyba
b64aa51c0c fix stream_types being an integer 2021-09-10 18:57:20 -03:00
Victor Shyba
8206441834 run CI for old and new setups 2021-09-10 18:57:20 -03:00
Victor Shyba
d713783736 ignore default values 2021-09-10 18:57:20 -03:00
Victor Shyba
57dffaa2ce update hub to beta release 2021-09-10 18:57:20 -03:00
Victor Shyba
9e81dd2360 refactor arguments fixup 2021-09-10 18:57:20 -03:00
Victor Shyba
e2798969d7 claim_id is an invertible field, not a repeated 2021-09-10 18:57:20 -03:00
Victor Shyba
1c31ec66f2 simplify operator handling 2021-09-10 18:57:20 -03:00
Victor Shyba
241f9fc7b0 not_claim_id/not_claim_ids is not a search parameter 2021-09-10 18:57:20 -03:00
Victor Shyba
270192486a translate grpc errors to RPCError 2021-09-10 18:57:20 -03:00
Victor Shyba
a799503c97 update fields from hub 2021-09-10 18:57:20 -03:00
Victor Shyba
9685928087 there is no first_search 2021-09-10 18:57:20 -03:00
Victor Shyba
0e4b2fad99 specify index name 2021-09-10 18:57:20 -03:00
Victor Shyba
3c4571a4e0 remove fallback 2021-09-10 18:57:20 -03:00
Jeffrey Picard
046147eb1d updates for fields 2021-09-10 18:57:20 -03:00
Jeffrey Picard
7834520e54 update code to be consistent with field renames 2021-09-10 18:57:20 -03:00
Jeffrey Picard
8e5b4d4b6f hardcode port 2021-09-10 18:57:20 -03:00
Jeffrey Picard
4544a074d9 Move the go hub settings from network to ledger config and hook reset
correctly.
2021-09-10 18:57:20 -03:00
Jeffrey Picard
9b78501392 Set default server to the networks default and use go hub by default 2021-09-10 18:57:20 -03:00
Jeffrey Picard
f59ddcc88d Forgot to remove duplicate tests 2021-09-10 18:57:20 -03:00
Jeffrey Picard
a4955a2b79 remove uneeded prints 2021-09-10 18:57:20 -03:00
Jeffrey Picard
92ae1a565b updates protobuf 2021-09-10 18:57:20 -03:00
Jeffrey Picard
15a56ca25e tons of small changes squashed together 2021-09-10 18:57:20 -03:00
Jeffrey Picard
9dcaa829ea update protobufs 2021-09-10 18:57:20 -03:00
Jeffrey Picard
9f65799a3d uncomment tests, add remove_duplicates param
Cleanup prints and commented out code

remove print

don't do list claims

cleanup
2021-09-10 18:57:20 -03:00
Jeffrey Picard
886587848b protobuf changes
more protobuf changes (fix imports)
2021-09-10 18:57:20 -03:00
Jeffrey Picard
a97fc6dba8 cleanup and reorgnazing some stuff
Fixing tests

relabel failing tests properly

run all the tests for the hub

cleanup HubNode
2021-09-10 18:57:20 -03:00
Jeffrey Picard
c124e88d12 grpc client for python 2021-09-10 18:57:20 -03:00
Jeffrey Picard
17f3870296 Add tests for hub
Have the basic starting /stopping / querying. Still don't have the hub
jsonrpc stuff working right and from the looks of it I need to clearify
some of the logic in the claim search function itself because it's not
returning the correct number of claims anyways.

get the integration working with grpcurl

Got tests working, still need to port the rest of them

ported all of the claim search tests

still a few failing due to not having inflation working, and there's something weird
with limit_claims_per_channel that needs to be fixed.
2021-09-10 18:57:20 -03:00
Lex Berezhny
4626d42d08
Merge pull request #3414 from cristi-zz/remove_comment_api
removed `comment` API endoints
2021-09-09 13:07:12 -04:00
Cristian Vicas
e1e760055c Drop comment_* apis.
Refresh documentation.
2021-09-02 11:38:29 +03:00
Cristian Vicas
45bf6c3bf3 Drop comment_* apis.
Refactored dangling functions.
Added unit test.
2021-09-02 11:38:29 +03:00
Cristian Vicas
fef0cc764d Drop comment_* apis
Removed the comment API
Removed tests for the comment API
Removed the documentation section
Removed the comment server configuration
2021-09-02 08:51:00 +03:00
Lex Berezhny
72049afcf6
Merge pull request #3410 from belikor/fix-docstring
jsonrpc_support_sum: remove the + signs from the docstring
2021-09-01 10:05:17 -04:00
belikor
d26c06dbf3 jsonrpc_support_sum: remove the + signs from the docstring
These symbols came from 0a0ac3b7c9 and were probably added
accidentally to the beginning of the line by copying and pasting
some diffs.
2021-08-25 13:28:02 -05:00
Lex Berezhny
268decd655 update readme 2021-08-21 20:01:49 -04:00
Lex Berezhny
7ae246c839 always run on push, otherwise master branch does not get coverage 2021-08-21 17:10:53 -04:00
Lex Berezhny
c7c454e4fb
Merge pull request #3406 from lbryio/coveralls
submit code coverage reports to coveralls
2021-08-21 17:07:22 -04:00
Lex Berezhny
8e27297a81 coveralls flag name fix 2021-08-21 16:47:17 -04:00
Lex Berezhny
2cdec72985 coveralls fix 2021-08-21 16:22:01 -04:00
Lex Berezhny
0085ac534d coverage fix 2021-08-21 15:41:06 -04:00
Lex Berezhny
7828a79a96 coverage combine corrected 2021-08-21 15:31:50 -04:00
Lex Berezhny
5576c21e67 coverage for integration tests 2021-08-21 15:26:14 -04:00
Lex Berezhny
e49cfb1d2b another attempt 2021-08-21 14:44:59 -04:00
Lex Berezhny
1e541d0225 explicit coverralls service 2021-08-21 13:52:33 -04:00
Lex Berezhny
0974afd26d guess coverralls service 2021-08-21 12:06:45 -04:00
Lex Berezhny
8d93594771 coverage on win and mac 2021-08-21 10:51:37 -04:00
Lex Berezhny
1136ac70e8 fix makefile 2021-08-21 10:26:55 -04:00
Lex Berezhny
dc8d5a39ea fix spaces 2021-08-21 09:42:18 -04:00
Lex Berezhny
8329e649b0 try python coveralls package insead of github action 2021-08-21 09:41:16 -04:00
Lex Berezhny
66da8b164f try AndreMiras/coveralls-python-action action 2021-08-21 09:28:20 -04:00
Lex Berezhny
ea48577864 github workflow syntax fix 2021-08-21 09:15:20 -04:00
Lex Berezhny
597146b136 submit coverage to coveralls 2021-08-21 09:04:44 -04:00
Lex Berezhny
30dd0c1e11
Merge pull request #3405 from lbryio/upgrade_pylint
upgrade pylint and fix lint errors
2021-08-20 23:06:44 -04:00
Lex Berezhny
88772c4266 update setup.py 2021-08-20 22:42:12 -04:00
Lex Berezhny
dc1d9e1c84 upgrade pylint and fix lint errors 2021-08-20 22:36:35 -04:00
Lex Berezhny
69ea65835d
Merge pull request #3402 from lbryio/save_files_default_false
changed default setting `save_files` to be false
2021-08-19 10:57:51 -04:00
Lex Berezhny
d5bae3a8c6 manually set save_files=True in unit tests 2021-08-19 09:31:17 -04:00
Lex Berezhny
f14010bd5b explicitly set save_files = True in tests 2021-08-17 16:36:48 -04:00
Lex Berezhny
87094fc83f changed default setting save_files to be false 2021-08-17 15:47:18 -04:00
Lex Berezhny
7c179cfeab missing closing squiggly bracket 2021-08-17 14:48:13 -04:00
Lex Berezhny
7582c221d1 v0.102.0 2021-08-17 14:16:17 -04:00
Lex Berezhny
c109895848
Merge pull request #3399 from lbryio/better-error-logging
Less verbose error logs, only log tracebacks for errors not defined in `lbry.error`
2021-08-17 14:14:27 -04:00
Jack Robison
eccedada40
add TODOs for errors raised that aren't defined in lbry.error 2021-08-17 12:31:03 -04:00
Jack Robison
25d54accf8
return api errors from wallet_add and wallet_create 2021-08-17 12:30:17 -04:00
Jack Robison
d07685f0e9
only log tracebacks for api errors not defined in lbry.error 2021-08-17 11:30:58 -04:00
Jack Robison
2445c00c7e
raise WalletNotLoadedError in get_wallet_or_error instead of ValueError 2021-08-17 11:30:58 -04:00
Lex Berezhny
4c1d3ef514
Merge pull request #3398 from lbryio/clean_blobs_after_delay
clean blobs after waiting interval instead of immediately on startup
2021-08-17 10:44:34 -04:00
Lex Berezhny
4614c7d4c2 clean blobs after waiting interval instead of immediately on startup 2021-08-17 09:52:44 -04:00
Lex Berezhny
bbf1ef0dc3
Merge pull request #3378 from lbryio/disk_management
ability to limit disk spaced used for blobs via `blob_storage_limit` setting (oldest blobs are deleted when disk space limit is reached)
2021-08-16 17:41:05 -04:00
Lex Berezhny
3433c9e708 return number of files deleted 2021-08-16 17:03:40 -04:00
Lex Berezhny
2cd5d75a2e return true/false if clean was performed 2021-08-16 17:02:13 -04:00
Lex Berezhny
2535b8adef fix disk space unit test 2021-08-16 14:54:17 -04:00
Lex Berezhny
4edab7bb7f fix sorting by DirEntry error 2021-08-16 14:41:16 -04:00
Lex Berezhny
fd8658e317 test component unit test 2021-08-16 14:35:32 -04:00
Lex Berezhny
51d21d8c86 working disk cleanup 2021-08-16 14:15:12 -04:00
Lex Berezhny
b4c3307cdf fixed tests 2021-08-13 10:32:46 -04:00
Lex Berezhny
4e8d10cb44 disk space manager and status API 2021-08-13 10:32:46 -04:00
Lex Berezhny
e96875a425 workflow syntax 2021-08-13 10:32:46 -04:00
Lex Berezhny
5ab0035348 run tests on windows and mac 2021-08-13 10:32:46 -04:00
Lex Berezhny
4ddff96b1e
Merge pull request #3395 from lbryio/libtorrent_optional
make libtorrent optional and skip test which depends on it
2021-08-13 10:31:06 -04:00
Lex Berezhny
a08d84c1df make libtorrent optional and skip test which depends on it 2021-08-13 10:07:06 -04:00
Victor Shyba
21c71bfac1 update sync utility 2021-08-09 18:33:47 -03:00
Victor Shyba
6baaed3581 refactor query with new fields 2021-08-09 18:33:47 -03:00
Victor Shyba
152dbfd5d1 reflect fee_currency, fee_amount and duration on repost searches 2021-08-09 18:33:47 -03:00
Victor Shyba
a56d14086b reflect media_type on repost searches 2021-08-09 18:33:47 -03:00
Victor Shyba
aee87693f8 reflect stream_type on repost searches 2021-08-09 18:33:47 -03:00
Alex Grin
976b4affd9
Merge pull request #3383 from lbryio/dht_log 2021-08-09 17:10:01 -04:00
Victor Shyba
e222b6ad9c log that a invalid query happened 2021-08-09 15:07:44 -03:00
Victor Shyba
19b17374e8 throttle instead of disconnecting 2021-08-09 15:07:44 -03:00
Victor Shyba
43989122bb add error type and message to error readme and update code 2021-08-09 15:07:44 -03:00
Victor Shyba
72712d6047 raise and disconnect if too many parameters are used on search 2021-08-09 15:07:44 -03:00
Victor Shyba
0b52d2cc15 log invalid port as a warning instead of an exception 2021-08-03 15:29:52 -03:00
Lex Berezhny
8304102136 move <!channel> out of markdown converter 2021-07-27 11:54:08 -04:00
Lex Berezhny
3381aefcfa notify channel of slack message 2021-07-27 11:39:35 -04:00
Lex Berezhny
279a365cb1 v0.101.1 2021-07-27 11:12:06 -04:00
Lex Berezhny
2c9e00da56 revert version 2021-07-27 11:10:16 -04:00
Lex Berezhny
f7cae69704 switch to using a custom GITHUB_TOKEN for doing releases 2021-07-27 11:07:25 -04:00
Lex Berezhny
b7d58bcdbc v0.101.1 2021-07-26 17:01:25 -04:00
Lex Berezhny
13a856b843 revert version 2021-07-26 17:00:32 -04:00
Lex Berezhny
8da38985c3 debugging for release 2021-07-26 16:58:11 -04:00
Lex Berezhny
60cf6c6b97 v0.101.1 2021-07-26 16:02:28 -04:00
Lex Berezhny
35c2b34564
Merge pull request #3372 from lbryio/release_process_fixup
github release process fixes
2021-07-26 15:58:10 -04:00
Lex Berezhny
ef2e048efc fixes for release process 2021-07-26 15:57:45 -04:00
Lex Berezhny
6b3261aa33
Merge pull request #3373 from lbryio/fix_typo
fix typo in kwargs key
2021-07-26 15:29:29 -04:00
Victor Shyba
1849c02cb6 fix typo in kwargs key 2021-07-26 16:02:48 -03:00
Lex Berezhny
1ec74a89e2
Merge pull request #3367 from belikor/fix-search-claim-id
fix error when using `--claim_id` with `lbrynet claim search`
2021-07-23 10:08:57 -04:00
Victor Shyba
c591792de9 has_source is a special case 2021-07-22 16:25:55 -03:00
Victor Shyba
3108543ae5 3 missing fields 2021-07-22 16:25:55 -03:00
Victor Shyba
1eb221c743 translate reposted, signature_valid and normalized 2021-07-22 16:25:55 -03:00
Alex Grin
bebf6bc2e7 Update constants.py 2021-07-22 16:25:55 -03:00
Alex Grin
9e91cc2138 Update constants.py 2021-07-22 16:25:55 -03:00
Victor Shyba
c5b939cfb7 fix tests 2021-07-22 16:25:55 -03:00
Victor Shyba
5bd411ca27 filtering hash->id 2021-07-22 16:25:55 -03:00
Victor Shyba
a533cda6f0 ES: all _hash to _id 2021-07-22 16:25:55 -03:00
Lex Berezhny
fe4b07b8ae v0.101.0 2021-07-21 12:35:16 -04:00
Lex Berezhny
f9f2ccd904 revert version 2021-07-21 12:28:41 -04:00
Lex Berezhny
d9e87d7c32 publish after uploading release artifacts 2021-07-21 12:27:04 -04:00
Lex Berezhny
a0092c0770 remove docker steps from github action build 2021-07-21 12:14:49 -04:00
Lex Berezhny
3100131125 checkout code in release job 2021-07-21 11:33:41 -04:00
Lex Berezhny
988880cf83 update set_build script to work on github 2021-07-21 11:32:37 -04:00
Lex Berezhny
c3fb9672c4 re-enable skipping failing DHT unit test 2021-07-21 11:25:44 -04:00
Lex Berezhny
0a2d94e425 updated set_build to use GITHUB_ env vars 2021-07-21 09:19:58 -04:00
Lex Berezhny
8d9073cd31 v0.101.0 2021-07-20 22:52:44 -04:00
Lex Berezhny
d075961ffa removed .gitlab-ci.yml 2021-07-20 22:52:05 -04:00
Lex Berezhny
7a72409b61 fix dht node test 2021-07-20 22:43:57 -04:00
Lex Berezhny
34fc530fba cleanup github actions to be able to drop gitlab 2021-07-20 22:43:57 -04:00
Jack Robison
f257ff2f97
Merge pull request #3369 from lbryio/fix-hanging-tx-notification
fix stuck transaction notification due to race in mempool when advancing a block
2021-07-20 18:54:22 -04:00
Jack Robison
7ad5822c5b
fix test 2021-07-20 16:03:34 -04:00
Jack Robison
9a8f9f0a94
fix stuck notification due to mempool/notification race 2021-07-20 15:14:10 -04:00
belikor
6421cecafb daemon: fix --claim_id with lbrynet claim search
For some reason, when using `claim_search`
with `--claim_id`, the arguments dictionary will also
contain `claim_ids` with an empty list, even if we didn't specify it.
```
lbrynet claim search --claim_id=8945573bcfcb7f8276187dfbb93545eac4ebf71a
```

Using both `claim_id` and `claim_ids` will raise a `ValueError`
exception so the daemon won't return a valid result
even if the claim ID is in fact valid.

So if `claim_id` exists, we need to discard `claim_ids`
if it is empty, before proceeding with the rest of the code.

On the other hand, if `claim_ids` is used, and `claim_id` is absent,
there will be no problem as `claim_id` won't be added to the dictionary.
```
lbrynet claim search --claim_ids=8945573bcfcb7f8276187dfbb93545eac4ebf71a
```
2021-07-19 22:24:43 -05:00
Alex Grin
be544d6d89
Merge pull request #3358 from belikor/improve-install-md 2021-07-19 14:20:44 -04:00
Alex Grin
3c89ecafdd
Merge branch 'master' into improve-install-md 2021-07-19 14:20:39 -04:00
Alex Grin
35ec4eec52
Update INSTALL.md 2021-07-19 14:20:15 -04:00
Alex Grin
e47f737a2f
Update INSTALL.md 2021-07-19 14:15:21 -04:00
Alex Grin
ac671a065b
Merge pull request #3356 from lbryio/propagate_external_ip_change 2021-07-19 14:12:26 -04:00
Alex Grin
74116cc550
Merge branch 'master' into propagate_external_ip_change 2021-07-19 14:12:19 -04:00
Alex Grin
406070a5c3
Merge pull request #3354 from belikor/note-download-blob-peer 2021-07-19 14:10:13 -04:00
Victor Shyba
0ccafd5b53 make get_or_create_usable_address respect the generator lock 2021-07-19 14:09:52 -04:00
Alex Grin
940f517aa3
Merge branch 'master' into note-download-blob-peer 2021-07-19 14:09:51 -04:00
Alex Grin
216e5f65ad
Merge pull request #3363 from lbryio/troubleshoot_p2p_script
add script with web endpoints that can troubleshoot p2p/dht
2021-07-19 14:04:28 -04:00
Victor Shyba
a74685d66d add script to troubleshoot p2p/dht 2021-07-19 15:01:37 -03:00
belikor
b7791d2845 exchange_rate_manager: raise exception if 'error' is in json_response
If the error is not handled, the running daemon will continuously
print the following error message:
```
Traceback (most recent call last):
  File "lbry/extras/daemon/exchange_rate_manager.py", line 77, in get_rate
  File "lbry/extras/daemon/exchange_rate_manager.py", line 189, in get_rate_from_response
KeyError: 0
```

This started happening when the UPBit exchange decided to delist
the LBC coin.

Normally `json_response` should be a dictionary, not a list,
so `json_response[0]` causes an error.

By checking for the `'error'` key, we can raise the proper exception.

Once this is done, the message will be a warning, not a traceback.
```
WARNING  lbry.extras.daemon.exchange_rate_manager:92:
Failed to get exchange rate from UPbit: result not found
```
2021-07-19 13:41:49 -04:00
Victor Shyba
d151a82d78 add libtool and automake to the dockerfiles so they can build coincurve 2021-07-15 17:11:19 -03:00
belikor
8ce61fbd52 INSTALL.md: break the big blocks of code, and remove the space
Remove the first space in the block of code as it is not necessary.

This
 ```
 $ python --version
 ```

Becomes this
```
$ python --version
```

Also break the big block of code into individual blocks.
2021-07-11 19:44:54 -05:00
belikor
90c24aade3 INSTALL.md: using Python 3.8 does not work, issue #2769
Because of issue #2769 at the moment the `lbrynet` daemon
will only work correctly with Python 3.7.

The `deadsnakes` personal package archive (PPA) provides
Python 3.7 for Ubuntu distributions that no longer have it
in their official repositories like 18.04 and 20.04.

If Python 3.8+ is used, the daemon will start but the RPC server
may not accept messages, returning the following:
```
Could not connect to daemon. Are you sure it's running?
```
2021-07-11 19:44:54 -05:00
belikor
6b3f787fee INSTALL.md: add more information on the virtual environments
Leave with `deactivate`.

Enter the environment again with
```
source lbry-venv/bin/activate
```

When developing, we can start the server interactively.
```
python lbry/extras/cli.py start
```

Parameters can be passed in the same way.
```
python lbry/extras/cli.py wallet balance
```

If a Python debugger (`pdb` or `ipdb`) is installed we can also start
it in this way, set up break points, and step through the code.
```
ipdb lbry/extras/cli.py
```
2021-07-11 19:44:25 -05:00
belikor
4ebe4ce1b7 scripts: note to further investigate in download_blob_from_peer
Currently `lbrynet blob get <hash>` does not work to download
single blobs which are not already present in the system.
The function locks up and never returns.
It only works for blobs that are in the `blobfiles` directory
already.

This bug is reported in lbryio/lbry-sdk, issue #2070.

Maybe this script can be investigated, and certain parts
can be added to `lbry.extras.daemon.daemon.jsonrpc_blob_get`
in order to solve the previous issue, and finally download
single blobs from the network (peers or reflector servers).
2021-07-09 11:53:35 -05:00
belikor
8c79740ee8 script/test_claim_search: fix the import of ClientSession
This is nothing special, it just allows the module
to run without throwing an error on the import.

From
```
from lbry.wallet.client.basenetwork import ClientSession
```

To
```
from lbry.wallet.network import ClientSession
```
2021-07-09 10:52:41 -04:00
belikor
59d027ca02 script/find_max_server: fix the import of ClientSession
This is nothing special, it just allows the module
to run without throwing an error on the import.

From
```
from lbry.wallet.client.basenetwork import ClientSession
```

To
```
from lbry.wallet.network import ClientSession
```
2021-07-09 10:52:41 -04:00
Ofek Lev
37a7345a90 Upgrade coincurve dependency 2021-07-09 10:51:03 -04:00
Victor Shyba
c519d4651b loop.time is not usable on advance time, use wall time 2021-07-08 03:55:21 -03:00
Victor Shyba
9b3b609e40 re-enable test_losing_connection 2021-07-08 03:46:48 -03:00
Victor Shyba
6254f53716 propagate external ip changes from upnp component to dht node protocol 2021-07-08 03:46:05 -03:00
Jack Robison
f05dc46432
Merge pull request #3342 from lbryio/bug_flush_counter
[resync required] Avoid flush counter overflows on long running hubs by increasing it to 32 bits
2021-07-07 23:45:47 -04:00
Victor Shyba
3de0982a4a limit request error logging to 16k 2021-07-07 18:39:38 -03:00
Victor Shyba
c2184fb3bf run migration on history db open 2021-07-07 18:39:38 -03:00
Victor Shyba
919c09fcb0 add migration 2021-07-07 18:39:38 -03:00
Victor Shyba
1d9dbd40ec increase flush counter to 32 bits 2021-07-07 18:39:38 -03:00
belikor
0cd953a6f3 script/checktrie: fix the import to SQLDB
This is nothing special, it just allows the module
to run without throwing an error.

From
```
from lbry.wallet.server.db import SQLDB
```

To
```
from lbry.wallet.server.db.writer import SQLDB
```
2021-07-07 11:02:28 -03:00
Alex Grin
4db2b72351
Merge pull request #3347 from kodxana/master 2021-07-02 11:16:40 -04:00
kodxana
dd54fcbdbd
Create README.md 2021-07-01 18:21:20 +02:00
kodxana
3123cf7ac6
Added docker-compose 2021-07-01 18:17:36 +02:00
Victor Shyba
6b579dd4ce add dockerfiles for web sdk 2021-06-30 18:03:00 -03:00
Alex Grin
16dfaa3e27
Merge pull request #3343 from lbryio/example_es
add small example script showing how to read and update values to ES as we currently use it
2021-06-30 11:30:06 -04:00
Victor Shyba
d7842b9f84 small script showing how to read/update values to ES as we currently use it 2021-06-25 12:41:05 -03:00
Alex Grin
115034fccb
Merge pull request #3232 from lbryio/timeout 2021-06-25 11:05:25 -04:00
Victor Shyba
309e957a85 add concurrent_hub_requests conf 2021-06-24 21:21:19 -03:00
Victor Shyba
d7007e402e move request semaphore to session and apply to all requests 2021-06-24 21:02:41 -03:00
Victor Shyba
91323a21cf add hub_timeout and propagate it to network code 2021-06-24 21:02:41 -03:00
Lex Berezhny
fea893d76c v0.100.0 2021-06-22 13:33:03 -04:00
Lex Berezhny
761bc6ba4c revert release and fix test 2021-06-22 13:32:41 -04:00
Lex Berezhny
75172feb4e v0.100.0 2021-06-22 12:53:23 -04:00
Lex Berezhny
3285fb1608 revert release 2021-06-22 12:52:48 -04:00
Lex Berezhny
03a4c6910d v0.100.0 2021-06-22 12:51:36 -04:00
Lex Berezhny
485b958599 revert release 2021-06-22 12:50:11 -04:00
Lex Berezhny
da47ba2f67 v0.100.0 2021-06-22 11:11:02 -04:00
Lex Berezhny
c39195488a bug fix 2021-06-22 11:07:58 -04:00
Lex Berezhny
227fb0ae9b network integration test fix 2021-06-22 11:07:58 -04:00
Lex Berezhny
b12ff5b503 test fixes 2021-06-22 11:07:58 -04:00
Lex Berezhny
0946c72b88 lint 2021-06-22 11:07:58 -04:00
Lex Berezhny
7d49b046d4 added support to config for determining if value is set and implemented hub selection logic 2021-06-22 11:07:58 -04:00
Lex Berezhny
5f0426c840 country jurisdiction added to hub UDP protocol 2021-06-22 11:07:58 -04:00
Lex Berezhny
73e239cc5f client side hub discovery pub/sub and hub metadata stored, removed old peers implementation 2021-06-22 11:07:58 -04:00
Lex Berezhny
ad670f721a working client peer hub 2021-06-22 11:07:58 -04:00
Lex Berezhny
028a4a70cf wallet server federation, client portion 2021-06-22 11:07:58 -04:00
Lex Berezhny
77d7960347 increase lbc exchange rate threshold 2021-06-18 11:26:30 -04:00
Lex Berezhny
39821146bd increase lbc threshold in exchange rate integration tests even more 2021-06-17 10:23:33 -04:00
Lex Berezhny
7d505a41ac drop sqlite indexes from test 2021-06-15 18:22:42 -04:00
Lex Berezhny
e457b2f0d6 fix trending to use built-in sqlite instead of apsw 2021-06-15 18:22:42 -04:00
Lex Berezhny
c9cf7fd4d4 drop apsw in wallet.server.db.elasticsearch.sync 2021-06-15 18:22:42 -04:00
Lex Berezhny
b0371dd33d update test reader to use plain sqlite 2021-06-15 18:22:42 -04:00
Lex Berezhny
25e16c3565 dropping apsw 2021-06-15 18:22:42 -04:00
Lex Berezhny
7b39527863 update exchange rate threshold in integration tests due to significant drop in LBC price 2021-06-15 15:58:59 -04:00
Alex Grin
d861b08866
Merge pull request #3323 from lbryio/dht_leak 2021-06-07 16:15:59 -04:00
Victor Shyba
fb438dc108 remove the unregister call 2021-06-04 12:47:16 -03:00
Victor Shyba
4e6b4f179b add items() to LRUCache 2021-06-04 12:20:44 -03:00
Victor Shyba
00d038c8f3 add default parameter to pop on LRUCache 2021-06-04 12:15:47 -03:00
Victor Shyba
a9f6a68952 use LRU caches for DHT metrics 2021-06-04 11:54:37 -03:00
Alex Grin
b9142bbc5a
Merge pull request #3319 from lbryio/support_value_type
drop `value_type` for supports
2021-06-04 08:39:40 -04:00
Victor Shyba
6c812f663e drop value_type for support 2021-06-02 18:01:54 -03:00
Alex Grin
a93ec9783a
Update README.md 2021-06-02 14:10:19 -04:00
Lex Berezhny
2d184d77b6 v0.99.0 2021-06-02 12:07:37 -04:00
Victor Shyba
bce299ccc7 fix docopt typo 2021-06-02 12:05:36 -04:00
Victor Shyba
235cebd14a fix test value 2021-06-02 12:05:36 -04:00
Victor Shyba
a638aa9d53 add and test support for support_create anonymous --comment 2021-06-02 12:05:36 -04:00
Victor Shyba
67cce0ef7e test+implement --comment for support_create 2021-06-02 12:05:36 -04:00
Victor Shyba
82f4267bf6 add comment property/setter to the signable support class 2021-06-02 12:05:36 -04:00
Victor Shyba
45a9ca29c4 update generated support protobuf with field 2021-06-02 12:05:36 -04:00
Victor Shyba
7f4e813277 document schema update process 2021-06-02 12:05:36 -04:00
Lex Berezhny
3805ff4a0c fix purchase test 2021-06-02 11:34:21 -04:00
Lex Berezhny
464cfd475e properly format scripthash address on output 2021-06-02 11:34:21 -04:00
Lex Berezhny
fe469ae57f create appropriate script for scripthash address 2021-06-02 11:34:21 -04:00
Lex Berezhny
550ef9a1c4 allows script addresses (beginning with r) to be used 2021-06-02 11:34:21 -04:00
Alex Grin
935adfb51a
Merge pull request #3301 from lbryio/no_repeat_claim_id
add `--remove_duplicates` to the search api
2021-05-28 11:01:15 -04:00
Victor Shyba
3974df4a62 fix interaction between two modes 2021-05-27 20:14:12 -03:00
Victor Shyba
4870974161 update json api 2021-05-27 20:14:12 -03:00
Victor Shyba
8c4b0037f5 API: add --remove_duplicates to claim_search 2021-05-27 20:14:12 -03:00
Victor Shyba
2c6f763ef2 test picking oldest when originals doesnt match 2021-05-27 20:14:12 -03:00
Victor Shyba
ca28de02d8 test and implementation for remove_duplicates on post-search filtering 2021-05-27 20:14:12 -03:00
Victor Shyba
bfc15ea029 handle limit being 0 and skip reordering if 0/none 2021-05-27 20:14:12 -03:00
Victor Shyba
6e8b8a5920 always call search_ahead 2021-05-27 20:14:12 -03:00
Jack Robison
099f3b6a62
Merge pull request #3308 from lbryio/reflect_more
Don't set stream as reflected until reflector says it doesn't need any blob
2021-05-27 19:13:23 -04:00
Victor Shyba
142d182bc1 if progress was made, retry without a delay 2021-05-27 18:24:58 -03:00
Victor Shyba
1437871d88 fix reflector client: only set completed when server says so 2021-05-27 18:24:58 -03:00
Victor Shyba
352bf69409 improve test 2021-05-27 18:24:58 -03:00
Victor Shyba
9bdf3d23e1 test bug 3296, failing 2021-05-27 18:24:58 -03:00
Victor Shyba
be8ecfa707 sort keys so helper scripts can send blobs using send_request 2021-05-27 18:24:58 -03:00
Lex Berezhny
51da0d0259 v0.98.0 2021-05-26 09:23:19 -04:00
Alex Grin
f55b78a994
Merge pull request #3306 from lbryio/fix-collectionUpdateWithReplace 2021-05-18 15:25:53 -04:00
Alex Grin
e1a44c93f8
Merge branch 'master' into fix-collectionUpdateWithReplace 2021-05-18 15:25:40 -04:00
Alex Grin
07e7087a09
Merge pull request #3303 from keikari/patch-1
Minor fix suggestion for issue #3240
2021-05-18 15:23:56 -04:00
Alex Grin
2c79c7e2f6
Merge branch 'master' into patch-1 2021-05-18 15:23:08 -04:00
Victor Shyba
09f6637fe0 remove unused multiprocessin.Manager 2021-05-17 15:07:32 -03:00
Victor Shyba
3784db3308 test collections update with --replace 2021-05-15 03:27:33 -03:00
zeppi
2b950ff5dd fix bug in collection_update --replace 2021-05-15 03:27:33 -03:00
Alex Grin
09339c9cfb
Merge pull request #3305 from lbryio/fix_migrator_tool
fix hub Elasticsearch sync/migrations tool for when the db exists already
2021-05-14 11:06:07 -04:00
Victor Shyba
ccadd88af5 fix cache call 2021-05-13 22:40:21 -03:00
Victor Shyba
cc02a0efc2 fix es migration bug, expand test case 2021-05-13 19:00:53 -03:00
Victor Shyba
43a1385b79 test sync helper 2021-05-13 19:00:53 -03:00
Victor Shyba
5101464e3b add integration tests command on install.md 2021-05-13 19:00:36 -03:00
Victor Shyba
3d71478d38 update install.md with ES instructions 2021-05-13 19:00:36 -03:00
Victor Shyba
4989ed445e add ES to makefile 2021-05-13 19:00:36 -03:00
keikari
d9413039ec
Fix suggestion for issue #3240
L135: If `getattr()` returns `None`, use `""` instead to avoid error in issue #3240
2021-05-12 18:30:38 +03:00
Jack Robison
eba0c9be34
fix typo 2021-05-07 16:51:19 -04:00
Jack Robison
48c9e9f3cc
Merge pull request #3294 from lbryio/versioned-es-search-index
add versioning to ES search index and automate resync on version bumps
2021-05-07 15:38:21 -04:00
Jack Robison
81ebde88db
resync ES search index on version bumps
-bump ES search index to version 1
2021-05-07 14:36:53 -04:00
Jack Robison
79ced9d0f8
Merge pull request #3262 from lbryio/channel_repost_has_source
Fix bug for `has_source=True` hiding channel reposts
2021-05-07 14:36:04 -04:00
Victor Shyba
a4058b84ce clean out unused sharding 2021-05-07 15:03:37 -03:00
Victor Shyba
7bf211a52b apply reposted_claim_type on es sync 2021-05-07 15:03:37 -03:00
Victor Shyba
d5f722792f fix and test has_source for channel reposts 2021-05-07 15:03:37 -03:00
Victor Shyba
0f02906c9b fix has_source for reposted channels 2021-05-07 15:03:37 -03:00
Victor Shyba
9582e228b1 assert instead of sleep 2021-05-07 15:02:31 -03:00
Victor Shyba
45f20431f9 update tests from the removed feed 2021-05-07 15:02:31 -03:00
Victor Shyba
7554e6d7f9 remove dead code 2021-05-07 15:02:31 -03:00
Victor Shyba
cb8f26f177 remove broken feed 2021-05-07 15:02:31 -03:00
Jack Robison
b5dfce7861
Revert "finished switch from using hash # in URLs to colon :"
This reverts commit 888aa558
2021-05-07 11:31:28 -04:00
Jack Robison
2ca5a65544
Revert "FindShortestID updated"
This reverts commit 8f04a50c
2021-05-07 11:30:31 -04:00
Jack Robison
17deb136db
Revert "StreamCommands"
This reverts commit 2a8ccb06
2021-05-07 11:29:45 -04:00
Jack Robison
8c9710c76c
Merge pull request #3293 from lbryio/fix-block-processor-crash-invalid-fee
fix invalid claim fees breaking the block processor
2021-05-07 10:03:40 -04:00
Jack Robison
32f7ecb261
fix invalid claim fees breaking the block processor 2021-05-06 11:18:58 -04:00
Victor Shyba
fb77fde710 for debug, it is always whole page 2021-05-04 22:22:07 -03:00
Victor Shyba
3c67bb90d7 don't fail when a single one go on maintenance and set completion event regardless of failures 2021-05-04 22:22:07 -03:00
Victor Shyba
dabb168853 dont log full exceptions on simple connection errors 2021-05-04 22:22:07 -03:00
Victor Shyba
45e5b3b219 dont log full pages 2021-05-04 22:22:07 -03:00
Alex Grin
a6b7469923
Update README.md 2021-05-03 18:04:29 -04:00
Jack Robison
cb5dab3033
Merge pull request #3285 from lbryio/restrict-udp-source
Restrict udp sources, add `ALLOW_LAN_UDP` hub setting
2021-04-29 10:19:32 -04:00
Jack Robison
21d0038ff2
add timestamps to hub log 2021-04-28 16:47:00 -04:00
Jack Robison
c094d8f2e8
add ALLOW_LAN_UDP hub setting 2021-04-28 16:47:00 -04:00
Jack Robison
c465d6a6c2
ignore udp packets with low source ports 2021-04-28 16:47:00 -04:00
Lex Berezhny
73d35bc985 v0.97.0 2021-04-28 16:23:46 -04:00
Lex Berezhny
2a8ccb065b StreamCommands 2021-04-28 16:21:01 -04:00
Lex Berezhny
8f04a50ce1 FindShortestID updated 2021-04-28 16:21:01 -04:00
Lex Berezhny
888aa5586b finished switch from using hash # in URLs to colon : 2021-04-28 16:21:01 -04:00
Lex Berezhny
99f56f5d22 v0.96.0 2021-04-28 15:26:58 -04:00
Jack Robison
ad6281090d
Merge pull request #3275 from lbryio/search_caching_issues
add caching to "search ahead" code and invalidate short_url cache on every block
2021-04-28 14:14:29 -04:00
Victor Shyba
f0d334d3e2 refactor from review 2021-04-28 13:28:38 -03:00
Victor Shyba
5f829b048f use separator to avoid cache key conflicts 2021-04-27 22:57:04 -03:00
Victor Shyba
1a961e66ff invalidate short_id cache on new block 2021-04-27 22:57:04 -03:00
Victor Shyba
fdb0e22656 cache search_ahead 2021-04-27 22:57:04 -03:00
Jack Robison
132ee1915f
Merge pull request #3283 from lbryio/fix_multiprocessing_db
fix multiprocessing support on client db
2021-04-27 16:46:28 -04:00
Victor Shyba
44bf4f3c8f fix if statement from always evaluating a string 2021-04-27 17:10:04 -03:00
Alex Grintsvayg
6237767d5a
Merge branch 'make-test'
* make-test:
  run tests using make
2021-04-23 15:26:26 -04:00
Alex Grintsvayg
dec9d96417
run tests using make 2021-04-23 15:25:40 -04:00
Lex Berezhny
b167c87267 v0.95.0 2021-04-23 14:55:38 -04:00
Lex Berezhny
2280fe8e8e default has_source to 1 2021-04-23 14:54:51 -04:00
Lex Berezhny
575d6dcd2d migration specifically for upgrading from client db v1.5 to v1.6 2021-04-23 14:54:51 -04:00
Lex Berezhny
f729490c6b pending claims ordered towards top in claim_list 2021-04-23 11:00:58 -04:00
Lex Berezhny
b32124cdd6 regenerate docs 2021-04-23 10:24:48 -04:00
Lex Berezhny
3d4321ee38 added --has_source/--has_no_source filters to claim_list 2021-04-23 10:24:48 -04:00
Alex Grintsvayg
85034b382e
Revert "run tests using make"
This reverts commit 77a51d1ad4.
2021-04-21 11:41:16 -04:00
Alex Grintsvayg
77a51d1ad4
run tests using make 2021-04-20 15:41:09 -04:00
Alex Grin
33e0cdc2d7
Update docker-compose-wallet-server.yml 2021-04-16 11:58:02 -04:00
Alex Grin
6519faa2fe
Update docker-compose-wallet-server.yml 2021-04-16 11:53:16 -04:00
Lex Berezhny
5e3a234cbe v0.94.1 2021-04-16 11:18:24 -04:00
Lex Berezhny
e54c31d2d5 fix bug in how reserved balance is calculated 2021-04-16 11:17:51 -04:00
Alex Grin
66c0537251
Create ossar-analysis.yml 2021-04-15 15:24:26 -04:00
Alex Grin
ac58516593
Create codeql-analysis.yml 2021-04-15 15:22:57 -04:00
Alex Grin
c3da6322b5
Create SECURITY.md 2021-04-15 15:21:17 -04:00
Lex Berezhny
3d241500cf v0.94.0 2021-04-14 19:55:35 -04:00
Lex Berezhny
ded8224f66 update docs 2021-04-14 19:52:50 -04:00
Lex Berezhny
f8814881a1 ability to set sd_hash, file_name and file_hash when updating a stream claim 2021-04-14 19:52:50 -04:00
Victor Shyba
cc2852cd48 new implementation for limit_claims_per_channel 2021-04-14 18:32:16 -04:00
Lex Berezhny
467637a9eb fix test 2021-04-14 11:24:58 -04:00
Lex Berezhny
3cfc292d84 lint 2021-04-14 11:24:58 -04:00
Lex Berezhny
6acf94a810 moved balance calculation to SQL 2021-04-14 11:24:58 -04:00
Jack Robison
31367fb4c4 show hostnames of spvs 2021-04-13 11:51:27 -04:00
Jack Robison
12d6074e3b fix typing 2021-04-13 11:51:27 -04:00
Lex Berezhny
ff30386051 lint 2021-04-06 21:22:27 -04:00
shubhendra
601f99ac16 Remove unnecessary generator
Signed-off-by: shubhendra <withshubh@gmail.com>
2021-04-06 21:22:27 -04:00
shubhendra
87fe5c6101 Refactor the comparison involving not
Signed-off-by: shubhendra <withshubh@gmail.com>
2021-04-06 21:22:27 -04:00
shubhendra
68399ca31c Iterate dictionary directly
Signed-off-by: shubhendra <withshubh@gmail.com>
2021-04-06 21:22:27 -04:00
shubhendra
2a6d7fd80f Remove methods with unnecessary super delegation.
Signed-off-by: shubhendra <withshubh@gmail.com>
2021-04-06 21:22:27 -04:00
shubhendra
4725f510d8 Remove unnecessary use of comprehension
Signed-off-by: shubhendra <withshubh@gmail.com>
2021-04-06 21:22:27 -04:00
shubhendra
be0ba22222 Remove unnecessary comprehension
Signed-off-by: shubhendra <withshubh@gmail.com>
2021-04-06 21:22:27 -04:00
Lex Berezhny
c8781392be added unit test for Access-Control HTTP headers 2021-04-06 17:12:05 -04:00
John Leith
b97164fcfb adding access control headers 2021-04-06 17:12:05 -04:00
Lex Berezhny
0dfb92281b v0.93.0 2021-03-30 20:59:47 -04:00
Victor Shyba
4fe80c40da also apply to test:json-api 2021-03-30 17:00:15 -04:00
Victor Shyba
f0fac5115a update tox to pass ELASTIC_HOST 2021-03-30 17:00:15 -04:00
Victor Shyba
46dd389d0d add elasticsearch service to gitlab 2021-03-30 17:00:15 -04:00
Jack Robison
1e28e21ab5
Merge pull request #3248 from lbryio/add-es-host-setting
add ELASTIC_HOST and ELASTIC_PORT settings to hub
2021-03-30 13:09:18 -04:00
Jack Robison
7832c62c5d
add ELASTIC_HOST and ELASTIC_PORT settings to hub 2021-03-30 12:48:13 -04:00
Lex Berezhny
d025ee9dbe revert release 2021-03-30 11:29:17 -04:00
Lex Berezhny
a9a9cb4319 v0.93.0 2021-03-30 10:15:31 -04:00
Victor Shyba
aa727cb9b1 show channels regardless of no_source 2021-03-30 09:47:08 -04:00
Victor Shyba
b8c9a99f20 fix no_source for reposts 2021-03-30 09:47:08 -04:00
Lex Berezhny
aff995b0d0 temporary fix for mempool sync failing during reorg 2021-03-29 16:11:03 -04:00
Jack Robison
2cc7e5dfdc
Merge pull request #3153 from lbryio/elasticsearch
hub: use Elasticsearch for `claim_search` and `resolve` calls
2021-03-24 16:44:14 -04:00
Victor Shyba
5235a150b1 add prog name to sync arg parser 2021-03-24 17:07:17 -03:00
Victor Shyba
c6372ea9de hub->lbry-hub 2021-03-24 17:03:57 -03:00
Victor Shyba
7df4cc44c4 fixes from review 2021-03-24 16:30:33 -03:00
Victor Shyba
d47cf40544 add reader.py for test_sqldb tests 2021-03-19 19:58:13 -03:00
Victor Shyba
7f5d88e95c remove dead/broken/unused API 2021-03-19 19:58:13 -03:00
Victor Shyba
d09663c066 remove flush call 2021-03-19 19:58:13 -03:00
Victor Shyba
ef97c9b69f torba-server -> hub 2021-03-19 19:58:13 -03:00
Victor Shyba
d855e6c8b1 move elasticsearch things into its own module 2021-03-19 19:58:13 -03:00
Victor Shyba
cd66f7eb43 if not no_totals, use default page size 2021-03-19 19:58:13 -03:00
Victor Shyba
6a35a7ba4c expand content filtering tests for no_totals 2021-03-19 19:58:13 -03:00
Victor Shyba
a3e146dc68 sort on index time 2021-03-19 19:58:13 -03:00
Victor Shyba
b81305a4a9 index and allow has_source 2021-03-19 19:58:13 -03:00
Victor Shyba
73884b34bc apply no_totals 2021-03-19 19:58:13 -03:00
Victor Shyba
6166a34db2 check cache item before locking 2021-03-19 19:58:13 -03:00
Victor Shyba
6fa7da4b1c less slices 2021-03-19 19:58:13 -03:00
Victor Shyba
c3e426c491 fix search by channel for invalid channel 2021-03-19 19:58:13 -03:00
Victor Shyba
21e023f0db fix search by channel 2021-03-19 19:58:13 -03:00
Victor Shyba
063be001b3 cache inner parsing 2021-03-19 19:58:13 -03:00
Victor Shyba
5dff02e8bc on resolve, get all claims at once 2021-03-19 19:58:13 -03:00
Victor Shyba
60a59407d8 cache the encoded output instead 2021-03-19 19:58:13 -03:00
Victor Shyba
20a5aecfca fix lib exception to asyncio TimeoutError 2021-03-19 19:58:13 -03:00
Victor Shyba
c2e7b5a67d restore some of the interrupt metrics 2021-03-19 19:58:13 -03:00
Victor Shyba
8f32303d07 apply search timeout 2021-03-19 19:58:13 -03:00
Victor Shyba
891b1e7782 track results up to 200 2021-03-19 19:58:13 -03:00
Victor Shyba
f26394fd3b report deletions on docs that doesnt exist, but dont raise 2021-03-19 19:58:13 -03:00
Victor Shyba
4d83d42b4c fix equality instead of mod 2021-03-19 19:58:13 -03:00
Victor Shyba
57f1108df2 fix query being json serializable 2021-03-19 19:58:13 -03:00
Victor Shyba
2641a9abe5 make better resolve cache 2021-03-19 19:58:13 -03:00
Victor Shyba
6b193ab350 make indexing cooperative 2021-03-19 19:58:13 -03:00
Victor Shyba
b1bb37511c use right key on cache 2021-03-19 19:58:13 -03:00
Victor Shyba
319187d6d6 log mempool task exceptions 2021-03-19 19:58:13 -03:00
Victor Shyba
02eb789f84 caching for resolve 2021-03-19 19:58:13 -03:00
Victor Shyba
5a9338a27f use a dict on set_reference 2021-03-19 19:58:13 -03:00
Victor Shyba
eb6924277f round time to 10 minutes and fetch referenced by id 2021-03-19 19:58:13 -03:00
Victor Shyba
325419404d update dockerfile 2021-03-19 19:58:13 -03:00
Victor Shyba
bd8f371fd5 bump referenced rows query limit up 2021-03-19 19:58:13 -03:00
Victor Shyba
1783ff2845 dont delete claims on reorg 2021-03-19 19:58:13 -03:00
Victor Shyba
d388527ffa log indexing errors 2021-03-19 19:58:13 -03:00
Victor Shyba
19494088bd generate from queue 2021-03-19 19:58:13 -03:00
Victor Shyba
920dad524a simplify sync and use asyncio Queue instead 2021-03-19 19:58:13 -03:00
Victor Shyba
ec89bcac8e improve sync script for no-downtime maintenance 2021-03-19 19:58:13 -03:00
Victor Shyba
a916c1f4ad check if db file exists before sync 2021-03-19 19:58:13 -03:00
Victor Shyba
a9a0ac92d7 ignore unset flag 2021-03-19 19:58:13 -03:00
Victor Shyba
da8a8bd1ef filter+fts and tests for edge cases 2021-03-19 19:58:13 -03:00
Victor Shyba
d9c746891d pin python3.7 2021-03-19 19:58:13 -03:00
Victor Shyba
67817005b5 check ES synced without a process and wait for ES 2021-03-19 19:58:13 -03:00
Jack Robison
24d11de5a7 torba-elastic-sync 2021-03-19 19:58:13 -03:00
Victor Shyba
9251c87323 refresh after sync 2021-03-19 19:58:13 -03:00
Victor Shyba
e12fab90d1 docker compose update 2021-03-19 19:58:13 -03:00
Victor Shyba
0a194b5b01 claim_ids query 2021-03-19 19:58:13 -03:00
Victor Shyba
8d028adc53 be a writer by default 2021-03-19 19:58:13 -03:00
Victor Shyba
dfca15395e claim id is also a keyword 2021-03-19 19:58:13 -03:00
Victor Shyba
e21f2362fe apply reorg deletion as well 2021-03-19 19:58:13 -03:00
Victor Shyba
1ce328e8a9 cache signature inspection 2021-03-19 19:58:13 -03:00
Victor Shyba
038a5f999f cache encoded headers 2021-03-19 19:58:13 -03:00
Victor Shyba
5d3704c7ea reader mode 2021-03-19 19:58:13 -03:00
Victor Shyba
87037c06c9 remove reader code 2021-03-19 19:58:13 -03:00
Victor Shyba
dd412c0f50 delete sqlite fts 2021-03-19 19:58:13 -03:00
Victor Shyba
bf44befff6 backport fixes from server 2021-03-19 19:58:13 -03:00
Victor Shyba
e61874bb6f only repeat search if it has blocked items 2021-03-19 19:58:13 -03:00
Victor Shyba
1e5331768f fix some of the tests 2021-03-19 19:58:13 -03:00
Victor Shyba
ec9a3a4f7c do not page filtered 2021-03-19 19:58:13 -03:00
Victor Shyba
e439a3a8dc advanced resolve 2021-03-19 19:58:13 -03:00
Victor Shyba
19f70d7a11 create changelog trigger 2021-03-19 19:58:13 -03:00
Victor Shyba
afe7ed5b05 adjust size 2021-03-19 19:58:13 -03:00
Victor Shyba
d4bf004d74 use a thread pool to sync changes 2021-03-19 19:58:13 -03:00
Victor Shyba
e4d06a088b include the channel being filtered/blocked 2021-03-19 19:58:13 -03:00
Victor Shyba
0929088b12 missing refresh step 2021-03-19 19:58:13 -03:00
Victor Shyba
7b4838fc9b dont update more than 400 items a time 2021-03-19 19:58:13 -03:00
Victor Shyba
0cf9533248 narrow update by query 2021-03-19 19:58:13 -03:00
Victor Shyba
84ff0b8a9f general timeout 2021-03-19 19:58:13 -03:00
Victor Shyba
d467dcfeaf increase sync queue 2021-03-19 19:58:13 -03:00
Victor Shyba
8e68ba4751 fix join, refresh before update 2021-03-19 19:58:13 -03:00
Victor Shyba
0f2a85ba9f simplify sync 2021-03-19 19:58:13 -03:00
Victor Shyba
7674a0a91e backport fixes from testing server 2021-03-19 19:58:13 -03:00
Victor Shyba
5bc1a66572 32 slices and add censor type to fields 2021-03-19 19:58:13 -03:00
Victor Shyba
9b56067213 raise request timeout for content filtering 2021-03-19 19:58:13 -03:00
Victor Shyba
9a9df2fc3c apply filtering only to whats unfiltered 2021-03-19 19:58:13 -03:00
Victor Shyba
9989d8d1d4 refresh after delete 2021-03-19 19:58:13 -03:00
Victor Shyba
f9471f297e apply filter and block from ES script lang 2021-03-19 19:58:13 -03:00
Victor Shyba
146b693e4a exclude title and description 2021-03-19 19:58:13 -03:00
Victor Shyba
7295b7e329 make sync parallel 2021-03-19 19:58:13 -03:00
Victor Shyba
e2441ea3e7 use prefix from ES docs 2021-03-19 19:58:13 -03:00
Victor Shyba
119e51912e fix partial id 2021-03-19 19:58:13 -03:00
Victor Shyba
dd950f5b0d tag can have empty space 2021-03-19 19:58:13 -03:00
Victor Shyba
78a9bad1e1 no indexer_task 2021-03-19 19:58:13 -03:00
Victor Shyba
0c6eaf5484 fix resolve partial id 2021-03-19 19:58:13 -03:00
Victor Shyba
1010068ddb disable refresh interval. start with 3 shards 2021-03-19 19:58:13 -03:00
Victor Shyba
82eec3d8d7 use multiple clients on sync script indexing 2021-03-19 19:58:13 -03:00
Victor Shyba
ee7b37d3f3 also normalize the name supplied by user 2021-03-19 19:58:13 -03:00
Victor Shyba
143d82d242 normalized, not normalized_name 2021-03-19 19:58:13 -03:00
Victor Shyba
8b91b38855 update winners in one go 2021-03-19 19:58:13 -03:00
Victor Shyba
1098f0d2a3 use normalized name instead 2021-03-19 19:58:13 -03:00
Victor Shyba
ab53cec022 fix is_controlling sync 2021-03-19 19:58:13 -03:00
Victor Shyba
6f5f8e5648 add elasticsearch dep 2021-03-19 19:58:13 -03:00
Victor Shyba
edfd707c22 run ES on github actions 2021-03-19 19:58:13 -03:00
Victor Shyba
1870f30af8 add sync script 2021-03-19 19:58:13 -03:00
Victor Shyba
90106f5f08 all test_claim_commands tests green 2021-03-19 19:58:13 -03:00
Victor Shyba
9924b7b438 reposts and tag inheritance 2021-03-19 19:58:13 -03:00
Victor Shyba
aa37faab0a use porter analyzer with weights on full text search 2021-03-19 19:58:13 -03:00
Victor Shyba
dc10f8ce72 ignore errors when deleting 2021-03-19 19:58:13 -03:00
Victor Shyba
996686c1da claim search and resolve translated to ES queries 2021-03-19 19:58:13 -03:00
Victor Shyba
488785d013 add indexer task 2021-03-19 19:58:13 -03:00
Victor Shyba
3abdc01230 index ES during sync 2021-03-19 19:58:13 -03:00
Victor Shyba
8da04a584f start waiting before generate 2021-03-19 18:01:29 -03:00
Victor Shyba
27cc61d45e limit test time to 2 minutes, then consider it a failure and log what was running 2021-03-19 18:01:29 -03:00
Lex Berezhny
7371c30064 v0.92.0 2021-03-15 13:07:30 -04:00
Lex Berezhny
140d163895 removed redundant comment 2021-03-14 10:11:42 -04:00
Victor Shyba
dc33bdc1dc update api json 2021-03-14 10:11:42 -04:00
Victor Shyba
74df4fab83 change column to has_source and document both flags 2021-03-14 10:11:42 -04:00
Victor Shyba
1e5cd3d7a1 typo, fix tests 2021-03-14 10:11:42 -04:00
Victor Shyba
a54e9b64aa add no_source claim_search filter 2021-03-14 10:11:42 -04:00
Victor Shyba
74660704e3 fix update 2021-03-14 10:11:42 -04:00
Victor Shyba
7439893a2a fix get for sourceless claims 2021-03-14 10:11:42 -04:00
Victor Shyba
e27e49e9dc call update only once 2021-03-14 10:11:42 -04:00
Victor Shyba
34ed729c59 there is no 'sd_hash' parameter for this API 2021-03-14 10:11:42 -04:00
Victor Shyba
adaeeca3fd let file_path be optional 2021-03-14 10:11:42 -04:00
Jack Robison
dac75563d3 add --no_file_path param to publish, stream_create, and stream_update 2021-03-14 10:11:42 -04:00
Alex Grintsvayg
cbc76adcaa only return unspent txos if is_spent flag is not used. fixes #2923 2021-03-13 06:44:20 -05:00
Lex Berezhny
69a9cb383d oops 2021-03-12 13:29:55 -05:00
Lex Berezhny
4343073c00 clients can connect to wallet server even when they are not reachable by UDP 2021-03-12 13:29:55 -05:00
Jack Robison
fe60d4be88
Merge pull request #3221 from lbryio/subscribe_hash_on_call
Improve performance of address subscriptions and transaction proofs
2021-03-10 15:58:50 -05:00
Victor Shyba
ae337807f5 get merkles outside thread cooperatively 2021-03-10 13:05:17 -03:00
Victor Shyba
9ae30ac08e during subscribe, hash address only when its time 2021-03-10 12:51:58 -03:00
Lex Berezhny
62fa85c0a4 fix test 2021-03-09 13:27:36 -05:00
Lex Berezhny
7bb873dad9 removed connection_status field from the status command, use wallet.connected instead to determine if SDK is connected 2021-03-09 13:27:36 -05:00
Lex Berezhny
5f6c1c14cb v0.91.0 2021-03-04 00:04:25 -05:00
Lex Berezhny
d43189ad33 regenerate docs 2021-03-04 00:03:16 -05:00
Lex Berezhny
fcad76fc51 lint 2021-03-04 00:03:16 -05:00
Lex Berezhny
97e6e1684e simplifying 2021-03-04 00:03:16 -05:00
zeppi
67a0d3e926 update docs 2021-03-04 00:03:16 -05:00
zeppi
183fb9f9ff provide --resolve tag for collection claim, separate from resolving its contents
bugfix and docs generation

review changes
2021-03-04 00:03:16 -05:00
Lex Berezhny
9815ddef1f fixes stalling client reconnect issue 2021-03-03 23:31:59 -05:00
Lex Berezhny
f6d0847453 v0.90.1 2021-03-01 11:57:40 -05:00
Lex Berezhny
b0b9f0d65f regenerate docs 2021-03-01 11:52:17 -05:00
Lex Berezhny
0cec80f676 fixes transaction signing bug when tx had no change outputs 2021-03-01 11:20:08 -05:00
Patrick Keane
48c64143e3 Add no_totals argument to claim search 2021-02-26 10:41:15 -05:00
Patrick Keane
a8712422bc Update daemon.py 2021-02-21 23:45:18 -05:00
Patrick Keane
97f65bd283 Minor spelling correction(s). 2021-02-21 23:45:18 -05:00
Lex Berezhny
fd3c1c50f1 v0.90.0 2021-02-16 11:13:14 -05:00
Lex Berezhny
b153e4bb9f added support to claim_search for filtering collections via --claim_type 2021-02-16 11:08:54 -05:00
Lex Berezhny
db9856a8db use median exchange rate when several exchange rates are available 2021-02-15 14:09:32 -05:00
Lex Berezhny
75ecea265d ensures content purchase fails appropriately when exchange rate is not available 2021-02-11 13:57:33 -05:00
Lex Berezhny
be8751cb73 integration test fixes 2021-02-10 22:14:13 -05:00
Lex Berezhny
fb25ecb4a1 unittest 2021-02-10 22:14:13 -05:00
Lex Berezhny
f1cb7d27ac lint 2021-02-10 22:14:13 -05:00
Lex Berezhny
dee494e12f converting from USD, BTC to LBC is now done via several exchange rate providers: Bittrex, Cryptonator, CoinEx, hotbit and UPbit 2021-02-10 22:14:13 -05:00
Lex Berezhny
b13a121915 v0.89.0 2021-02-09 10:12:05 -05:00
Lex Berezhny
7486ee9537 do not tx._reset() in generate_channel_private_key 2021-02-08 09:56:03 -05:00
Lex Berezhny
4a20ccc28e tx._reset() the transaction after more internal changes 2021-02-08 09:56:03 -05:00
Lex Berezhny
f80dd2b307 tx._reset() the transaction after internal changes 2021-02-08 09:56:03 -05:00
Lex Berezhny
b208cf6d32 significant performance improvement when creating transactions and with txo spend specifically 2021-02-08 09:56:03 -05:00
Lex Berezhny
39e78ff17e updated exchange rate manager to use v3 bittrex API (old one is deprecated) 2021-02-03 14:15:19 -05:00
Alex Grin
a8177ea7fe
Merge pull request #3171 from lbryio/fix-collectionChannel 2021-02-02 14:16:42 -05:00
Alex Grin
bedcfc154b
Merge branch 'master' into fix-collectionChannel 2021-02-02 11:25:52 -05:00
zeppi
4c38b4aa3c docs and reenable gitlab json generation 2021-02-01 19:19:46 -05:00
Victor Shyba
f6cfe266e0 specify Returns correctly for docs gen 2021-02-01 19:19:46 -05:00
zeppi
4905e65f14 fix comment tests and json generat 2021-02-01 19:19:46 -05:00
Lex Berezhny
ccb250b410 txo_list --resolve now works for signed supports by resolving the signing channel 2021-01-26 20:53:36 -05:00
zeppi
aca57ffc62 allows cli to accept channel_id for collections 2021-01-26 10:02:01 -05:00
Dispatch
7f375f42d8 Fix / Add correct mimetypes for HLS / DASH 2021-01-25 15:55:56 -05:00
Lex Berezhny
eedcc2034d fixes #3166 2021-01-25 10:56:48 -05:00
Jack Robison
24c9a167d7
dockerfile 2021-01-22 12:28:04 -05:00
Jack Robison
909df8ef1f
dockerfile 2021-01-22 11:54:56 -05:00
Jack Robison
3b27cb3671 v0.88.0 2021-01-22 10:41:06 -05:00
Lex Berezhny
3fe0db4a7d fix integration tests 2021-01-22 07:51:13 -05:00
Lex Berezhny
8b55814ab2 fixes #2943 bug where get forced user to purchase their own priced stream 2021-01-22 07:51:13 -05:00
Jack Robison
575e471553
Merge pull request #3161 from lbryio/ledger-compatibility
handle connections from ledger clients in spv server
2021-01-22 00:09:50 -05:00
Jack Robison
0f5f1aebed
accept connections from ledger clients in spv server 2021-01-21 21:37:39 -05:00
Jack Robison
50e17eb1ab
Merge pull request #3148 from lbryio/udp-wallet-server-status
UDP based spv server status, improved server selection
2021-01-21 20:55:16 -05:00
Jack Robison
158cc2f660
skip test_single_server_payment 2021-01-21 19:29:59 -05:00
Jack Robison
1066a31acd
fix test 2021-01-21 18:46:28 -05:00
Jack Robison
1f9d0f4582
pylint 2021-01-21 16:30:54 -05:00
Jack Robison
a6d65233f1
fallback to getting external ip from spv servers instead of internal apis 2021-01-21 16:20:53 -05:00
Jack Robison
eff2fe7a1b
update tests 2021-01-21 16:20:01 -05:00
Jack Robison
20efdc70b3
use UDP ping for wallet server selection
-only connect to one spv server at a time
-remove session pool
2021-01-21 16:15:42 -05:00
Jack Robison
f0d8fb8f1a
add UDP based ping protocol for spv servers 2021-01-21 16:08:33 -05:00
Jack Robison
f7a380e9b7
start prometheus before block processing 2021-01-21 16:01:33 -05:00
Jack Robison
e9c7cf6f63
logging 2021-01-21 14:51:59 -05:00
Jack Robison
68f1661452
add LRUCache with no prometheus metrics 2021-01-21 14:37:08 -05:00
Jack Robison
36fd1b91ae
cache server features 2021-01-21 14:37:08 -05:00
Jack Robison
a4ec430ac0
improve subscription performance 2021-01-21 14:37:08 -05:00
Jack Robison
519614b2fd
skip libtorrent component in tests 2021-01-21 14:37:08 -05:00
Lex Berezhny
bf0118c8ef added support for --not_channel_id to txo_list commands 2021-01-21 14:03:59 -05:00
Jack Robison
a4db0820bc
Merge pull request #3136 from lbryio/fix-reflector-loop
fix reflector loop crashing upon an unexpected error
2021-01-08 12:40:18 -05:00
Jack Robison
ee7528413e
update tests 2021-01-08 11:54:02 -05:00
Jack Robison
7952fc8324
fix reflector loop propagating cancelled error 2021-01-08 09:53:12 -05:00
Jack Robison
652773d2cf
Merge pull request #3129 from lbryio/signing_api
add `channel_sign` api for signing data with a channel identity
2021-01-05 14:13:09 -05:00
Victor Shyba
2a17787242 fix test 2021-01-04 18:04:06 -03:00
Victor Shyba
0a53ad5721 use the comment api signing implementation 2021-01-04 18:04:06 -03:00
Victor Shyba
6da6bdc863 adds jsonrpc_channel_sign signing api 2021-01-04 18:04:06 -03:00
Jack Robison
42ad2bb83f
Merge pull request #3135 from lbryio/loop-metrics
add prometheus metrics for asyncio loop
2021-01-02 11:54:21 -05:00
Jack Robison
f309a65cb4
fix 2021-01-01 15:25:31 -05:00
Jack Robison
77e19ab1a4
prometheus metrics for asyncio loop responsiveness 2021-01-01 15:25:31 -05:00
Jack Robison
1a996b6ef3
Merge pull request #3134 from lbryio/log_checkpoint
log checkpoint finishing so we can gather information on leftover files bug
2021-01-01 15:24:30 -05:00
Victor Shyba
b882f1a010 log checkpoint finishing for gathering information on wal/shm file bug 2020-12-31 16:11:22 -03:00
Jack Robison
82a030e6ff
Merge pull request #3130 from lbryio/block-cache-metrics
skip null args in `channel_ids` given to `claim_search`
2020-12-30 20:57:56 -05:00
Jack Robison
0758b85179
skip null args given to channel_ids in claim search 2020-12-29 21:41:07 -05:00
Jack Robison
ab3d9bd080
block cache metrics 2020-12-29 21:40:39 -05:00
Thomas Zarebczan
8ff813f689
Take 5!
Why you try to pass version param to other installs?
2020-12-26 12:31:51 -05:00
Thomas Zarebczan
88ae72c0d3
Take 3
Why you fail on uninstall?
2020-12-24 20:28:50 -05:00
Thomas Zarebczan
312aa4be26
Fix Windows build take 2 2020-12-24 04:41:21 -05:00
Thomas Zarebczan
cbb06fce9d
Fix windows build, Merry christmas! 2020-12-24 04:13:03 -05:00
Thomas Zarebczan
f259e497c4
Uninstall old python first 2020-12-24 03:48:17 -05:00
Thomas Zarebczan
dd4172ac66
64 bit windows 2020-12-24 03:16:07 -05:00
Jack Robison
66029e60d3 v0.87.0 2020-12-23 21:51:13 -05:00
Jack Robison
364f484f04
Merge pull request #3127 from lbryio/lru-cache-metrics
Add metrics for lru caches
2020-12-23 21:08:01 -05:00
Jack Robison
9dd5159414
increase server cache sizes 2020-12-23 20:47:05 -05:00
Jack Robison
13e38d6fd8
use LRUCache instead of pylru in wallet server 2020-12-23 20:47:05 -05:00
Jack Robison
10dcb64715
lru cache metrics 2020-12-23 20:47:03 -05:00
Jack Robison
7551b51e7d
Merge pull request #3128 from lbryio/sync_smaller_batches
Improve memory usage during wallet sync
2020-12-23 20:45:03 -05:00
Victor Shyba
adb418aafc group cache hits in a single dict 2020-12-23 21:37:20 -03:00
Victor Shyba
270da80d64 return whole batches 2020-12-23 19:10:19 -03:00
Victor Shyba
b2027cfd66 sync in batches of 10, clearing after 2020-12-23 18:30:52 -03:00
Jack Robison
7f1f4eeac6
Merge pull request #3125 from lbryio/fix_order42
fix wallet db integrity / sync bugs
2020-12-22 20:50:46 -05:00
Jack Robison
7a7446c8bd
force resync blockchain.db 2020-12-22 20:08:26 -05:00
Jack Robison
ddbae294e6
skip doc test on gitlab 2020-12-22 16:55:33 -05:00
Jack Robison
8c71b744f3
fix request_synced_transactions edge cases 2020-12-22 15:19:08 -05:00
Jack Robison
479b5d31a9
fix test 2020-12-22 14:06:53 -05:00
Jack Robison
4cbf4230e8
fix txi.txo_ref 2020-12-22 14:05:37 -05:00
Victor Shyba
6a610187e0 cache bypass 2020-12-22 13:01:30 -03:00
Victor Shyba
eb2a4aebba unrestricted and reusing verified cache hits 2020-12-21 16:49:08 -03:00
Victor Shyba
21a2e67755 fix rebase 2020-12-20 21:39:36 -03:00
Victor Shyba
3b9e312615 fix verification 2020-12-20 21:25:26 -03:00
Victor Shyba
26dab04c9e checkpoint 2020-12-20 21:25:26 -03:00
Victor Shyba
00713c0d11 asyncgens 2020-12-20 21:25:26 -03:00
Victor Shyba
751b5f3027 refactor duplicate code 2020-12-20 21:25:26 -03:00
Victor Shyba
e8261b000e wip, see jack 2020-12-20 21:23:58 -03:00
Victor Shyba
41ecb70297 join network can only happen after initial header sync returns 2020-12-20 21:15:11 -03:00
Jack Robison
09ee104b8c
Merge pull request #3120 from lbryio/fix-reorg-crash
Fix off by one in wallet server reorg
2020-12-16 02:36:11 -05:00
Jack Robison
e3a4964787
catch any remaining index errors in fs_tx_hash 2020-12-16 01:28:30 -05:00
Jack Robison
9bf72910a4
fix off by one error in reorg 2020-12-16 01:27:03 -05:00
Jack Robison
ee39e20e6d
logging 2020-12-16 01:26:19 -05:00
Jack Robison
399d6db6f6
Merge branch 'disable-refresh-histogram' 2020-12-15 20:13:20 -05:00
Jack Robison
0821ce44b5
remove unused heavy call 2020-12-15 19:22:12 -05:00
Jack Robison
ea279111c6
Merge pull request #3113 from lbryio/leveldb-performance
Improve performance of fetching transactions and sending address notifications
2020-12-14 16:39:50 -05:00
Jack Robison
674ce02e58
logging 2020-12-14 14:38:36 -05:00
Jack Robison
8dfa2767ec
new_touched 2020-12-14 13:52:26 -05:00
Jack Robison
20dad7f07f
only notify hashxs touched since last notification 2020-12-14 13:42:20 -05:00
Jack Robison
751cc4c44d
don't deserialize mempool in a thread 2020-12-13 13:48:50 -05:00
Jack Robison
2318e6d8e9
faster fs_transactions 2020-12-13 13:48:02 -05:00
Jack Robison
61b4a492c3 v0.86.1 2020-12-08 16:41:46 -05:00
Jack Robison
9db3d01e09
Merge pull request #3105 from lbryio/dont-rerequest-mempool
fix mempool txs being re-requested
2020-12-08 16:00:24 -05:00
Jack Robison
8da73ad3dd
improve hash_to_hex_str performance 2020-12-08 15:39:03 -05:00
Jack Robison
b8c16d8ac5
fix mempool txs being re-requested with each address notification 2020-12-08 15:39:03 -05:00
Jack Robison
429c0951f3 v0.86.0 2020-12-07 21:19:24 -05:00
Jack Robison
74e103c791
version 2020-12-07 21:15:15 -05:00
Jack Robison
f941950ee2
fix comment_create docs 2020-12-07 21:13:43 -05:00
Jack Robison
846df2eef1 v0.86.0 2020-12-07 15:58:05 -05:00
Jack Robison
34ed058c97
Merge pull request #3103 from lbryio/fix-mempool-bloat
fix mempool notification bloat
2020-12-07 15:09:13 -05:00
Jack Robison
eae0290978
fix mempool notification bloat 2020-12-07 13:16:37 -05:00
Jack Robison
561368570e
Merge pull request #3102 from lbryio/faster-notifications
Improve performance of mempool processing loop
2020-12-03 22:43:11 -05:00
Jack Robison
3467d1fed0
fix tests 2020-12-03 22:16:28 -05:00
Jack Robison
d02ff232e5
_single_batch 2020-12-03 20:50:25 -05:00
Jack Robison
2d1c6a5402
fix 2020-12-03 19:15:15 -05:00
Jack Robison
eab3b65629
calculate notifications for all subscriptions instead of per session 2020-12-03 19:15:15 -05:00
Jack Robison
20b435732a
Merge pull request #3099 from lbryio/increase-tx-cache-size
Increase default `tx_cache_size`
2020-12-03 19:14:34 -05:00
Jack Robison
929617273d
resync blockchain.db 2020-12-03 15:56:35 -05:00
Jack Robison
2717bf7d49
increase default tx_cache_size setting 2020-12-03 15:56:35 -05:00
Jack Robison
5cd2ebc960
Merge pull request #3098 from lbryio/cached-txids
store txids in memory, faster address subscription/history
2020-12-02 14:08:07 -05:00
Jack Robison
9b4afe9816
store txids in memory, faster address subscription/history 2020-11-25 16:30:11 -05:00
Jack Robison
23bb5598d5
Merge pull request #3095 from lbryio/fix-transaction-getbatch-order
ensure transactions are returned in the order they're requested
2020-11-24 17:21:12 -05:00
Jack Robison
af1d7813e9
ensure transactions are returned in the order they're requested 2020-11-24 11:52:07 -05:00
Jack Robison
16c2e5a585
Merge pull request #3093 from lbryio/fix-merkle
fix off-by-one when getting block txs for tx_merkle
2020-11-23 19:08:11 -05:00
Jack Robison
c02750edbd
fix tx_merkle 2020-11-23 16:48:24 -05:00
Jack Robison
7204ddafec
Merge pull request #3092 from lbryio/faster-transaction-getbatch
faster `blockchain.transaction.get_batch`
2020-11-23 12:29:14 -05:00
Jack Robison
faeba9a7e4
faster blockchain.transaction.get_batch 2020-11-23 12:05:47 -05:00
Jack Robison
190d238a1f
faster read_headers 2020-11-23 12:03:11 -05:00
Jack Robison
715451b5fb v0.85.0 2020-11-21 00:14:20 -05:00
Jack Robison
87f1895405
Merge pull request #3090 from lbryio/fast-headers
store headers in memory on wallet server
2020-11-20 23:33:18 -05:00
Jack Robison
923d817751
store headers in memory 2020-11-20 22:37:42 -05:00
Jack Robison
0728209b66
Merge pull request #3089 from lbryio/force_checkpoint
Ensure no client .wal files remain on shutdown
2020-11-20 21:12:33 -05:00
Victor Shyba
b8b9dcc2ee WAL_CHECKPOINT on close 2020-11-20 20:16:03 -03:00
Jack Robison
f35e879852
Merge pull request #3088 from lbryio/batched-inflate-outputs
fix flooding with resolve/claim search
2020-11-20 17:43:56 -05:00
Jack Robison
34f4f12eb9
pylint 2020-11-20 16:15:57 -05:00
Jack Robison
fa63bf758d
delete single_call_context, use session pool 2020-11-20 15:52:11 -05:00
Jack Robison
f6b396ae64
Merge branch 'fix-reorg' 2020-11-20 13:32:59 -05:00
Jack Robison
2c7fd58e34
threaded compress headers 2020-11-20 13:09:54 -05:00
Jack Robison
982f2c9634
fix test 2020-11-20 10:57:28 -05:00
Jack Robison
f2fd42b47a
use write_batch 2020-11-19 18:37:49 -05:00
Jack Robison
1b4ccad938
allow settings TRENDING_ALGORITHMS to nothing 2020-11-19 18:36:34 -05:00
Jack Robison
a9de1ce8e0
fix fs_transactions 2020-11-19 12:40:46 -05:00
Jack Robison
ac752d5ec2
Merge pull request #3082 from lbryio/faster-leveldb
store transactions for wallet server in leveldb instead of requesting from lbrycrd
2020-11-17 17:02:11 -05:00
Jack Robison
632d8d02d2
remove dead code 2020-11-17 16:17:24 -05:00
Jack Robison
48aeb26e02
threaded_get_merkle
-remove dead code
2020-11-17 16:05:08 -05:00
Jack Robison
1694af8b5e
get transactions from leveldb instead of lbrycrd rpc 2020-11-17 15:30:48 -05:00
Jack Robison
83bcab9cd2
cleanup 2020-11-17 15:30:48 -05:00
Jack Robison
bdc7f4b3f5
combine tx_count_db and hashes_db, add tx db 2020-11-17 15:30:48 -05:00
Lex Berezhny
39202a3d79 old sequence resolution test 2020-11-17 11:24:29 -05:00
Lex Berezhny
912065a121 remove url sequence feature from code and tests 2020-11-17 11:24:29 -05:00
Thomas Zarebczan
c8466afac2
Remove * from invalid characters
This is already part of other URLs and channels, will be removed from spec as it won't be used for sequencing.
2020-11-16 14:09:04 -05:00
Victor Shyba
2619e162c1 recover invalid state described on #3026 2020-11-09 16:03:38 -05:00
Lex Berezhny
e1112e17f8 v0.84.0 2020-11-08 19:42:33 -05:00
Jack Robison
92b2ead74c
Merge pull request #3078 from lbryio/null-language
support `none` argument to `any_languages` filter in `claim_search`
2020-11-05 22:09:11 -05:00
Jack Robison
bbed9b94c1
null language filtering for claim search 2020-11-05 21:23:49 -05:00
Jack Robison
73d07311db
Merge branch 'brendon' 2020-11-05 21:21:24 -05:00
Brendon J. Brewer
1cdff47477
Rewrite of variable_decay.py for speed improvements 2020-11-05 21:15:55 -05:00
Jack Robison
511a5c3f82
Merge pull request #3058 from lbryio/faster-resolve
faster resolve and claim_search
2020-11-05 21:09:07 -05:00
Jack Robison
853885e2ff
debug 2020-11-05 19:25:34 -05:00
Jack Robison
d83936a66a
fix uncaught error 2020-11-04 22:21:35 -05:00
Jack Robison
5517d2bf56
fix new_sdk_server arg for resolve 2020-11-03 16:23:31 -05:00
Jack Robison
f21ab49ac5
bump aioupnp requirement 2020-11-03 10:39:38 -05:00
Jack Robison
925a458abe
tags 2020-11-03 10:39:08 -05:00
Jack Robison
76946c447f
use single_call_context for claim_search and resolve 2020-11-03 10:39:08 -05:00
Jack Robison
2faa29b1c4
fix dht_monitor script 2020-11-03 10:39:08 -05:00
Lex Berezhny
6826cc311d update test to use new url spec 2020-10-30 11:37:54 -04:00
Lex Berezhny
5e17ce0a0b minor fixup 2020-10-30 11:37:54 -04:00
Kevin Raoofi
e8d299d3b6 Allow : or # for claim_id
This removes the code for trying multiple patterns and the setup for it

Added a few unit tests to check that the parsed URL is as expected
2020-10-30 11:37:54 -04:00
Kevin Raoofi
7637aa2ab6 Added support for spec compliant URL parsing
Legacy URLs are preserved by attempting to parse the new URL format and,
on failing that, it'll attempt the legacy one.

Tests had to be updated such that the correct things are asserted
against each other.
2020-10-30 11:37:54 -04:00
Jack Robison
ab067d1d3a fix unclean closing of blockchain.db 2020-10-26 16:23:28 -04:00
Lex Berezhny
e6f84666c7 added glorious data in dictionary doc for support_sum 2020-10-16 12:03:01 -04:00
Lex Berezhny
4c5429af15 fix lint 2020-10-16 12:03:01 -04:00
Alex Grintsvayg
0a0ac3b7c9 pass-through for new support_sum api 2020-10-16 12:03:01 -04:00
Lex Berezhny
24833ce9fb v0.83.0 2020-10-10 13:03:42 -04:00
Lex Berezhny
ec2c18dc87 regenerate docs 2020-10-10 13:02:38 -04:00
jessop
7384609e74 support comment pinning 2020-10-10 13:02:38 -04:00
Lex Berezhny
3047649650 v0.82.0 2020-09-29 08:49:16 -04:00
kodxana
d298dac3f3 Fixed docker-compose - no snapshot error
Curently new containers do not get provide snapshots from LBRYinc (Wallet and CRD) that causes containers to get into restart loop.
2020-09-29 08:43:49 -04:00
Lex Berezhny
1574bca8a8
Merge pull request #3054 from lbryio/ios-support
iOS support
2020-09-29 08:42:17 -04:00
Lex Berezhny
ec2f6c6b80 lint 2020-09-29 07:52:33 -04:00
Akinwale Ariwodola
838cc60161 conditional import 2020-09-29 07:52:33 -04:00
Akinwale Ariwodola
310c61a5cc check KIVY_BUILD env variable 2020-09-29 07:52:33 -04:00
Lex Berezhny
318cc15323
Merge pull request #3045 from lbryio/bencode-byte-keys
add forward compatibility for byte datagram keys
2020-09-29 07:23:49 -04:00
Jack Robison
3a64ceb4d6
add forward compatibility for byte datagram keys 2020-09-28 15:56:13 -04:00
Lex Berezhny
d0f21c0095
Merge pull request #3050 from lbryio/language-indexes
add indexes for `any_languages` argument to `claim_search`
2020-09-28 15:52:41 -04:00
Jack Robison
46dc15dd29
use language indexes 2020-09-28 14:45:51 -04:00
Jack Robison
8dc654b513
add language table and indexes 2020-09-28 14:45:51 -04:00
Jack Robison
7000ac3f3f
update indexed tags 2020-09-28 14:45:51 -04:00
Lex Berezhny
43c2e8d8e9
Merge pull request #3051 from lbryio/comments-v2
Comments v2 with Reactions
2020-09-28 14:44:54 -04:00
jessop
0231139b01 support anonymous react_list 2020-09-27 16:36:51 -04:00
jessop
d6ee6446dd react list takes comment_ids 2020-09-26 15:21:26 -04:00
Lex Berezhny
7b666efcf8 lbrycrd download path 2020-09-24 20:11:40 -04:00
Lex Berezhny
eb5d2198fc lint 2020-09-24 20:09:46 -04:00
Lex Berezhny
34e44ebd1c minor cleanup and lint fixes 2020-09-24 20:03:22 -04:00
jessop
bf2f4bc040 wip 2020-09-24 17:47:59 -04:00
jessop
9dc4559aba new comment and reactions api 2020-09-23 16:43:28 -04:00
Lex Berezhny
eba8856261 v0.81.0 2020-08-19 15:47:53 -04:00
Lex Berezhny
a8c5aa471a update docs 2020-08-19 15:46:29 -04:00
Lex Berezhny
52f6dcf092
Merge pull request #3021 from lbryio/debug-reflector
Improve reflector upload cancellation handling
2020-08-19 15:44:27 -04:00
Jack Robison
dec79f3742 pylint 2020-08-19 15:24:17 -04:00
Jack Robison
8bdcac0f3e close reader before deleting blob 2020-08-19 15:24:17 -04:00
Jack Robison
8426b674a3 improve reflector upload cancellation handling 2020-08-19 15:24:17 -04:00
Lex Berezhny
da391bcc8d
Merge pull request #3028 from lbryio/limit_claims_per_channel
added `--limit_claims_per_channel` argument to `claim_search` to only return up to the specified number of claims per channel
2020-08-19 15:22:53 -04:00
Lex Berezhny
2d7443d454 slight cleanup 2020-08-19 13:59:26 -04:00
Lex Berezhny
991987ed76 docopt fix 2020-08-19 12:08:41 -04:00
Lex Berezhny
ec24ebf2cf added --limit_claims_per_channel argument to claim_search to only return up to the specified number of claims per channel 2020-08-19 10:51:31 -04:00
Lex Berezhny
6ba0976085
Merge pull request #3022 from btzr-io/patch-2
Add webvtt mime type ( subtitles, captions )
2020-08-12 18:58:07 -04:00
Baltazar Gomez
2b88d01a01
Add webvtt mime type ( subtitles, captions ) 2020-08-12 02:25:01 -05:00
Lex Berezhny
0c09f24cbf v0.80.0 2020-08-04 16:53:52 -04:00
Lex Berezhny
61d22afeba
Merge pull request #3016 from lbryio/release_all_outputs_on_start
release reserved outputs on startup
2020-08-04 16:53:15 -04:00
Lex Berezhny
9f1ed6e8c3 release_all_outputs on startup 2020-08-04 15:03:58 -04:00
Lex Berezhny
bbc4113cac
Merge pull request #3015 from lbryio/new_sdk_forwarding
added support for using the new SDK for `resolve` and `claim_search` by passing `--new_sdk_server`
2020-08-04 15:03:39 -04:00
Lex Berezhny
91194bf422 doc fix 2020-08-04 14:12:40 -04:00
Lex Berezhny
9c5f940b00 claim search forwarding to new sdk 2020-08-04 12:11:02 -04:00
Lex Berezhny
455b4043b8 new resolve 2020-08-04 11:33:39 -04:00
Lex Berezhny
bd83ee7931
Merge pull request #3010 from lbryio/feat-commentFaster
allow faster comment retrieval
2020-07-31 17:32:06 -04:00
jessop
f6bdf7c09a allow faster comment retrieval
switch default for include replies to false
2020-07-31 10:07:12 -04:00
Lex Berezhny
2db8afb8c2 v0.79.1 2020-07-21 21:59:15 -04:00
Lex Berezhny
e033129dd3
Merge pull request #3003 from lbryio/more_timestamp_fix
fix `modified_on` to always cast value to integer, also cast preferences timestamp to int
2020-07-21 21:58:48 -04:00
Lex Berezhny
f9dc590100 update tests 2020-07-21 20:56:50 -04:00
Lex Berezhny
8996aafe0d always cast modified_on to int() 2020-07-21 19:14:57 -04:00
Lex Berezhny
9dc25ef7af v0.79.0 2020-07-20 20:23:51 -04:00
Lex Berezhny
2b84f4d407
Merge pull request #3002 from lbryio/modified_on_int
fixed sync'ing issues related to `modified_on` timestamps having varying floating value between OSes
2020-07-20 20:22:43 -04:00
Lex Berezhny
097c8b674c fix modified_on when modifying account 2020-07-20 14:28:19 -04:00
Lex Berezhny
ba649d4b94 convert modified_on timestamp to int to avoid floating point conflicts when syncing 2020-07-20 14:24:53 -04:00
Lex Berezhny
6ed1614db0
Merge pull request #2997 from lbryio/fix-reflector-lost-connection
Fix uncaught reflector connection errors
2020-07-20 13:48:26 -04:00
Jack Robison
df5b6a8380
feedback 2020-07-20 12:20:00 -04:00
Jack Robison
1f82a8b99e
catch reflector error if a blob is deleted while it's being sent 2020-07-16 12:15:51 -04:00
Jack Robison
0c95d96f32
update pyyaml requirement to 5.3.1 2020-07-16 12:15:51 -04:00
Jack Robison
c2f5f84118
fix uncaught errors from broken reflector connections 2020-07-16 12:15:47 -04:00
Jack Robison
b3b5e3d8f0
test reflector connection breaking mid-transfer 2020-07-15 16:42:55 -04:00
Lex Berezhny
506d3f3cd9 v0.78.0 2020-07-14 23:33:34 -04:00
Lex Berezhny
516a8c5ee5
Merge pull request #2995 from lbryio/batch-sync
Batched sync for wallet transactions
2020-07-14 23:33:13 -04:00
Lex Berezhny
2f81e9d374 pylint 2020-07-14 23:13:14 -04:00
Jack Robison
2d8703bb8d pylint 2020-07-14 23:13:14 -04:00
Jack Robison
76e60d9bc3 logging 2020-07-14 23:13:14 -04:00
Jack Robison
9d5370be5f fix 2020-07-14 23:13:14 -04:00
Jack Robison
fc1a06bc45 fix 2020-07-14 23:13:14 -04:00
Jack Robison
fce80374f4 batched sync 2020-07-14 23:13:14 -04:00
Jack Robison
420c9f10c2 remove _update_cache_item 2020-07-14 23:13:14 -04:00
Jack Robison
5a39681a2e log 2020-07-14 23:13:14 -04:00
Jack Robison
7a1b7db7c8 support multiple blocks with blockchain.transaction.get_batch 2020-07-14 23:13:14 -04:00
Jack Robison
03a643da52 use block cache 2020-07-14 23:13:14 -04:00
Lex Berezhny
383f0ce450
Merge pull request #2986 from Death916/master
check sample rate and lower if too high
2020-07-14 23:01:05 -04:00
Lex Berezhny
c6c668676c changed video_bitrate_maximum from 8.4MB to 5MB 2020-07-14 21:54:42 -04:00
Trent N
7b01dde063 convert sample_rate to integer and dont force a volume_filter
added volume_filter example

deleted _get_volume_filter
2020-07-14 21:49:25 -04:00
Trent
8c25f65024 check sample rate and lower if too high 2020-07-14 21:49:25 -04:00
Lex Berezhny
1e478e3545
Merge pull request #2996 from DispatchCommit/patch-1
updates HLS .m3u8 media type and adds new DASH .mpd media type
2020-07-14 19:47:22 -04:00
Dispatch
6c75a8978b Fix bug in python CICD pipeline
implemented fix recommended by:
https://stackoverflow.com/a/61693590
https://github.com/pypa/setuptools/issues/1963
2020-07-14 19:24:48 -04:00
Lex Berezhny
640267fc2b
Merge pull request #2992 from lbryio/2987_debug
log task errors and add debug information for reflector task
2020-07-14 19:20:15 -04:00
Dispatch
33c7c3ee12
Fixes HLS .m3u8 and DASH .mpd extension
HLS uses a playlist .m3u8 file to provide video content Transport Segments (.ts) files.
DASH uses a playlist .mpd file to provide video content of a few different containers.
2020-07-10 05:39:46 -07:00
Victor Shyba
f9b41d34ae expose reflector task errors, add debug statements for ongoing issue 2020-07-07 00:39:59 -03:00
Lex Berezhny
d4bec79451
Merge pull request #2981 from lbryio/wallet-server-meta-db
Move wallet server tx hashes and headers to leveldb to speed up the rate mempool notifications are sent
2020-07-06 10:50:24 -04:00
Jack Robison
ac1a8b4daf
metric for time to send notifications 2020-07-02 18:57:36 -04:00
Jack Robison
28838c1759
notifications_in_flight_metric 2020-07-02 18:57:36 -04:00
Jack Robison
50ecb0dac9
fix notify 2020-07-02 18:57:36 -04:00
Jack Robison
e22bc01cbd
re-add wakeup event, add address history metric 2020-07-02 18:57:36 -04:00
Jack Robison
6c28713a4c
read history in one loop 2020-07-02 18:57:36 -04:00
Jack Robison
fc9023386c
non-blocking history lookup in notify 2020-07-02 18:57:36 -04:00
Jack Robison
e6cae9bcc3
remove mempool wakeup event, lower refresh delay 2020-07-02 18:57:36 -04:00
Jack Robison
a9eeca1302
mempool processing time metric 2020-07-02 18:57:36 -04:00
Jack Robison
8c695e42ca
fix sqlite coin chooser floor 2020-07-02 18:57:36 -04:00
Jack Robison
0aa7fd47d5
combine loops 2020-07-02 18:57:36 -04:00
Jack Robison
70596042d6
mempool_process_time_metric 2020-07-02 18:57:36 -04:00
Jack Robison
caf616234b
flush databases during sync 2020-07-02 18:57:36 -04:00
Jack Robison
375187aa70
tx hashes db 2020-07-02 18:57:36 -04:00
Jack Robison
71eccdc0e3
hashes path 2020-07-02 18:57:36 -04:00
Jack Robison
639b1e48f5
blocks dir 2020-07-02 18:57:36 -04:00
Jack Robison
0bb4cb4472
tx count db 2020-07-02 18:57:36 -04:00
Jack Robison
cc51543851
headers db 2020-07-02 18:57:36 -04:00
Jack Robison
22540390e1
break the wallet server with chris45 test 2020-07-02 18:57:36 -04:00
Jack Robison
98565eb67c
run read_raw_block in executor 2020-07-02 18:57:36 -04:00
Lex Berezhny
cf6a47ecb7
Merge pull request #2989 from lbryio/blocking-wallet-send
add `--blocking` to `account_send` and `wallet_send`
2020-07-02 18:45:43 -04:00
Jack Robison
fa60b9f9d3
logging 2020-07-02 17:35:38 -04:00
Jack Robison
644120ca31
add --blocking to account_send 2020-07-02 17:32:32 -04:00
Jack Robison
a50a625b3b
add --blocking arg to wallet_send 2020-07-02 17:32:32 -04:00
Lex Berezhny
7297c13331 v0.77.0 2020-06-24 11:32:41 -04:00
Jack Robison
f73399bfac fix test 2020-06-24 11:13:29 -04:00
Jack Robison
a056cd78f7 remove loggly 2020-06-24 11:13:29 -04:00
Lex Berezhny
a30f3c86c2 v0.76.0 2020-06-08 23:11:41 -04:00
Lex Berezhny
e70bdd86a7
Merge pull request #2959 from lbryio/sqlite-coin-chooser
Add all sqlite coin chooser
2020-06-08 18:46:42 -04:00
Jack Robison
bc9f33c2e0
fix test 2020-06-08 18:06:58 -04:00
Jack Robison
872b89ee93
fix mempool conflicts following cancelled api calls that send transactions with the blocking flag 2020-06-05 20:06:42 -04:00
Jack Robison
ae53062518
integration test for sqlite coin chooser 2020-06-05 20:06:42 -04:00
Jack Robison
17f76c9cb3
leave inputs for rejected tx reserved 2020-06-05 20:06:42 -04:00
Jack Robison
5de944146a
logging 2020-06-05 20:06:42 -04:00
Jack Robison
9dc6092cb0
update tests 2020-06-05 20:06:42 -04:00
Jack Robison
a32a2ef04e
add sqlite coin chooser 2020-06-05 20:06:42 -04:00
Lex Berezhny
ecfa0ae3da
Merge pull request #2974 from lbryio/signed_supports
added signed supports
2020-06-05 16:28:15 -04:00
Lex Berezhny
03595052ce fix regular support create tests 2020-06-05 16:08:25 -04:00
Lex Berezhny
1f94c53dd2 added support protobuf 2020-06-05 15:50:00 -04:00
Lex Berezhny
9c426373f2 lint 2020-06-05 15:49:18 -04:00
Lex Berezhny
c03e30a01f added support for signed supports 2020-06-05 15:49:18 -04:00
Lex Berezhny
07f7a77ac0
Merge pull request #2971 from lbryio/faster-publish
Speed up publish response time by not blocking on updating the content claim db
2020-06-05 15:47:40 -04:00
Lex Berezhny
3b9ea2c9a4
Merge pull request #2972 from lbryio/history-cache
Improve wallet server address history cache and the rate of sent notifications
2020-06-05 15:47:07 -04:00
Jack Robison
1beb13dd80
fix attribute error 2020-06-04 09:31:37 -04:00
Jack Robison
ddae84abb3
fix pending_count 2020-06-04 09:31:37 -04:00
Jack Robison
863b9a2c98
don't block returning from publish on updating the content claim table 2020-06-04 09:31:37 -04:00
Jack Robison
9d44bbdb48
don't block the notification loop on sending the notifications 2020-06-04 09:25:41 -04:00
Jack Robison
8d93dd5adc
improve wallet server address history cache 2020-06-04 09:15:21 -04:00
Lex Berezhny
48502961cf v0.75.0 2020-06-03 14:44:02 -04:00
Lex Berezhny
3c8bec61d3
Merge pull request #2966 from lbryio/check-origin
add `allowed_origin` to config, by default no longer allow any requests which pass Origin in header
2020-06-03 14:39:57 -04:00
Lex Berezhny
7296c7df1a Origin: null no longer allowed 2020-06-03 14:19:16 -04:00
Lex Berezhny
f3ee6603de improve allowed_origin request handling 2020-06-03 13:55:20 -04:00
Lex Berezhny
ee0aabda1d backwards compatible allowed_origin, default browsers not allowed 2020-06-03 13:28:32 -04:00
Jack Robison
08d37a4b0f add allowed_origin to config
-raise 403 error if a request doesn't have a matching origin
2020-06-03 12:55:24 -04:00
Jack Robison
f975ea99cb
Merge branch 'endes123321-master' 2020-06-02 15:59:53 -04:00
Jack Robison
f030d41dc7
add test_is_valid_ipv4 2020-06-02 15:56:57 -04:00
Jack Robison
8d079bfcd1
Merge branch 'master' into master 2020-06-02 15:52:04 -04:00
Lex Berezhny
d39b8654a6
Merge pull request #2968 from lbryio/transcode-adjust
more aggressive video transcoding to reduce file sizes
2020-06-01 13:56:33 -04:00
Thomas Zarebczan
ce7816a968 more aggressive video transdoing
Have noticed the defaults aren't aggressive enough to stream smoothly (yet). Downgrade max rate to 5500K, higher crf = smaller file size for now.
2020-06-01 13:11:56 -04:00
Jack Robison
2ee572e68f
Merge pull request #2967 from lbryio/transaction-cache-size
add `transaction_cache_size` to config
2020-05-27 00:57:48 -04:00
Jack Robison
4bbd850898
fix uncaught ValueError in hashX_unsubscribe 2020-05-25 10:25:04 -04:00
Jack Robison
34eae6e608
fix wallet server prometheus bucket sizes 2020-05-25 10:24:31 -04:00
Jack Robison
6a0302fec6
fix uncaught dht DecodeError 2020-05-25 10:23:11 -04:00
Jack Robison
c94cc293c2
fix uncaught errors in test_component_manager 2020-05-25 10:21:36 -04:00
Jack Robison
cae7792a1e
add transaction_cache_size to config 2020-05-25 10:16:18 -04:00
Lex Berezhny
7f6b2fe4f1 v0.74.0 2020-05-18 16:56:58 -04:00
Lex Berezhny
eba430bbc0
Merge pull request #2957 from lbryio/file-list-download-path
added `download_path` as a filter for `file_list` command
2020-05-18 14:15:44 -04:00
Akinwale Ariwodola
01280c8d04 update docstring for download_path 2020-05-18 18:52:13 +01:00
Lex Berezhny
ec141ac9c9
Merge pull request #2960 from lbryio/leaky_ship
fixed bug with leaky information between outputs
2020-05-18 13:21:08 -04:00
Lex Berezhny
590c892a6a re-set channel on txo 2020-05-18 12:27:22 -04:00
Lex Berezhny
ff8a50c366 fixed bug with leaky information between outputs 2020-05-18 11:16:01 -04:00
Lex Berezhny
46ef6c8ab7
Merge pull request #2947 from thebubbleindex/patch-1
properly handle integer environment values, fixes issue with specifying ports via environment vars
2020-05-18 11:11:10 -04:00
Lex Berezhny
b09eabc478 minor simplifcation 2020-05-18 08:53:34 -04:00
thebubbleindex
e49fcea6e3 fix issue with specifying ports via env vars
make sure tcp and udp port for dht are int type
2020-05-18 08:53:07 -04:00
Akinwale Ariwodola
68ed9f4ffc add download_path property to managed_stream 2020-05-17 12:12:31 +01:00
Akinwale Ariwodola
af94687d45 add download_path as a filter field for file_list 2020-05-17 10:32:26 +01:00
Jack Robison
26964ecf0f
fix download_blob_from_peer.py 2020-05-13 09:24:35 -04:00
Jack Robison
77d19af359 v0.73.1 2020-05-13 09:21:48 -04:00
Jack Robison
120b82f243
Merge pull request #2952 from lbryio/newwait
fix case where publish hangs until a new block
2020-05-12 23:10:03 -04:00
Victor Shyba
a0fea30a11
make wait check every second instead of once 2020-05-12 19:44:02 -04:00
Jack Robison
8bb3e0a64d
Merge pull request #2955 from lbryio/fix-set-node
fix node not being set on the downloader in some cases
2020-05-12 18:49:03 -04:00
Jack Robison
bbded12923
fix node not being set on the downloader in some cases 2020-05-12 17:50:20 -04:00
Lex Berezhny
e8ba5d7606 v0.73.0 2020-05-11 19:47:08 -04:00
Jack Robison
af66b31a44
Merge pull request #2954 from lbryio/file-list-reflector-args
add `uploading_to_reflector`,  `is_fully_reflected`, `completed`, and `status` filter arguments to `file_list`, update to match all filters rather than any filters
2020-05-11 19:45:51 -04:00
Jack Robison
b000a40f28
add completed filter arg to file_list 2020-05-11 19:22:53 -04:00
Jack Robison
3c85322523
add status arg to file_list cli 2020-05-11 19:16:08 -04:00
Jack Robison
a469b8bc04
return streams matching all file_list filters rather than those matching any
-fix filter fields when using sets
2020-05-11 19:11:41 -04:00
Jack Robison
78b8261a3a
cancel pending reflector request when connection is lost
-add 180s timeout
2020-05-11 16:08:48 -04:00
Jack Robison
f20ca70c01
add uploading_to_reflector and is_fully_reflected filter arguments to file_list 2020-05-11 15:48:34 -04:00
Jack Robison
4e4148fc1c
Merge pull request #2953 from lbryio/ecdsa-executor
fix channel key generation blocking globally
2020-05-11 15:16:17 -04:00
Jack Robison
c22482f907
channel private key generation in a thread pool 2020-05-11 14:54:31 -04:00
Jack Robison
870e139fce
Merge pull request #2951 from lbryio/add-fixed-peers-conf
add `fixed_peers` setting to config
2020-05-11 14:54:01 -04:00
Jack Robison
4d58648c02
update default fixed peer to cdn.reflector.lbry.com 2020-05-11 14:52:31 -04:00
Jack Robison
ebbb182537
fix test 2020-05-11 14:06:23 -04:00
Jack Robison
1cd5377b45
split fixed peer setting out from reflector_servers 2020-05-11 13:43:13 -04:00
Lex Berezhny
1d1f0527ee
Merge pull request #2948 from lpessin/patch-1
fix duplicate line on api doc
2020-05-07 17:59:57 -04:00
Jack Robison
37a5f77415
Merge pull request #2930 from lbryio/source_manager_ 2020-05-07 16:10:01 -04:00
Luiz
ced3c7efe4
fix duplicate line on api doc
delete line 3616 (duplicate line 3618)
2020-05-07 13:37:46 -03:00
Victor Shyba
c3b8f366ed fixes from review 2020-05-07 04:04:55 -03:00
Victor Shyba
de78876b1a fix test purchase 2020-05-07 04:04:55 -03:00
Victor Shyba
64c25b049c fixup get_filtered from rebase 2020-05-07 04:04:55 -03:00
Victor Shyba
8a4fe4f3ad lbry-libtorrent is now on pypi 2020-05-07 04:04:55 -03:00
Victor Shyba
8811b8c1fd use ubuntu 18 on gitlab, temporarly 2020-05-07 04:04:55 -03:00
Victor Shyba
190b01fdf9 calculate total bytes outside of dict 2020-05-07 04:04:55 -03:00
Victor Shyba
f145d08c10 tell progress, stop trying to read first piece 2020-05-07 04:04:55 -03:00
Victor Shyba
53382b7e15 wait started event 2020-05-07 04:04:55 -03:00
Victor Shyba
6ad0242617 find and show largest file 2020-05-07 04:04:55 -03:00
Victor Shyba
a7c2408c0a fix and test delete with torrents 2020-05-07 04:04:55 -03:00
Victor Shyba
ce1eabaed6 fix moving to a new btih 2020-05-07 04:04:55 -03:00
Victor Shyba
f602541ede fix not knowing a torrent exists 2020-05-07 04:04:55 -03:00
Victor Shyba
3f718e6efc gitlab: use ubuntu on datanetwork tests 2020-05-07 04:04:54 -03:00
Victor Shyba
ce7a985df6 add boost on gitlab, fix failing test, add libtorrent to linux build 2020-05-07 04:04:54 -03:00
Victor Shyba
6d83f7e7bd correct wheel with boost 1.65 2020-05-07 04:04:54 -03:00
Victor Shyba
b73c00943c linting and minor refactor 2020-05-07 04:04:54 -03:00
Victor Shyba
abaac8ef48 fixes from rebase, install libtorrent from s3 2020-05-07 04:04:54 -03:00
Victor Shyba
a2f8e7068e pylint 2020-05-07 04:04:54 -03:00
Victor Shyba
4d47873219 working file list after torrent get 2020-05-07 04:04:54 -03:00
Victor Shyba
cf985486e5 torrent test and misc fixes 2020-05-07 04:04:54 -03:00
Victor Shyba
b930c3fc93 fix torrent and stream manager reference leftovers 2020-05-07 04:04:54 -03:00
Victor Shyba
dd26a96828 adds more torrent parts 2020-05-07 04:04:54 -03:00
Victor Shyba
6865ddfc12 torrent manager and torrent source 2020-05-07 04:04:54 -03:00
Victor Shyba
e888e69d4d fix unit tests 2020-05-07 04:04:54 -03:00
Victor Shyba
2089059792 pylint 2020-05-07 04:04:54 -03:00
Victor Shyba
27739e0364 fix save from resolve 2020-05-07 04:04:54 -03:00
Victor Shyba
698ee271d6 stream manager component becomes file manager component 2020-05-07 04:04:54 -03:00
Victor Shyba
543c75b293 wip 2020-05-07 04:04:54 -03:00
Victor Shyba
b09c46f6f7 add torrent component 2020-05-07 04:04:54 -03:00
Jack Robison
f2cc19e6aa add lbry.torrent 2020-05-07 04:04:54 -03:00
Jack Robison
814a0a123f file manager refactor 2020-05-07 04:04:54 -03:00
Jack Robison
179383540f ManagedDownloadSource and SourceManager refactor 2020-05-07 04:04:54 -03:00
Lex Berezhny
8be1c8310d v0.72.0 2020-05-04 13:53:13 -04:00
Jack Robison
ef02d776ca
Merge pull request #2937 from lbryio/daemon-prometheus
fix database lockup and add prometheus db metrics
2020-05-04 13:51:12 -04:00
Jack Robison
750ff448ad
comments 2020-05-04 13:47:37 -04:00
Jack Robison
e3abab6d4d
pylint 2020-05-04 12:15:48 -04:00
Jack Robison
d3ffae72fb
buckets 2020-05-02 22:30:25 -04:00
Jack Robison
87f751188e
cancelled and failed api request metrics 2020-05-02 21:58:41 -04:00
Jack Robison
3469abaefd
write lock metrics 2020-05-02 21:23:17 -04:00
Jack Robison
797364ee5c
refactor prometheus metrics 2020-05-02 15:01:07 -04:00
Jack Robison
36c05fc4b9
move wallet server prometheus
-only run wallet server metrics for the wallet server
2020-05-01 12:54:35 -04:00
Jack Robison
79624febc0
prevent pileup of writes blocking reads 2020-05-01 12:48:41 -04:00
Jack Robison
0a9d4de126
include write lock in try/finally 2020-05-01 11:40:57 -04:00
Jack Robison
b6f6994db4
Merge pull request #2934 from lbryio/fix-disconnect-on-cancelled-request
don't close the wallet server connection upon a cancelled request
2020-04-30 12:22:46 -04:00
Jack Robison
ff7bed720a
don't close the connection upon a cancelled request 2020-04-29 12:32:31 -04:00
Lex Berezhny
00c0f48b02 v0.71.0 2020-04-27 12:21:04 -04:00
Lex Berezhny
94c45cf2a0
Merge pull request #2922 from lbryio/estimate_on_demand
estimate block timestamps on client only when necessary (header hasn't been downloaded yet)
2020-04-27 12:17:54 -04:00
Victor Shyba
58f77b2a1c load/dump header file using executor 2020-04-27 10:34:49 -04:00
Victor Shyba
7170e69b22 test fixes + leave tx plot always on estimations 2020-04-27 10:34:49 -04:00
Victor Shyba
239ee2437c estimate only whats not downloaded 2020-04-27 10:34:49 -04:00
Victor Shyba
ced368db31 hold headers file in memory during runtime 2020-04-27 10:34:49 -04:00
Lex Berezhny
6a991e5c15
Merge pull request #2914 from StripedMonkey/SM-Config-patch
use home directory if xdg directory not found (Linux)
2020-04-27 10:33:57 -04:00
Lex Berezhny
523d22262b pin pylint to 2.4.4 2020-04-27 10:18:39 -04:00
Noah
08197a327e fix Missing xdg download location
Fixes  an error in detection xdg config locations when XDG_DOWNLOAD_DIR is not present in `user-dirs.dirs`
2020-04-27 10:11:16 -04:00
Jack Robison
b7cb2a7aa5
Merge pull request #2926 from lbryio/blocking-file-io
make file i/o in stream and blob creation and reads non-blocking
2020-04-24 11:07:24 -04:00
Jack Robison
decc5c74ef
don't block when reading a file when creating a stream 2020-04-23 00:35:08 -04:00
Jack Robison
fbe0f886b6
non blocking blob creation 2020-04-23 00:35:08 -04:00
Jack Robison
7e23d6e2ef
Merge pull request #2925 from lbryio/fix-reposts-tag
fix: reposts being returned for single tags
2020-04-23 00:29:04 -04:00
Tom
49458d1085 fix: reposts being returned for single tags 2020-04-22 23:16:30 -04:00
Lex Berezhny
289c12bd9a
Merge pull request #2924 from lbryio/claim_list_is_spent_fix
fix for `claim_list` incorrectly handling `--is_spent` flag
2020-04-22 11:34:16 -04:00
Lex Berezhny
9a6326b027 fix for claim_list incorrectly handling --is_spent flag 2020-04-22 10:36:09 -04:00
Lex Berezhny
51f573f1ea v0.70.0 2020-04-20 13:37:40 -04:00
Jack Robison
c8b7cd8862
Merge pull request #2920 from lbryio/uploading-to-reflector-status
add `uploading_to_reflector` to `file_list` results
2020-04-20 13:35:02 -04:00
Jack Robison
21c112d059
lbrycrd url 2020-04-20 12:16:31 -04:00
Jack Robison
9432e1b5b2
add uploading_to_reflector to file_list results 2020-04-20 11:57:09 -04:00
endes
8269d2f83c better lint style 2020-04-19 20:20:19 +01:00
endes
5a4b6be974 fix 2020-04-19 20:14:21 +01:00
endes123321
35e8ce60a9
Merge branch 'master' into master 2020-04-19 19:54:24 +01:00
endes
8b6dd9f603 better lint style 2020-04-19 19:51:03 +01:00
Lex Berezhny
084f0ebdab v0.69.1 2020-04-17 12:55:07 -04:00
Lex Berezhny
58640c1521
Merge pull request #2917 from lbryio/faster-repost-search-query
improve how claim search query handles reposts
2020-04-17 12:53:50 -04:00
Jack Robison
7ffdfd12f8
faster not tags 2020-04-17 10:47:01 -04:00
Jack Robison
cb9a30f285
faster query 2020-04-17 10:47:01 -04:00
Lex Berezhny
e48bef809f
Merge pull request #2919 from lbryio/exclude_internal_transfers_fix
fix issue with `--exclude_internal_transfers` where it was filtering out sent payments
2020-04-16 20:26:37 -04:00
Lex Berezhny
f5d7570102 fix issue with --exclude_internal_transfers where it was filtering out sent payments 2020-04-16 17:55:49 -04:00
Lex Berezhny
a600c60cf8 v0.69.0 2020-04-13 15:36:27 -04:00
Jack Robison
695eabd026
Merge pull request #2911 from lbryio/file_list_claim_ids
support `claim_id`, `channel_claim_id`, and `outpoint` args given as lists of values to match in `file_list`
2020-04-13 14:58:13 -04:00
Jack Robison
e81b51a647
support claim_id, channel_claim_id, and outpoint args in file_list being lists 2020-04-13 13:19:25 -04:00
Jack Robison
b7e95ff090
Merge pull request #2912 from lbryio/reflector-progress
add `reflector_progress` to `file_list` results
2020-04-13 13:00:59 -04:00
Jack Robison
3ca41be686
add reflector_progress to file_list results 2020-04-13 12:08:22 -04:00
Lex Berezhny
3152046173
Merge pull request #2910 from osilkin98/comment-schema-update
Comment_hide update
2020-04-13 10:34:30 -04:00
Lex Berezhny
3a98dc8a95 run apt-get update before installing 2020-04-13 10:14:52 -04:00
Lex Berezhny
d737b28916 trying actions/checkout v2 2020-04-13 09:57:59 -04:00
Oleg Silkin
97c0dac876 linter 2020-04-07 19:28:26 -04:00
Oleg Silkin
006494b1fa hide_comments now returns lists for both hidden and visible comments 2020-04-07 19:17:27 -04:00
Alex Grintsvayg
149d343201
drop a few unused conf vars 2020-04-07 16:11:23 -04:00
Lex Berezhny
496cc79ba8 v0.68.0 2020-04-06 12:55:21 -04:00
Jack Robison
3591768745
Merge pull request #2908 from lbryio/batched-resolve
Fix connection to wallet server breaking upon giant resolve requests
2020-04-06 12:52:51 -04:00
Jack Robison
d615f6761a
automatically batch large resolve requests 2020-04-06 12:03:33 -04:00
Jack Robison
5b29894048
add reset clients counter to prometheus 2020-04-06 12:03:33 -04:00
Lex Berezhny
d17f0cfa40
Merge pull request #2909 from lbryio/none_for_unconfirmed_time
do not estimate timestamp for unconfirmed txs
2020-04-06 11:41:17 -04:00
Victor Shyba
b2f70c7120 return none for unconfirmed time estimation 2020-04-06 06:03:35 -03:00
Jack Robison
238707bd93
Merge pull request #2897 from lbryio/reorg-claims
handle claim reorgs in the wallet server
2020-04-05 17:48:56 -04:00
Jack Robison
87cdf1e3a0
improve test_reorg_change_claim_height 2020-04-05 16:58:36 -04:00
Jack Robison
57fd47022e
test_reorg_change_claim_height 2020-04-03 13:39:44 -04:00
Jack Robison
5eafd3bf6b
feedback 2020-04-03 13:39:44 -04:00
Jack Robison
640b5b0ea9
delete_claims_above_height with thread lock 2020-04-03 13:39:44 -04:00
Jack Robison
e4fb2f4680
test_reorg_dropping_claim 2020-04-03 13:39:44 -04:00
Jack Robison
a4909f54e4
test reorg count metric 2020-04-03 13:39:44 -04:00
Jack Robison
f7065c6f0c
add reorg count metric to prometheus 2020-04-03 13:39:44 -04:00
Jack Robison
64f7f837e7
delete claims above reorg height from the database 2020-04-03 13:39:44 -04:00
Jack Robison
130d36acd9
Merge pull request #2905 from lbryio/testnet-headers
fix header checkpoints on testnet
2020-04-03 13:38:47 -04:00
Jack Robison
e94c28cfa2
fix header checkpoints on testnet 2020-04-03 12:21:17 -04:00
Jack Robison
a697dec30c
Merge pull request #2902 from lbryio/bump_checkpoints
bump checkpoints
2020-04-03 12:20:38 -04:00
Victor Shyba
bac09e9b9f bump checkpoints 2020-04-03 12:26:52 -03:00
Lex Berezhny
b179f8f86a
Merge pull request #2901 from lbryio/txo_is_spent_filter
replaced `--unspent` with `--is_not_spent` on several commands and added `--is_spent` to many claim/txo `*_list` commands, `support_list` got three new flags `--sent --received --staked`
2020-04-02 22:24:47 -04:00
Lex Berezhny
962d04ae17 fix txo_spend 2020-04-01 21:03:56 -04:00
Lex Berezhny
f28e3bfe37 lint 2020-04-01 20:53:09 -04:00
Lex Berezhny
6474c86d32 cleaned up *_list commands 2020-04-01 20:44:34 -04:00
Lex Berezhny
769ea8cdfe added --is_spent filter to txo list/sum commands 2020-03-31 23:08:51 -04:00
Lex Berezhny
052e77dd5a v0.67.2 2020-03-31 17:47:19 -04:00
Lex Berezhny
1a4b0cb12a
Merge pull request #2900 from lbryio/wallet_status_does_not_fail
`wallet_status` no longer fails if wallet component has not started
2020-03-31 17:45:51 -04:00
Lex Berezhny
f9aa95c987 default to None for all values in wallet_status when wallet_manager not started yet 2020-03-31 17:22:13 -04:00
Lex Berezhny
a5d06fb4a4 wallet_status no longer fails if wallet component has not started 2020-03-31 16:20:13 -04:00
Lex Berezhny
d5e5d90bdc v0.67.1 2020-03-31 12:38:38 -04:00
Lex Berezhny
e62678e4e6
Merge pull request #2898 from lbryio/duration_is_int
`ffmpeg` file analysis returns duration as integer now
2020-03-31 12:06:44 -04:00
Lex Berezhny
558ac24f7e fix test 2020-03-31 11:29:58 -04:00
Lex Berezhny
6f22f6a59f use ceil() on duration float() instead of int() directly 2020-03-31 10:57:37 -04:00
Lex Berezhny
5ec74f8abe ffmpeg file analysis returns duration as integer now 2020-03-31 10:28:04 -04:00
Lex Berezhny
16d7547e03 changed default txo_spend batch_size default to 500 from 1000 2020-03-30 22:48:05 -04:00
Lex Berezhny
767112dcda updated txo_spend docs 2020-03-30 21:45:58 -04:00
Lex Berezhny
81e23d1d8c v0.67.0 2020-03-30 19:43:45 -04:00
Lex Berezhny
ace90a4354
Merge pull request #2890 from lbryio/faster-status
improve speed of `status`
2020-03-30 19:42:57 -04:00
Jack Robison
267e7096cc add test for rechecking ffmpeg installation 2020-03-30 19:11:04 -04:00
Jack Robison
e7cded7511 check ffmpeg/ffmprobe paths in a thread 2020-03-30 19:11:04 -04:00
Jack Robison
33fbd715c0 don't block status on connectivity check 2020-03-30 19:11:04 -04:00
Jack Robison
25ba5b867c dont recheck ffmpeg installation in status 2020-03-30 19:11:04 -04:00
Lex Berezhny
c8f431447c
Merge pull request #2885 from lbryio/repair_tip_on_open
repair headers tip on open and let new headers come in during background fetching
2020-03-30 19:03:09 -04:00
Victor Shyba
1b83a1d09a test and fix verifying from middle 2020-03-30 19:42:27 -03:00
Victor Shyba
d2fb7a7151 lock only when fetching, giving a chance for tip updates 2020-03-30 19:42:27 -03:00
Victor Shyba
7ad3447598 repair tip on open 2020-03-30 19:42:27 -03:00
Lex Berezhny
5e5bc8e705
Merge pull request #2887 from lbryio/txo_spend
added `txo_spend` command to support liquidating large number of txos (eg. tips) by batching them across several transactions
2020-03-30 18:41:05 -04:00
Lex Berezhny
886d1e8a19 added --include_full_tx option to txo_list 2020-03-30 18:15:13 -04:00
Lex Berezhny
6de7a035fa added preview/blocking back into doc string 2020-03-30 17:47:38 -04:00
Lex Berezhny
48d2497eb2 added txo_spend command to support liquidating large number of txos (eg. tips) 2020-03-30 17:47:38 -04:00
Lex Berezhny
6494754ab9
Merge pull request #2896 from lbryio/first_on_ready
fix for startup issues when connecting to wallet servers
2020-03-30 17:41:14 -04:00
Lex Berezhny
a8153627c6 move on_read.first to earlier 2020-03-30 17:02:08 -04:00
Lex Berezhny
ed38966edb add test to verify we listen to on_ready before it actually triggers 2020-03-30 15:26:04 -04:00
Lex Berezhny
85551d1e54 remove no-op test, segwit is always on now 2020-03-30 15:17:25 -04:00
Lex Berezhny
ca31363180 listen for on_read.first before it is triggered 2020-03-30 14:53:52 -04:00
Lex Berezhny
a8f11eb3c3
Merge pull request #2893 from lbryio/fix_float_timestamp
fix estimated timestamp to return integer instead of float
2020-03-30 10:58:32 -04:00
Lex Berezhny
151805121c increase wallet server payment test timeout 2020-03-29 20:00:23 -04:00
Lex Berezhny
0422d2a021 estimated timestamp should be integer 2020-03-29 19:39:37 -04:00
endes123321
8586762dde
fixed is_valid_public_ipv4 2020-03-29 16:12:36 +01:00
Lex Berezhny
ffdf70257b
Merge pull request #2888 from lbryio/m4a-content-type
add m4a content type
2020-03-27 17:54:45 -04:00
Thomas Zarebczan
6079d60aa1
add m4a content type 2020-03-26 23:08:52 -04:00
Jack Robison
d46d012f8c
Revert "delete live bittrex test"
This reverts commit f70343bb
2020-03-26 20:31:39 -04:00
Jack Robison
baf926b360 v0.66.0 2020-03-26 17:59:02 -04:00
Jack Robison
82339869dc
Revert "v0.66.0"
This reverts commit 4ab56ae3
2020-03-26 17:50:44 -04:00
Jack Robison
f70343bb63
delete live bittrex test 2020-03-26 17:50:31 -04:00
Jack Robison
4ab56ae3d1 v0.66.0 2020-03-26 16:49:54 -04:00
Jack Robison
39b4031684
Merge pull request #2880 from lbryio/handle_mkv_files
Handle MKV files
2020-03-26 16:45:20 -04:00
Brannon King
71f8965393
re-use ffprobe info in stream_type
avoid duplicate args


handle review comment
2020-03-26 16:20:12 -04:00
Brannon King
66857e72a4
ensure only webm-type MKV files 2020-03-26 16:20:12 -04:00
Jack Robison
15eb5d47eb
Merge pull request #2881 from lbryio/wallet_sync_status
add `wallet_syncing` to `status`, which is set to true when processing transaction updates
2020-03-26 15:28:20 -04:00
Jack Robison
86242139da
update tests 2020-03-26 13:56:51 -04:00
Victor Shyba
460bdc4148
move wallet_syncing to wallet status is_syncing 2020-03-26 11:43:48 -04:00
Victor Shyba
1052126522
add wallet_syncing status 2020-03-26 11:43:48 -04:00
Jack Robison
fdb42ac876
Merge pull request #2852 from lbryio/faster-transaction-sync
More efficient syncing of wallet transactions
2020-03-26 11:43:20 -04:00
Jack Robison
6486f986e8
bump min required server version 2020-03-26 11:41:49 -04:00
Jack Robison
bc4075e2ed
create_task instead of ensure_future 2020-03-26 11:41:49 -04:00
Jack Robison
11525c7d0d
fix caching transaction_show 2020-03-26 11:41:49 -04:00
Jack Robison
bb12ae9ce6
fix duplicate get_merkle requests 2020-03-26 11:41:49 -04:00
Jack Robison
66b4ad0c44
improve log 2020-03-26 11:41:49 -04:00
Jack Robison
4bff2d718e
clean up test 2020-03-26 11:41:49 -04:00
Jack Robison
af7a7b4dc6
fix maybe_migrate_certificates blocking the api while populating channel signing keys 2020-03-26 11:41:49 -04:00
Jack Robison
ec20d9a2a8
faster wallet sync with get_transaction_and_merkle 2020-03-26 11:41:49 -04:00
Lex Berezhny
b0aee3d335
Merge pull request #2884 from lbryio/txo_plot
added `txo_plot` command to allow plotting txo sums over time
2020-03-26 10:14:02 -04:00
Lex Berezhny
5e2ddbfd86 txo_plot returns lbc instead of dewies 2020-03-26 01:13:09 -04:00
Lex Berezhny
f086ebbb8e lint 2020-03-26 00:42:11 -04:00
Lex Berezhny
e5bf6a5bfc added txo_plot command to allow plotting txo sums over time 2020-03-26 00:37:13 -04:00
Lex Berezhny
76fa86d54b v0.65.0 2020-03-23 14:56:45 -04:00
Jack Robison
60d89506a5
Merge pull request #2877 from lbryio/on_ready_fixup
wait for tx sync to finish but not header sync
2020-03-23 13:46:47 -04:00
Victor Shyba
2893f1eb9e add other taskgroup for non-start-important tasks 2020-03-23 12:36:05 -03:00
Victor Shyba
a08cbf412d on_ready.wait so we start after tx syncs 2020-03-23 12:26:07 -03:00
Lex Berezhny
0bd65356f9 increase timeout in usage payment test 2020-03-23 11:06:29 -04:00
Lex Berezhny
2e2b39455f
Merge pull request #2875 from lbryio/1080p
default to 1080P for anything higher than 1080P
2020-03-23 10:44:18 -04:00
Lex Berezhny
1f9fbe34e4
Merge pull request #2863 from lbryio/async_headers
fetch headers on demand
2020-03-23 10:41:31 -04:00
Thomas Zarebczan
506d7ae50e
default to 1080P for anything higher than 1080P
Previously would do 1440p
2020-03-23 10:20:13 -04:00
Victor Shyba
342cb00625 less concurrent repeated header checks 2020-03-23 01:19:34 -03:00
Victor Shyba
952fc01efd add script that generates checkpoints 2020-03-23 00:05:36 -03:00
Victor Shyba
3eebe301fe move checkpoints out of folder into file 2020-03-22 23:45:14 -03:00
Victor Shyba
19c0a81c42 fix bad usages of hash and some tests 2020-03-22 23:31:47 -03:00
Victor Shyba
9fc7f9904b fix tests, delete old code 2020-03-22 23:31:47 -03:00
Victor Shyba
b04a516063 better locking, stop corrupting headers, fix some tests 2020-03-22 23:31:47 -03:00
Victor Shyba
241e946d91 first experimental version 2020-03-22 23:31:45 -03:00
Victor Shyba
e45375dc26 more async parts 2020-03-22 23:30:11 -03:00
Victor Shyba
ec8e243323 estimate timestamps instead of using block headers 2020-03-22 23:30:11 -03:00
Victor Shyba
af0e9368d4 headers get now async 2020-03-22 23:30:11 -03:00
Lex Berezhny
285483b81a
Merge pull request #2872 from lbryio/claim_list_include_received_tips
added `--include_received_tips` to `claim_list` and `txo_list` commands
2020-03-22 18:00:35 -04:00
Lex Berezhny
73a91d5569 added --include_received_tips to claim_list/txo_list 2020-03-22 17:22:15 -04:00
Lex Berezhny
e24c78be32
Merge pull request #2871 from lbryio/wallet_txo_data_leak
fix to prevent transaction cache from leaking some information about outputs between unrelated wallets
2020-03-22 13:16:03 -04:00
Lex Berezhny
b11184de68 fix purchase_create 2020-03-22 12:23:21 -04:00
Lex Berezhny
61d02fc5d7 fix to prevent transaction cache from leaking some information about outputs between unrelated wallets 2020-03-22 11:45:40 -04:00
Lex Berezhny
f0217f6821
Merge pull request #2870 from lbryio/resolve_includes
added new flags to `resolve` (and a few to `claim_search`) commands: `--include_purchase_receipt`, `--include_is_my_output`, `--include_sent_supports`, `--include_sent_tips` and `--include_received_tips`
2020-03-22 02:08:28 -04:00
Lex Berezhny
09644914a6 added --include_is_my_output to claim_search 2020-03-22 01:51:09 -04:00
Lex Berezhny
b5c24d6a48 fixing unit tests 2020-03-22 01:29:26 -04:00
Lex Berezhny
bdd2ac2c25 renamed some flags 2020-03-22 01:13:26 -04:00
Lex Berezhny
76376f0d33 lint 2020-03-22 00:24:38 -04:00
Lex Berezhny
9749da46ae added flags to resolve: --include_purchase_receipt, --include_is_my_output, --include_my_supports, --include_my_tips 2020-03-21 23:44:57 -04:00
Lex Berezhny
1c05295e89 updated doc 2020-03-21 20:15:21 -04:00
Lex Berezhny
4d0f28215a added release-text-lines: to release script to support multi-line release notes 2020-03-21 20:09:39 -04:00
Lex Berezhny
0bba72bc5a
Merge pull request #2862 from lbryio/txo_list_support_sent_outputs
`txo_list` adds many new ownership filters such as `--is_my_input`, `--is_my_output`, etc and some metadata filters such as `--channel_id`, `--reposted_claim_id`, etc and a new command `txo_sum` which takes the same arguments as `txo_list` and produces sum of outputs
2020-03-21 19:48:34 -04:00
Lex Berezhny
87089b8e83 fix support test 2020-03-21 18:48:06 -04:00
Lex Berezhny
6a58148a89 added support for --order_by=none 2020-03-21 18:16:25 -04:00
Lex Berezhny
15091052be added --no_totals to txo_list 2020-03-21 18:06:05 -04:00
Lex Berezhny
5cd7e9a9b8 increment scema version and force specific indexes to be used for get_txos() 2020-03-21 15:08:14 -04:00
Lex Berezhny
7cb530c334 added --channel_id and --order_by to txo_list 2020-03-20 23:19:26 -04:00
Lex Berezhny
5e0324cc91 added --reposted_claim_id to txo_list 2020-03-20 20:22:57 -04:00
Lex Berezhny
6293e227ea added txo_sum command 2020-03-20 19:08:47 -04:00
Lex Berezhny
d6d83a5c76 integration test fix 2020-03-20 19:08:47 -04:00
Lex Berezhny
93fc883b90 fixing unit tests 2020-03-20 19:08:47 -04:00
Lex Berezhny
dd21803598 working --is_my_input_or_output --is_my_input --is_my_output etc 2020-03-20 19:08:47 -04:00
Lex Berezhny
af2f2282c2 txo_list returns txo funded by my account but sent to external address 2020-03-20 19:08:47 -04:00
Jack Robison
869a76c9bb
Merge pull request #2851 from lbryio/transaction-info
Add `blockchain.transaction.info` and `blockchain.transaction.get_batch` to the wallet server for more efficient transaction fetching
2020-03-20 10:25:45 -04:00
Jack Robison
44b2964a6a
add blockchain.transaction.get_batch 2020-03-19 19:20:30 -04:00
Jack Robison
8fddb57e0a
reduce block lru cache sizes 2020-03-19 19:20:30 -04:00
Jack Robison
c7a5a0cab0
add blockchain.transaction.info 2020-03-19 19:20:30 -04:00
Jack Robison
c13aab3ffc
Merge pull request #2823 from lbryio/multiple-db-readers
Use multiple processes for querying the db for api calls
2020-03-19 19:19:53 -04:00
Jack Robison
95a9a76598
windows 2020-03-19 19:19:02 -04:00
Jack Robison
923cfa3d50
another 2020-03-19 19:19:02 -04:00
Jack Robison
2c489168c2
feedback 2020-03-19 19:19:02 -04:00
Jack Robison
f3292b4d34
feedback 2020-03-19 19:19:02 -04:00
Jack Robison
9769829b72
fix android executor 2020-03-19 19:19:02 -04:00
Jack Robison
36243d15cc
fix peer_list 2020-03-19 19:19:02 -04:00
Jack Robison
f4645f570c
more read only calls 2020-03-19 19:19:02 -04:00
Jack Robison
61603ccfce
write lock 2020-03-19 19:19:02 -04:00
Jack Robison
a26cfc639c
WAL 2020-03-19 19:19:02 -04:00
Jack Robison
90602931d8
multiple readers for transaction_list 2020-03-19 19:19:02 -04:00
Jack Robison
d1b330028c
multiple db reader processeses 2020-03-19 19:19:02 -04:00
Jack Robison
7a6b1930bf
rename executor -> writer_executor 2020-03-19 19:19:02 -04:00
Lex Berezhny
c271361552
Merge pull request #2865 from lbryio/lbrycrd-1744
upgrade lbrycrd to v0.17.4.4
2020-03-19 17:54:26 -04:00
Lex Berezhny
17789bc814
Merge pull request #2866 from lbryio/dont_shlex_windows
fixed issue with shell parsing on Windows
2020-03-19 17:53:47 -04:00
Brannon King
d7c16e161a fixed issue with shell parsing on Windows 2020-03-19 13:11:57 -06:00
Lex Berezhny
e71c17d7e7 upgrade lbrycrd to v0.17.4.4 2020-03-19 12:54:19 -04:00
Lex Berezhny
17d87eb157
Merge pull request #2861 from lbryio/ffmpeg_fixes
transcode in stream_update, re-enable transcode on Windows
2020-03-19 12:20:47 -04:00
Brannon King
bf11bcc084 fixed transcoding tests for non-async execute
status ordering broke path check
2020-03-18 12:43:27 -06:00
Brannon King
bd291109df addressing code review comments 2020-03-18 12:00:52 -06:00
Brannon King
ac89ba9b8d don't require ProactorEventLoop on Windows
fix linter errors
2020-03-18 12:00:52 -06:00
Brannon King
5ab634e375 support search path for ffmpeg 2020-03-18 12:00:52 -06:00
Brannon King
bb1978d976 ffmpeg now invoked via stream update 2020-03-18 12:00:52 -06:00
Lex Berezhny
bea94ce8ac
Merge pull request #2854 from lbryio/tracemalloc_api
Adds tracemalloc api for memory troubleshooting
2020-03-16 22:15:41 -04:00
Victor Shyba
9561f93594 disable tracemalloc after test finishes so it doesnt slow down everything else 2020-03-16 11:29:16 -03:00
Victor Shyba
d44d5c3304 enable/disable instead of set 2020-03-16 06:40:22 -03:00
Victor Shyba
ec541e2057 lint and improve tests 2020-03-11 19:56:37 -03:00
Victor Shyba
8169bf6b97 top objects api 2020-03-11 19:56:37 -03:00
Victor Shyba
56c8ad1221 start/stop tracemalloc over api 2020-03-11 19:56:37 -03:00
Lex Berezhny
6814f2e38c
Merge pull request #2857 from eggplantbren/master
Trending algorithm with time delay and variable decay rate
2020-03-11 18:34:52 -04:00
Brendon J. Brewer
5f043b9a78 variable decay 2020-03-12 10:33:15 +13:00
Lex Berezhny
c9092cd1c7 v0.64.0 2020-03-09 13:32:19 -04:00
Lex Berezhny
86cc65d894
Merge pull request #2848 from lbryio/claim_transaction_list_filtering
added new `txo_list` command with filtering for `--claim_id`, claim `--name` and `--is_received`/`--is_not_received`, also commands `claim_list`/`stream_list`/`channel_list`/`support_list` are based on `txo_list` and thus support most of the new filters
2020-03-08 23:33:38 -04:00
Lex Berezhny
8ef2647fa9 is_received 2020-03-08 23:11:03 -04:00
Lex Berezhny
3ff9e99416 added txo_list command 2020-03-08 16:22:49 -04:00
Lex Berezhny
1731046011
Merge pull request #2846 from lbryio/txo_is_spent
added `is_spent` attribute to transaction outputs
2020-03-06 21:10:30 -05:00
Lex Berezhny
c3d96184b6 fix test 2020-03-06 20:35:26 -05:00
Lex Berezhny
c255c606a7 added is_spent attribute to transaction outputs 2020-03-06 20:12:38 -05:00
Jack Robison
cc69faa1fd v0.63.0 2020-03-06 15:58:34 -05:00
Jack Robison
0f53cd86c8
Merge branch 'debug_find_ffmpeg' 2020-03-06 15:54:24 -05:00
Brannon King
212c8f188d
only change LD vars if running from bundle
missed a word


moved is_running_from_bundle
2020-03-06 15:53:09 -05:00
Brannon King
dec248adec
repair env modified by pyinstaller
see https://pyinstaller.readthedocs.io/en/stable/runtime-information.html#ld-library-path-libpath-considerations
2020-03-06 15:53:09 -05:00
Brannon King
98e6a066f4
add unit test and debug output 2020-03-06 15:53:09 -05:00
Jack Robison
c90db54a3d
Revert "v0.63.0"
This reverts commit 927d1569
2020-03-06 15:51:42 -05:00
Jack Robison
927d156933 v0.63.0 2020-03-06 14:49:53 -05:00
Jack Robison
c71f91073f
Merge pull request #2845 from lbryio/temp-disable-windows-ffmpeg
fix DHT setup and disable ffmpeg on windows temporarily
2020-03-06 14:20:18 -05:00
Jack Robison
ab28387692
remove ProactorEventLoop and disable ffmpeg on windows temporarily 2020-03-06 13:45:34 -05:00
Lex Berezhny
c4905d02b9
Merge pull request #2793 from lbryio/dont_validate_non_video
don't validate and repair files that aren't videos; add maximum video bit rate setting
2020-03-04 09:16:37 -05:00
Brannon King
ee39880fb5 fix items from review 2020-03-03 21:38:30 -07:00
Brannon King
926b3e56b9 take a hint, lint 2020-03-03 21:38:30 -07:00
Brannon King
e060df5367 hide ValueError 2020-03-03 21:38:30 -07:00
Brannon King
a3294d4a0d make bit_rate check support maximum 2020-03-03 21:38:30 -07:00
Brannon King
19ce0ab246 ogg -> ogv, ignore files that aren't video 2020-03-03 21:38:30 -07:00
Lex Berezhny
75a1cc0d33
Merge pull request #2840 from lbryio/status_returns_server_features
`status` command returns features available of currently connected wallet server, including trending algorithm
2020-03-03 20:47:04 -05:00
Lex Berezhny
05228529b0 fix tests 2020-03-03 20:26:07 -05:00
Lex Berezhny
3cbeadfbc3 lbrynet status command returns features available of currenty connected wallet server, including thetrending algorithm used 2020-03-03 20:04:57 -05:00
Lex Berezhny
3d4938c0e2
Merge pull request #2839 from lbryio/file_analysis_join_path
use `os.path.join()` instead of string addition when searching for file analysis binaries
2020-03-03 10:48:06 -05:00
Lex Berezhny
1b850b8a2b dont prepend path when doing which ffmpeg 2020-03-02 21:11:11 -05:00
Lex Berezhny
ebd33f1869 use os.path.join instead of string addition when searching for file analysis binaries 2020-03-02 20:34:54 -05:00
Jack Robison
ac846b4df3 v0.62.0 2020-03-01 20:29:32 -05:00
Jack Robison
0d427c9b90
Merge pull request #2836 from lbryio/wallet-server-lru-caches
Add LRU caches for blocks and block hashes in the wallet server
2020-03-01 15:19:28 -05:00
Jack Robison
dc6194f862
perf counter 2020-03-01 15:17:21 -05:00
Jack Robison
de50214e1f
block lru cache 2020-03-01 15:17:21 -05:00
Jack Robison
5ec4a88c35
block hash lru cache 2020-03-01 15:17:05 -05:00
Jack Robison
31c141e757
Merge pull request #2835 from lbryio/wallet-server-threadpools
Improve wallet server performance with separate thread pools for leveldb, the block processor, and mempool
2020-03-01 15:15:20 -05:00
Jack Robison
9d5760d899
fix test 2020-03-01 14:41:14 -05:00
Jack Robison
7945e1ea3c
mempool threadpool 2020-03-01 12:40:13 -05:00
Jack Robison
d94c40e371
leveldb threadpool 2020-03-01 12:40:13 -05:00
Jack Robison
31f22122e8
block processor threadpool 2020-03-01 12:40:13 -05:00
Lex Berezhny
506582aa2b
Merge pull request #2834 from lbryio/list_resolve_includes_local_data
when using `--resolve` with local claim `*_list` commands, update resolved result with local metadata (such as `is_mine`)
2020-02-29 15:48:49 -05:00
Lex Berezhny
a02b251c9b when using --resolve with local claim list commands, update resolved result with local metadata (such as is_mine and is_mine) 2020-02-29 09:42:11 -05:00
Lex Berezhny
34cbb6fa79
Merge pull request #2831 from lbryio/claim_type_index_improvement
update `claim_type_*` db indexes to be more performant
2020-02-27 14:27:24 -05:00
Lex Berezhny
33679b56fd update claim_type_* indexes to be more peformant 2020-02-27 12:07:38 -05:00
Lex Berezhny
8897c4d560
Merge pull request #2816 from osilkin98/removes-anonymous
removes ability to comment anonymously
2020-02-27 10:37:47 -05:00
Oleg Silkin
f0c18ccbe7 Removes testing for anonymous comments, fixes up existing asserts 2020-02-26 19:59:38 -05:00
Oleg Silkin
e560d83c51 drops support for commenting without a channel 2020-02-26 19:58:03 -05:00
Lex Berezhny
eca7addc67 v0.61.0 2020-02-24 10:01:03 -05:00
Lex Berezhny
263f3ba5c9
Merge pull request #2828 from lbryio/update_lbrycrd
update lbrycrd in tests to v0.17.4.3
2020-02-22 14:47:48 -05:00
Lex Berezhny
f514123ef0 update lbrycrd 2020-02-22 13:42:28 -05:00
Lex Berezhny
9f461db0d0
Merge pull request #2827 from lbryio/repost_inherit_tags
reposts inherit the tags of the original claim, filtered/blocked claims cause their reposts to also be filtered/blocked
2020-02-22 13:04:31 -05:00
Lex Berezhny
7de8670616 blocked original repost causes all reposts to also be blocked 2020-02-22 12:49:20 -05:00
Lex Berezhny
8c7908b200 reposts inherit the tags of the original claim 2020-02-22 12:23:11 -05:00
Lex Berezhny
e7722e039f
Merge pull request #2826 from lbryio/fix_not_channel_ids
`claim_search --not_channel_ids` excludes the channel itself and not just claims in the channel
2020-02-21 23:01:00 -05:00
Lex Berezhny
039bc0208a fix tests 2020-02-21 22:22:28 -05:00
Lex Berezhny
d57900a069 claim_search --not_channel_ids excludes the channel itself and not just claims in the channel 2020-02-21 21:59:46 -05:00
Jack Robison
c1153302aa v0.60.1 2020-02-21 16:13:30 -05:00
Victor Shyba
1086a3297f check its broadcasted and use a retriable call 2020-02-21 17:51:46 -03:00
Jack Robison
5c613934ca
Revert "v0.60.1"
This reverts commit 7f03b13579.
2020-02-21 15:05:33 -05:00
Jack Robison
6d7e9092f8
Merge pull request #2825 from lbryio/tx_from_stream_controller
use the stream controller tx instead of local history
2020-02-21 15:04:57 -05:00
Victor Shyba
8bb6f328dc use the stream controller tx instead of local history 2020-02-21 16:41:23 -03:00
Jack Robison
7f03b13579 v0.60.1 2020-02-21 14:14:15 -05:00
Jack Robison
09008ea991
Merge pull request #2824 from lbryio/skip_wallet_server_payment
max wallet server payment fee defaults to 0 and component doesnt start when its not a positive value
2020-02-21 13:27:58 -05:00
Jack Robison
ee234212e6
Merge branch 'master' into skip_wallet_server_payment 2020-02-21 13:26:38 -05:00
Jack Robison
277b243f52
Merge pull request #2772 from lbryio/reflector-status
Add `is_fully_reflected` to `file_list` response
2020-02-21 13:21:36 -05:00
Jack Robison
0973ac753f
add is_fully_reflected to file_list response 2020-02-21 12:49:19 -05:00
Victor Shyba
f41cfbfb97 max wallet server payment fee defaults to 0 and component doesnt start in that case 2020-02-21 14:47:09 -03:00
Lex Berezhny
a70980c81a reverting v0.60.1 release 2020-02-21 12:43:43 -05:00
Lex Berezhny
12a962f656
Merge pull request #2822 from lbryio/stream_controller_payment
use stream controller and documented errors for wallet server payments
2020-02-21 12:34:07 -05:00
Lex Berezhny
5936444f3e more reliable wait for wallet server payment test 2020-02-21 12:18:44 -05:00
Lex Berezhny
c7d42f00c6 shorten error message for ServerPaymentFeeAboveMaxAllowed 2020-02-21 10:16:56 -05:00
Lex Berezhny
b1318a9958 minor cleanup 2020-02-21 10:05:46 -05:00
Victor Shyba
d2560d260c use stream controller and documented errors 2020-02-21 00:04:37 -03:00
Jack Robison
f40a61cf9a v0.60.1 2020-02-20 17:56:54 -05:00
Jack Robison
e0623578bf
Merge pull request #2821 from lbryio/ffmpeg-find
add `ffmpeg_find` api to check ffmpeg installation status, don't recheck for ffmpeg in `status`
2020-02-20 17:48:36 -05:00
Jack Robison
ead8daaa14
doc 2020-02-20 17:27:18 -05:00
Jack Robison
23b4b9e230
add ffmpeg_find api, don't recheck for it in status 2020-02-20 16:43:41 -05:00
Jack Robison
947017e334
Merge pull request #2820 from lbryio/disconnect-on-oversized
disconnect from client sending message larger than MAX_RECEIVE
2020-02-20 16:24:26 -05:00
Jack Robison
7fd0d6507f
disconnect from client sending message larger than MAX_RECEIVE 2020-02-20 16:08:21 -05:00
Jack Robison
ad1e9ef086
Merge pull request #2819 from lbryio/max-receive
add MAX_RECEIVE setting to wallet server
2020-02-20 14:38:11 -05:00
Jack Robison
2f1d08e417
add MAX_RECEIVE wallet server setting 2020-02-20 14:11:16 -05:00
Lex Berezhny
55f4eb80ba
Merge pull request #2815 from lbryio/apply_share_usage_data_live
apply share_usage_data as its set, without restarting
2020-02-20 13:35:47 -05:00
Victor Shyba
7c160ff65e remove unused parameters 2020-02-20 14:38:13 -03:00
Victor Shyba
fc5d5faaed use conf directly isntead of lambda 2020-02-20 14:27:39 -03:00
Victor Shyba
5394f1763c simplify external ip logic 2020-02-20 12:12:40 -03:00
Victor Shyba
050b67c9d6 apply share_usage_data as its set 2020-02-20 12:12:40 -03:00
Lex Berezhny
4b87cb45ee
Merge pull request #2707 from lbryio/pay_wallet_servers
wallet servers requiring a daily fee will automatically get paid by client
2020-02-18 17:40:23 -05:00
Victor Shyba
f0e1db319c make wallet server payments a component 2020-02-18 19:18:30 -03:00
Victor Shyba
3950715237 add max_wallet_server_fee conf to limit daily wallet server payments 2020-02-18 18:10:58 -03:00
Victor Shyba
ae9ba14b59 use both donation and payment addresses separately 2020-02-18 18:10:58 -03:00
Victor Shyba
ad6c6fbe35 fixes from review and add analytics 2020-02-18 18:10:58 -03:00
Victor Shyba
ffa5c20c88 fix hanging test 2020-02-18 18:10:58 -03:00
Victor Shyba
5a6218eeca improve tests, fix types 2020-02-18 18:10:58 -03:00
Victor Shyba
200531dd96 wait for tx instead 2020-02-18 18:10:58 -03:00
Victor Shyba
15a2f048ac pay wallet server test and initial prototype 2020-02-18 18:10:58 -03:00
Victor Shyba
d317a4042c change daily_fee to be lbc instead of dewies 2020-02-18 18:10:58 -03:00
Victor Shyba
c2d717aba5 old code from #2683 2020-02-18 18:10:58 -03:00
Lex Berezhny
ef74777df1 v0.60.0 2020-02-17 17:52:11 -05:00
Lex Berezhny
f6dbf99fb5
Merge pull request #2811 from lbryio/index_fixes
wallet server indexes
2020-02-17 17:43:55 -05:00
Lex Berezhny
0b2d9e15b4 added order bys for fee_amount 2020-02-17 17:24:48 -05:00
Lex Berezhny
d14e5e75e8 all three order bys for duration filter 2020-02-17 17:24:48 -05:00
Lex Berezhny
df11ef34d2 index cleanup 2020-02-17 17:24:48 -05:00
Lex Berezhny
4fc619262c replacing indexes on trending_global, trending_mixed to be trending_group, trending_mixed 2020-02-17 17:24:48 -05:00
Lex Berezhny
2750eee86d
Merge pull request #2810 from lbryio/no_default_order_by
removed default `order_by` for `claim_search`
2020-02-17 17:04:41 -05:00
Lex Berezhny
71cb03345b removed default order_by for claim_search 2020-02-17 16:26:29 -05:00
Lex Berezhny
22c8ca6c40
Merge pull request #2806 from lbryio/privacyDefault
change default privacy setting
2020-02-17 16:21:34 -05:00
Victor Shyba
237a8965eb fix tests 2020-02-17 18:00:23 -03:00
jessop
7d872c7863 change default privacy setting 2020-02-13 18:56:11 -05:00
Jack Robison
0d08858dfb v0.59.2 2020-02-13 17:22:21 -05:00
Jack Robison
5c6d63c93d
Merge pull request #2803 from lbryio/fix-2802
fix claim apis failing with --resolve if no claims are present
2020-02-13 15:15:56 -05:00
Jack Robison
c8c10d2bb0
fix claim apis failing with --resolve if no claims are present
fixes https://github.com/lbryio/lbry-sdk/issues/2802
2020-02-13 14:58:51 -05:00
Jack Robison
f4d3e9ea6a
add development wallet server deploy script 2020-02-13 13:54:46 -05:00
Jack Robison
3713d3488d
Merge pull request #2800 from lbryio/fix-2746
Fix race condition setting/clearing the claim_search and resolve cache
2020-02-13 11:39:27 -05:00
Jack Robison
9b46d03c91
fix race between setting items in and clearing the resolve/claim_search cache 2020-02-13 11:17:21 -05:00
Jack Robison
fe4e738580
Merge pull request #2798 from lbryio/repost-index
Add sqlite index for reposted claims
2020-02-13 10:33:36 -05:00
Jack Robison
205b0c4263
call get_platform only once 2020-02-13 10:04:50 -05:00
Jack Robison
78eef25f1a
add reposted_claim_hash index
fixes https://github.com/lbryio/lbry-sdk/issues/2782
2020-02-12 17:39:57 -05:00
Lex Berezhny
c9305fd070
Merge pull request #2796 from lbryio/xenial_build
linux build of LBRY SDK for Ubuntu 16.04 Xenial
2020-02-12 14:28:41 -05:00
Lex Berezhny
20774280b9
Merge pull request #2790 from lbryio/sql_in_for_single_value
SQL generation fix to handle IN operation for one value lists
2020-02-12 14:27:36 -05:00
Lex Berezhny
74bd2557af linux build of LBRY SDK for Ubuntu 16.05 Xenial 2020-02-12 14:20:06 -05:00
Lex Berezhny
605fc8ecd8 claim_search by --claim_type=repost support 2020-02-12 11:51:35 -05:00
Lex Berezhny
6b745c53dc make sure SQL IN works with python set 2020-02-12 11:41:32 -05:00
Lex Berezhny
98c60b38fc disable too-many-nested-blocks lint rule 2020-02-12 10:47:40 -05:00
Lex Berezhny
be4515ec9a use sets for __in constraint values 2020-02-12 10:39:25 -05:00
Lex Berezhny
dcb1b64696 SQL generation fix to handle IN operation for one value lists 2020-02-12 10:31:27 -05:00
Jack Robison
1190a60c7a v0.59.1 2020-02-11 16:56:37 -05:00
Jack Robison
82ecf207c6
Merge pull request #2789 from lbryio/update-default-servers
update default wallet servers
2020-02-11 16:36:31 -05:00
Jack Robison
84c7f1bf58
update test 2020-02-11 16:13:55 -05:00
Jack Robison
5ac453fe07
update default wallet servers 2020-02-11 13:51:15 -05:00
Lex Berezhny
c2f39d653d v0.59.0 2020-02-10 12:08:24 -05:00
Lex Berezhny
748178f536 Merge pull request #2780 from lbryio/release_current
manually incremented version and added support to release script to r…
2020-02-09 20:41:00 -05:00
Lex Berezhny
ffbef69dc7 manually incremented version and added support to release script to release current version 2020-02-09 20:26:15 -05:00
Lex Berezhny
3757124323
Merge pull request #2771 from lbryio/update_url_regex
update url regex to include missing cases
2020-02-09 18:17:35 -05:00
Victor Shyba
e65f214b3c fix test using space in name 2020-02-09 17:55:29 -05:00
Victor Shyba
f14004e56b update url regex to cases tom reported 2020-02-09 17:55:29 -05:00
Lex Berezhny
dadc004dd6
Merge pull request #2765 from eggplantbren/master
fixed some issues with AR trending
2020-02-09 17:53:27 -05:00
Brendon J. Brewer
3697d9e1f0 Tweaked some parameters, back to 134 2020-02-09 17:38:34 -05:00
Brendon J. Brewer
94524f7330 lint 2020-02-09 17:38:34 -05:00
Brendon J. Brewer
17d8a3e5d6 Use claim_hash not claim_id 2020-02-09 17:38:24 -05:00
Brendon J. Brewer
39996d7612 Only create file during install if logging is turned on 2020-02-09 17:37:55 -05:00
Lex Berezhny
601ee4320f Merge pull request #2768 from mkroman/master
update cffi to v1.13.2
2020-02-09 17:35:06 -05:00
Mikkel Kroman
6e4f6fa92d Update cffi to v1.13.2
This fixes compilation issues for CPython 3.8+
2020-02-09 17:19:53 -05:00
Lex Berezhny
b7103c29dd Merge pull request #2773 from lbryio/fix_video_analysis_error
fix video analysis error
2020-02-09 17:19:23 -05:00
Brannon King
e53209e5ae handle early lbrycrd exit in tests 2020-02-09 17:02:16 -05:00
Brannon King
501fbd3114 handle strange file metadata 2020-02-09 17:02:16 -05:00
Lex Berezhny
5626f43e2b
Merge pull request #2779 from lbryio/github_actions
build community PRs using GitHub Actions
2020-02-09 16:49:40 -05:00
Lex Berezhny
0185cb48fb run lint and tests in parallel 2020-02-09 16:34:04 -05:00
Lex Berezhny
9a49eb06da fix import path in test_transcoding.py 2020-02-09 16:15:20 -05:00
Lex Berezhny
3fbc012231 moved blockchain/test_transcoding.py -> other/test_transcoding.py 2020-02-09 16:15:20 -05:00
Lex Berezhny
3938cf33d8 take github actions for a spin 2020-02-09 16:15:04 -05:00
Lex Berezhny
38b3d0c87c
Merge pull request #2777 from lbryio/claim_list_multi_type
`claim_list` argument `--claim_type` can now be repeated to pass a list of claim_types
2020-02-08 23:34:34 -05:00
Lex Berezhny
15a2fa6199 claim_list --claim_type argument can be repeated 2020-02-08 23:34:04 -05:00
Lex Berezhny
e834209d40 Merge pull request #2776 from lbryio/resolve_includes_censoring_channels_url
`resolve` results which are censored now include full details of channel which did the censoring
2020-02-08 23:32:44 -05:00
Lex Berezhny
9607d21828 censored searches/resolves include metadata of channel which did the censoring 2020-02-07 18:50:29 -05:00
Lex Berezhny
6fbbf36143 pylint 2020-02-07 18:44:30 -05:00
Lex Berezhny
db25f3282e lbry.error: save arguments on self and support for functions around arguments 2020-02-07 18:25:06 -05:00
Jack Robison
ceb92838e7 v0.58.2 2020-02-06 11:37:09 -05:00
Jack Robison
afb98f2cfc
Merge pull request #2770 from lbryio/disable-save-content-claim
Add `save_resolved_claims` config setting for better performance when not using file_x commands
2020-02-06 11:32:49 -05:00
Jack Robison
ed785fb087
add save_resolved_claims config setting to disable saving claims upon resolving them 2020-02-06 11:27:26 -05:00
231 changed files with 19437 additions and 17492 deletions

206
.github/workflows/main.yml vendored Normal file
View file

@ -0,0 +1,206 @@
name: ci
on: ["push", "pull_request", "workflow_dispatch"]
jobs:
lint:
name: lint
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: extract pip cache
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
restore-keys: ${{ runner.os }}-pip-
- run: pip install --user --upgrade pip wheel
- run: pip install -e .[lint]
- run: make lint
tests-unit:
name: "tests / unit"
strategy:
matrix:
os:
- ubuntu-20.04
- macos-latest
- windows-latest
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: set pip cache dir
shell: bash
run: echo "PIP_CACHE_DIR=$(pip cache dir)" >> $GITHUB_ENV
- name: extract pip cache
uses: actions/cache@v3
with:
path: ${{ env.PIP_CACHE_DIR }}
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
restore-keys: ${{ runner.os }}-pip-
- id: os-name
uses: ASzc/change-string-case-action@v5
with:
string: ${{ runner.os }}
- run: python -m pip install --user --upgrade pip wheel
- if: startsWith(runner.os, 'linux')
run: pip install -e .[test]
- if: startsWith(runner.os, 'linux')
env:
HOME: /tmp
run: make test-unit-coverage
- if: startsWith(runner.os, 'linux') != true
run: pip install -e .[test]
- if: startsWith(runner.os, 'linux') != true
env:
HOME: /tmp
run: coverage run --source=lbry -m unittest tests/unit/test_conf.py
- name: submit coverage report
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
COVERALLS_FLAG_NAME: tests-unit-${{ steps.os-name.outputs.lowercase }}
COVERALLS_PARALLEL: true
run: |
pip install coveralls
coveralls --service=github
tests-integration:
name: "tests / integration"
runs-on: ubuntu-20.04
strategy:
matrix:
test:
- datanetwork
- blockchain
- claims
- takeovers
- transactions
- other
steps:
- name: Configure sysctl limits
run: |
sudo swapoff -a
sudo sysctl -w vm.swappiness=1
sudo sysctl -w fs.file-max=262144
sudo sysctl -w vm.max_map_count=262144
- name: Runs Elasticsearch
uses: elastic/elastic-github-actions/elasticsearch@master
with:
stack-version: 7.12.1
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: '3.9'
- if: matrix.test == 'other'
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends ffmpeg
- name: extract pip cache
uses: actions/cache@v3
with:
path: ./.tox
key: tox-integration-${{ matrix.test }}-${{ hashFiles('setup.py') }}
restore-keys: txo-integration-${{ matrix.test }}-
- run: pip install tox coverage coveralls
- if: matrix.test == 'claims'
run: rm -rf .tox
- run: tox -e ${{ matrix.test }}
- name: submit coverage report
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
COVERALLS_FLAG_NAME: tests-integration-${{ matrix.test }}
COVERALLS_PARALLEL: true
run: |
coverage combine tests
coveralls --service=github
coverage:
needs: ["tests-unit", "tests-integration"]
runs-on: ubuntu-20.04
steps:
- name: finalize coverage report submission
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
pip install coveralls
coveralls --service=github --finish
build:
needs: ["lint", "tests-unit", "tests-integration"]
name: "build / binary"
strategy:
matrix:
os:
- ubuntu-20.04
- macos-latest
- windows-latest
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: '3.9'
- id: os-name
uses: ASzc/change-string-case-action@v5
with:
string: ${{ runner.os }}
- name: set pip cache dir
shell: bash
run: echo "PIP_CACHE_DIR=$(pip cache dir)" >> $GITHUB_ENV
- name: extract pip cache
uses: actions/cache@v3
with:
path: ${{ env.PIP_CACHE_DIR }}
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
restore-keys: ${{ runner.os }}-pip-
- run: pip install pyinstaller==4.6
- run: pip install -e .
- if: startsWith(github.ref, 'refs/tags/v')
run: python docker/set_build.py
- if: startsWith(runner.os, 'linux') || startsWith(runner.os, 'mac')
name: Build & Run (Unix)
run: |
pyinstaller --onefile --name lbrynet lbry/extras/cli.py
dist/lbrynet --version
- if: startsWith(runner.os, 'windows')
name: Build & Run (Windows)
run: |
pip install pywin32==301
pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico --onefile --name lbrynet lbry/extras/cli.py
dist/lbrynet.exe --version
- uses: actions/upload-artifact@v3
with:
name: lbrynet-${{ steps.os-name.outputs.lowercase }}
path: dist/
release:
name: "release"
if: startsWith(github.ref, 'refs/tags/v')
needs: ["build"]
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v1
- uses: actions/download-artifact@v2
- name: upload binaries
env:
GITHUB_TOKEN: ${{ secrets.RELEASE_API_TOKEN }}
run: |
pip install githubrelease
chmod +x lbrynet-macos/lbrynet
chmod +x lbrynet-linux/lbrynet
zip --junk-paths lbrynet-mac.zip lbrynet-macos/lbrynet
zip --junk-paths lbrynet-linux.zip lbrynet-linux/lbrynet
zip --junk-paths lbrynet-windows.zip lbrynet-windows/lbrynet.exe
ls -lh
githubrelease release lbryio/lbry-sdk info ${GITHUB_REF#refs/tags/}
githubrelease asset lbryio/lbry-sdk upload ${GITHUB_REF#refs/tags/} \
lbrynet-mac.zip lbrynet-linux.zip lbrynet-windows.zip
githubrelease release lbryio/lbry-sdk publish ${GITHUB_REF#refs/tags/}

22
.github/workflows/release.yml vendored Normal file
View file

@ -0,0 +1,22 @@
name: slack
on:
release:
types: [published]
jobs:
release:
name: "slack notification"
runs-on: ubuntu-20.04
steps:
- uses: LoveToKnow/slackify-markdown-action@v1.0.0
id: markdown
with:
text: "There is a new SDK release: ${{github.event.release.html_url}}\n${{ github.event.release.body }}"
- uses: slackapi/slack-github-action@v1.14.0
env:
CHANGELOG: '<!channel> ${{ steps.markdown.outputs.text }}'
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_RELEASE_BOT_WEBHOOK }}
with:
payload: '{"type": "mrkdwn", "text": ${{ toJSON(env.CHANGELOG) }} }'

7
.gitignore vendored
View file

@ -6,12 +6,17 @@
/.coverage* /.coverage*
/lbry-venv /lbry-venv
/venv /venv
/lbry/blockchain
lbry.egg-info lbry.egg-info
__pycache__ __pycache__
_trial_temp/ _trial_temp/
trending*.log
/tests/integration/blockchain/files /tests/integration/claims/files
/tests/.coverage.* /tests/.coverage.*
/lbry/wallet/bin /lbry/wallet/bin
/.vscode
/.gitignore

View file

@ -1,209 +0,0 @@
default:
image: python:3.7
#cache:
# directories:
# - $HOME/venv
# - $HOME/.cache/pip
# - $HOME/Library/Caches/pip
# - $HOME/Library/Caches/Homebrew
# - $TRAVIS_BUILD_DIR/.tox
stages:
- test
- build
- assets
- release
.tagged:
rules:
- if: '$CI_COMMIT_TAG =~ /^v[0-9\.]+$/'
when: on_success
test:lint:
stage: test
script:
- make install tools
- make lint
test:unit:
stage: test
script:
- make install tools
- HOME=/tmp coverage run -p --source=lbry -m unittest discover -vv tests.unit
test:datanetwork-integration:
stage: test
script:
- pip install tox-travis
- tox -e datanetwork
test:blockchain-integration:
stage: test
script:
- apt-get update
- apt-get install -y --no-install-recommends ffmpeg
- pip install tox-travis
- tox -e blockchain
test:other-integration:
stage: test
script:
- pip install tox-travis
- tox -e other
test:json-api:
stage: test
script:
- make install tools
- HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
.build:
stage: build
artifacts:
expire_in: 1 day
paths:
- lbrynet-${OS}.zip
script:
- pip install --upgrade 'setuptools<45.0.0'
- pip install pyinstaller
- pip install -e .
- python3.7 docker/set_build.py # must come after lbry is installed because it imports lbry
- pyinstaller --onefile --name lbrynet lbry/extras/cli.py
- chmod +x dist/lbrynet
- zip --junk-paths ${CI_PROJECT_DIR}/lbrynet-${OS}.zip dist/lbrynet # gitlab expects artifacts to be in $CI_PROJECT_DIR
- openssl dgst -sha256 ${CI_PROJECT_DIR}/lbrynet-${OS}.zip | egrep -o [0-9a-f]+$ # get sha256 of asset. works on mac and ubuntu
- dist/lbrynet --version
build:linux:
extends: .build
image: ubuntu:18.04 # cant use python3.7 image - binary won't run on ubuntu
variables:
OS: linux
before_script:
- apt-get update
- apt-get install -y --no-install-recommends software-properties-common zip curl build-essential
- add-apt-repository -y ppa:deadsnakes/ppa
- apt-get install -y --no-install-recommends python3.7-dev python3-setuptools python3-wheel
- python3.7 <(curl -q https://bootstrap.pypa.io/get-pip.py) # make sure we get pip with python3.7
build:mac:
extends: .build
tags: [macos] # makes gitlab use the mac runner
variables:
OS: mac
GIT_DEPTH: 5
VENV: /tmp/gitlab-lbry-sdk-venv
before_script:
# - brew upgrade python || true
- python3 --version | grep -q '^Python 3\.7\.' # dont upgrade python on every run. just make sure we're on the right Python
# - pip3 install --user --upgrade pip virtualenv
- pip3 --version | grep -q '\(python 3\.7\)'
- virtualenv --python=python3.7 "${VENV}"
- source "${VENV}/bin/activate"
after_script:
- rm -rf "${VENV}"
build:windows:
extends: .build
tags: [windows] # makes gitlab use the windows runner
variables:
OS: windows
GIT_DEPTH: 5
before_script:
- ./docker/install_choco.ps1
- choco install -y --x86 python3 7zip checksum
- python --version # | findstr /B "Python 3\.7\." # dont upgrade python on every run. just make sure we're on the right Python
- pip --version # | findstr /E '\(python 3\.7\)'
- pip install virtualenv pywin32
- virtualenv venv
- venv/Scripts/activate.ps1
- pip install pip==19.3.1; $true # $true ignores errors. need this to get the correct coincurve wheel. see commit notes for details.
after_script:
- rmdir -Recurse venv
script:
- pip install --upgrade 'setuptools<45.0.0'
- pip install pyinstaller==3.5
- pip install -e .
- python docker/set_build.py # must come after lbry is installed because it imports lbry
- pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico -F -n lbrynet lbry/extras/cli.py
- 7z a -tzip $env:CI_PROJECT_DIR/lbrynet-${OS}.zip ./dist/lbrynet.exe
- checksum --type=sha256 --file=$env:CI_PROJECT_DIR/lbrynet-${OS}.zip
- dist/lbrynet.exe --version
# s3 = upload asset to s3 (build.lbry.io)
.s3:
stage: assets
variables:
GIT_STRATEGY: none
script:
- "[ -f lbrynet-${OS}.zip ]" # check that asset exists before trying to upload
- pip install awscli
- S3_PATH="daemon/gitlab-build-${CI_PIPELINE_ID}_commit-${CI_COMMIT_SHA:0:7}$( if [ ! -z ${CI_COMMIT_TAG} ]; then echo _tag-${CI_COMMIT_TAG}; else echo _branch-${CI_COMMIT_REF_NAME}; fi )"
- AWS_ACCESS_KEY_ID=${ARTIFACTS_KEY} AWS_SECRET_ACCESS_KEY=${ARTIFACTS_SECRET} AWS_REGION=${ARTIFACTS_REGION}
aws s3 cp lbrynet-${OS}.zip s3://${ARTIFACTS_BUCKET}/${S3_PATH}/lbrynet-${OS}.zip
s3:linux:
extends: .s3
variables: {OS: linux}
needs: ["build:linux"]
s3:mac:
extends: .s3
variables: {OS: mac}
needs: ["build:mac"]
s3:windows:
extends: .s3
variables: {OS: windows}
needs: ["build:windows"]
# github = upload assets to github when there's a tagged release
.github:
extends: .tagged
stage: assets
variables:
GIT_STRATEGY: none
script:
- "[ -f lbrynet-${OS}.zip ]" # check that asset exists before trying to upload. githubrelease won't error if its missing
- pip install githubrelease
- githubrelease --no-progress --github-token ${GITHUB_CI_USER_ACCESS_TOKEN} asset lbryio/lbry-sdk upload ${CI_COMMIT_TAG} lbrynet-${OS}.zip
github:linux:
extends: .github
variables: {OS: linux}
needs: ["build:linux"]
github:mac:
extends: .github
variables: {OS: mac}
needs: ["build:mac"]
github:windows:
extends: .github
variables: {OS: windows}
needs: ["build:windows"]
publish:
extends: .tagged
stage: release
variables:
GIT_STRATEGY: none
script:
- pip install githubrelease
- githubrelease --no-progress --github-token ${GITHUB_CI_USER_ACCESS_TOKEN} release lbryio/lbry-sdk publish ${CI_COMMIT_TAG}
- >
curl -X POST -H 'Content-type: application/json' --data '{"text":"<!channel> There is a new SDK release: https://github.com/lbryio/lbry-sdk/releases/tag/'"${CI_COMMIT_TAG}"'\n'"$(curl -s "https://api.github.com/repos/lbryio/lbry-sdk/releases/tags/${CI_COMMIT_TAG}" | egrep '\w*\"body\":' | cut -d':' -f 2- | tail -c +3 | head -c -2)"'", "channel":"tech"}' "$(echo ${SLACK_WEBHOOK_URL_BASE64} | base64 -d)"

View file

@ -9,20 +9,29 @@ Here's a video walkthrough of this setup, which is itself hosted by the LBRY net
## Prerequisites ## Prerequisites
Running `lbrynet` from source requires Python 3.7 or higher. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/). Running `lbrynet` from source requires Python 3.7. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/).
After installing python 3, you'll need to install some additional libraries depending on your operating system. After installing Python 3.7, you'll need to install some additional libraries depending on your operating system.
Because of [issue #2769](https://github.com/lbryio/lbry-sdk/issues/2769)
at the moment the `lbrynet` daemon will only work correctly with Python 3.7.
If Python 3.8+ is used, the daemon will start but the RPC server
may not accept messages, returning the following:
```
Could not connect to daemon. Are you sure it's running?
```
### macOS ### macOS
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/). macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
These environment variables also need to be set: These environment variables also need to be set:
1. PYTHONUNBUFFERED=1 ```
2. EVENT_NOKQUEUE=1 PYTHONUNBUFFERED=1
EVENT_NOKQUEUE=1
```
Remaining dependencies can then be installed by running: Remaining dependencies can then be installed by running:
``` ```
brew install python protobuf brew install python protobuf
``` ```
@ -31,14 +40,17 @@ Assistance installing Python3: https://docs.python-guide.org/starting/install3/o
### Linux ### Linux
On Ubuntu (16.04 minimum, we recommend 18.04), install the following: On Ubuntu (we recommend 18.04 or 20.04), install the following:
``` ```
sudo add-apt-repository ppa:deadsnakes/ppa sudo add-apt-repository ppa:deadsnakes/ppa
sudo apt-get update sudo apt-get update
sudo apt-get install build-essential python3.7 python3.7-dev git python3.7-venv libssl-dev python-protobuf sudo apt-get install build-essential python3.7 python3.7-dev git python3.7-venv libssl-dev python-protobuf
``` ```
The [deadsnakes PPA](https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa) provides Python 3.7
for those Ubuntu distributions that no longer have it in their
official repositories.
On Raspbian, you will also need to install `python-pyparsing`. On Raspbian, you will also need to install `python-pyparsing`.
If you're running another Linux distro, install the equivalent of the above packages for your system. If you're running another Linux distro, install the equivalent of the above packages for your system.
@ -47,62 +59,119 @@ If you're running another Linux distro, install the equivalent of the above pack
### Linux/Mac ### Linux/Mac
To install on Linux/Mac:
```
Clone the repository: Clone the repository:
$ git clone https://github.com/lbryio/lbry-sdk.git ```bash
$ cd lbry-sdk git clone https://github.com/lbryio/lbry-sdk.git
cd lbry-sdk
```
Create a Python virtual environment for lbry-sdk: Create a Python virtual environment for lbry-sdk:
$ python3.7 -m venv lbry-venv ```bash
python3.7 -m venv lbry-venv
Activating lbry-sdk virtual environment:
$ source lbry-venv/bin/activate
Make sure you're on Python 3.7+ (as the default Python in virtual environment):
$ python --version
Install packages:
$ make install
If you are on Linux and using PyCharm, generates initial configs:
$ make idea
``` ```
To verify your installation, `which lbrynet` should return a path inside of the `lbry-venv` folder created by the `python3.7 -m venv lbry-venv` command. Activate virtual environment:
```bash
source lbry-venv/bin/activate
```
Make sure you're on Python 3.7+ as default in the virtual environment:
```bash
python --version
```
Install packages:
```bash
make install
```
If you are on Linux and using PyCharm, generates initial configs:
```bash
make idea
```
To verify your installation, `which lbrynet` should return a path inside
of the `lbry-venv` folder.
```bash
(lbry-venv) $ which lbrynet
/opt/lbry-sdk/lbry-venv/bin/lbrynet
```
To exit the virtual environment simply use the command `deactivate`.
### Windows ### Windows
To install on Windows:
```
Clone the repository: Clone the repository:
> git clone https://github.com/lbryio/lbry-sdk.git ```bash
> cd lbry-sdk git clone https://github.com/lbryio/lbry-sdk.git
cd lbry-sdk
```
Create a Python virtual environment for lbry-sdk: Create a Python virtual environment for lbry-sdk:
> python -m venv lbry-venv ```bash
python -m venv lbry-venv
```
Activating lbry-sdk virtual environment: Activate virtual environment:
> lbry-venv\Scripts\activate ```bash
lbry-venv\Scripts\activate
```
Install packages: Install packages:
> pip install -e . ```bash
pip install -e .
``` ```
## Run the tests ## Run the tests
### Elasticsearch
To run the unit tests from the repo directory: For running integration tests, Elasticsearch is required to be available at localhost:9200/
The easiest way to start it is using docker with:
```bash
make elastic-docker
```
Alternative installation methods are available [at Elasticsearch website](https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html).
To run the unit and integration tests from the repo directory:
``` ```
python -m unittest discover tests.unit python -m unittest discover tests.unit
python -m unittest discover tests.integration
``` ```
## Usage ## Usage
To start the API server: To start the API server:
`lbrynet start` ```
lbrynet start
```
Whenever the code inside [lbry-sdk/lbry](./lbry)
is modified we should run `make install` to recompile the `lbrynet`
executable with the newest code.
## Development
When developing, remember to enter the environment,
and if you wish start the server interactively.
```bash
$ source lbry-venv/bin/activate
(lbry-venv) $ python lbry/extras/cli.py start
```
Parameters can be passed in the same way.
```bash
(lbry-venv) $ python lbry/extras/cli.py wallet balance
```
If a Python debugger (`pdb` or `ipdb`) is installed we can also start it
in this way, set up break points, and step through the code.
```bash
(lbry-venv) $ pip install ipdb
(lbry-venv) $ ipdb lbry/extras/cli.py
```
Happy hacking! Happy hacking!

View file

@ -1,6 +1,6 @@
The MIT License (MIT) The MIT License (MIT)
Copyright (c) 2015-2020 LBRY Inc Copyright (c) 2015-2022 LBRY Inc
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,

View file

@ -1,23 +1,26 @@
.PHONY: install tools lint test idea .PHONY: install tools lint test test-unit test-unit-coverage test-integration idea
install: install:
CFLAGS="-DSQLITE_MAX_VARIABLE_NUMBER=2500000" pip install -U https://github.com/rogerbinns/apsw/releases/download/3.30.1-r1/apsw-3.30.1-r1.zip \
--global-option=fetch \
--global-option=--version --global-option=3.30.1 --global-option=--all \
--global-option=build --global-option=--enable --global-option=fts5
pip install -e . pip install -e .
tools:
pip install mypy==0.701
pip install coverage astroid pylint
lint: lint:
pylint --rcfile=setup.cfg lbry pylint --rcfile=setup.cfg lbry
#mypy --ignore-missing-imports lbry #mypy --ignore-missing-imports lbry
test: test: test-unit test-integration
test-unit:
python -m unittest discover tests.unit
test-unit-coverage:
coverage run --source=lbry -m unittest discover -vv tests.unit
test-integration:
tox tox
idea: idea:
mkdir -p .idea mkdir -p .idea
cp -r scripts/idea/* .idea cp -r scripts/idea/* .idea
elastic-docker:
docker run -d -v lbryhub:/usr/share/elasticsearch/data -p 9200:9200 -p 9300:9300 -e"ES_JAVA_OPTS=-Xms512m -Xmx512m" -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.12.1

View file

@ -1,10 +1,10 @@
# <img src="https://raw.githubusercontent.com/lbryio/lbry-sdk/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [![Gitlab CI Badge](https://ci.lbry.tech/lbry/lbry-sdk/badges/master/pipeline.svg)](https://ci.lbry.tech/lbry/lbry-sdk) # <img src="https://raw.githubusercontent.com/lbryio/lbry-sdk/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [![build](https://github.com/lbryio/lbry-sdk/actions/workflows/main.yml/badge.svg)](https://github.com/lbryio/lbry-sdk/actions/workflows/main.yml) [![coverage](https://coveralls.io/repos/github/lbryio/lbry-sdk/badge.svg)](https://coveralls.io/github/lbryio/lbry-sdk)
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers. LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
LBRY SDK for Python is currently the most fully featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components include: LBRY SDK for Python is currently the most fully featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components include:
* Built on Python 3.7+ and `asyncio`. * Built on Python 3.7 and `asyncio`.
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbry.dht](https://github.com/lbryio/lbry-sdk/tree/master/lbry/dht)). * Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbry.dht](https://github.com/lbryio/lbry-sdk/tree/master/lbry/dht)).
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbry.blob_exchange](https://github.com/lbryio/lbry-sdk/tree/master/lbry/blob_exchange)). * Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbry.blob_exchange](https://github.com/lbryio/lbry-sdk/tree/master/lbry/blob_exchange)).
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbry.schema](https://github.com/lbryio/lbry-sdk/tree/master/lbry/schema)). * Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbry.schema](https://github.com/lbryio/lbry-sdk/tree/master/lbry/schema)).
@ -41,7 +41,7 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
## Security ## Security
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.com/faq/gpg-key) if you need it. We take security seriously. Please contact security@lbry.com regarding any security issues. [Our PGP key is here](https://lbry.com/faq/pgp-key) if you need it.
## Contact ## Contact

9
SECURITY.md Normal file
View file

@ -0,0 +1,9 @@
# Security Policy
## Supported Versions
While we are not at v1.0 yet, only the latest release will be supported.
## Reporting a Vulnerability
See https://lbry.com/faq/security

View file

@ -0,0 +1,43 @@
FROM debian:10-slim
ARG user=lbry
ARG projects_dir=/home/$user
ARG db_dir=/database
ARG DOCKER_TAG
ARG DOCKER_COMMIT=docker
ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT
RUN apt-get update && \
apt-get -y --no-install-recommends install \
wget \
automake libtool \
tar unzip \
build-essential \
pkg-config \
libleveldb-dev \
python3.7 \
python3-dev \
python3-pip \
python3-wheel \
python3-setuptools && \
update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 && \
rm -rf /var/lib/apt/lists/*
RUN groupadd -g 999 $user && useradd -m -u 999 -g $user $user
COPY . $projects_dir
RUN chown -R $user:$user $projects_dir
RUN mkdir -p $db_dir
RUN chown -R $user:$user $db_dir
USER $user
WORKDIR $projects_dir
RUN python3 -m pip install -U setuptools pip
RUN make install
RUN python3 docker/set_build.py
RUN rm ~/.cache -rf
VOLUME $db_dir
ENTRYPOINT ["python3", "scripts/dht_node.py"]

View file

@ -1,4 +1,4 @@
FROM ubuntu:19.10 FROM debian:10-slim
ARG user=lbry ARG user=lbry
ARG db_dir=/database ARG db_dir=/database
@ -13,10 +13,14 @@ RUN apt-get update && \
wget \ wget \
tar unzip \ tar unzip \
build-essential \ build-essential \
python3 \ automake libtool \
pkg-config \
libleveldb-dev \
python3.7 \
python3-dev \ python3-dev \
python3-pip \ python3-pip \
python3-wheel \ python3-wheel \
python3-cffi \
python3-setuptools && \ python3-setuptools && \
update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 && \ update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 && \
rm -rf /var/lib/apt/lists/* rm -rf /var/lib/apt/lists/*

45
docker/Dockerfile.web Normal file
View file

@ -0,0 +1,45 @@
FROM debian:10-slim
ARG user=lbry
ARG downloads_dir=/database
ARG projects_dir=/home/$user
ARG DOCKER_TAG
ARG DOCKER_COMMIT=docker
ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT
RUN apt-get update && \
apt-get -y --no-install-recommends install \
wget \
automake libtool \
tar unzip \
build-essential \
pkg-config \
libleveldb-dev \
python3.7 \
python3-dev \
python3-pip \
python3-wheel \
python3-setuptools && \
update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 && \
rm -rf /var/lib/apt/lists/*
RUN groupadd -g 999 $user && useradd -m -u 999 -g $user $user
RUN mkdir -p $downloads_dir
RUN chown -R $user:$user $downloads_dir
COPY . $projects_dir
RUN chown -R $user:$user $projects_dir
USER $user
WORKDIR $projects_dir
RUN pip install uvloop
RUN make install
RUN python3 docker/set_build.py
RUN rm ~/.cache -rf
# entry point
VOLUME $downloads_dir
COPY ./docker/webconf.yaml /webconf.yaml
ENTRYPOINT ["/home/lbry/.local/bin/lbrynet", "start", "--config=/webconf.yaml"]

9
docker/README.md Normal file
View file

@ -0,0 +1,9 @@
### How to run with docker-compose
1. Edit config file and after that fix permissions with
```
sudo chown -R 999:999 webconf.yaml
```
2. Start SDK with
```
docker-compose up -d
```

View file

@ -1,34 +1,49 @@
version: "3" version: "3"
volumes: volumes:
lbrycrd:
wallet_server: wallet_server:
es01:
services: services:
lbrycrd:
image: lbry/lbrycrd:${LBRYCRD_TAG:-latest-release}
restart: always
ports: # accessible from host
- "9246:9246" # rpc port
expose: # internal to docker network. also this doesn't do anything. its for documentation only.
- "9245" # node-to-node comms port
volumes:
- "lbrycrd:/data/.lbrycrd"
environment:
- RUN_MODE=default
- SNAPSHOT_URL=${LBRYCRD_SNAPSHOT_URL-https://lbry.com/snapshot/blockchain}
- RPC_ALLOW_IP=0.0.0.0/0
wallet_server: wallet_server:
image: lbry/wallet-server:${WALLET_SERVER_TAG:-latest-release}
depends_on: depends_on:
- lbrycrd - es01
image: lbry/wallet-server:${WALLET_SERVER_TAG:-latest-release}
restart: always restart: always
network_mode: host
ports: ports:
- "50001:50001" # rpc port - "50001:50001" # rpc port
- "50005:50005" # websocket port - "2112:2112" # uncomment to enable prometheus
#- "2112:2112" # uncomment to enable prometheus
volumes: volumes:
- "wallet_server:/database" - "wallet_server:/database"
environment: environment:
- SNAPSHOT_URL=${WALLET_SERVER_SNAPSHOT_URL-https://lbry.com/snapshot/wallet} - DAEMON_URL=http://lbry:lbry@127.0.0.1:9245
- DAEMON_URL=http://lbry:lbry@lbrycrd:9245 - MAX_QUERY_WORKERS=4
- CACHE_MB=1024
- CACHE_ALL_TX_HASHES=
- CACHE_ALL_CLAIM_TXOS=
- MAX_SEND=1000000000000000000
- MAX_RECEIVE=1000000000000000000
- MAX_SESSIONS=100000
- HOST=0.0.0.0
- TCP_PORT=50001
- PROMETHEUS_PORT=2112
- FILTERING_CHANNEL_IDS=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8
- BLOCKING_CHANNEL_IDS=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6
es01:
image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0
container_name: es01
environment:
- node.name=es01
- discovery.type=single-node
- indices.query.bool.max_clause_count=8192
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms4g -Xmx4g" # no more than 32, remember to disable swap
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- es01:/usr/share/elasticsearch/data
ports:
- 127.0.0.1:9200:9200

View file

@ -0,0 +1,9 @@
version: '3'
services:
websdk:
image: vshyba/websdk
ports:
- '5279:5279'
- '5280:5280'
volumes:
- ./webconf.yaml:/webconf.yaml

View file

@ -20,7 +20,7 @@ def _check_and_set(d: dict, key: str, value: str):
def main(): def main():
build_info = {item: build_info_mod.__dict__[item] for item in dir(build_info_mod) if not item.startswith("__")} build_info = {item: build_info_mod.__dict__[item] for item in dir(build_info_mod) if not item.startswith("__")}
commit_hash = os.getenv('DOCKER_COMMIT', os.getenv('CI_COMMIT_SHA', os.getenv('TRAVIS_COMMIT'))) commit_hash = os.getenv('DOCKER_COMMIT', os.getenv('GITHUB_SHA'))
if commit_hash is None: if commit_hash is None:
raise ValueError("Commit hash not found in env vars") raise ValueError("Commit hash not found in env vars")
_check_and_set(build_info, "COMMIT_HASH", commit_hash[:6]) _check_and_set(build_info, "COMMIT_HASH", commit_hash[:6])
@ -30,8 +30,10 @@ def main():
_check_and_set(build_info, "DOCKER_TAG", docker_tag) _check_and_set(build_info, "DOCKER_TAG", docker_tag)
_check_and_set(build_info, "BUILD", "docker") _check_and_set(build_info, "BUILD", "docker")
else: else:
ci_tag = os.getenv('CI_COMMIT_TAG', os.getenv('TRAVIS_TAG')) if re.match(r'refs/tags/v\d+\.\d+\.\d+$', str(os.getenv('GITHUB_REF'))):
_check_and_set(build_info, "BUILD", "release" if re.match(r'v\d+\.\d+\.\d+$', str(ci_tag)) else "qa") _check_and_set(build_info, "BUILD", "release")
else:
_check_and_set(build_info, "BUILD", "qa")
log.debug("build info: %s", ", ".join([f"{k}={v}" for k, v in build_info.items()])) log.debug("build info: %s", ", ".join([f"{k}={v}" for k, v in build_info.items()]))
with open(build_info_mod.__file__, 'w') as f: with open(build_info_mod.__file__, 'w') as f:

View file

@ -6,7 +6,7 @@ set -euo pipefail
SNAPSHOT_URL="${SNAPSHOT_URL:-}" #off by default. latest snapshot at https://lbry.com/snapshot/wallet SNAPSHOT_URL="${SNAPSHOT_URL:-}" #off by default. latest snapshot at https://lbry.com/snapshot/wallet
if [[ -n "$SNAPSHOT_URL" ]] && [[ ! -f /database/claims.db ]]; then if [[ -n "$SNAPSHOT_URL" ]] && [[ ! -f /database/lbry-leveldb ]]; then
files="$(ls)" files="$(ls)"
echo "Downloading wallet snapshot from $SNAPSHOT_URL" echo "Downloading wallet snapshot from $SNAPSHOT_URL"
wget --no-verbose --trust-server-names --content-disposition "$SNAPSHOT_URL" wget --no-verbose --trust-server-names --content-disposition "$SNAPSHOT_URL"
@ -20,4 +20,6 @@ if [[ -n "$SNAPSHOT_URL" ]] && [[ ! -f /database/claims.db ]]; then
rm "$filename" rm "$filename"
fi fi
/home/lbry/.local/bin/torba-server "$@" /home/lbry/.local/bin/lbry-hub-elastic-sync
echo 'starting server'
/home/lbry/.local/bin/lbry-hub "$@"

9
docker/webconf.yaml Normal file
View file

@ -0,0 +1,9 @@
allowed_origin: "*"
max_key_fee: "0.0 USD"
save_files: false
save_blobs: false
streaming_server: "0.0.0.0:5280"
api: "0.0.0.0:5279"
data_dir: /tmp
download_dir: /tmp
wallet_dir: /tmp

File diff suppressed because one or more lines are too long

View file

@ -1,2 +1,2 @@
__version__ = "0.58.1" __version__ = "0.113.0"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name

View file

@ -1,5 +1,6 @@
import os import os
import re import re
import time
import asyncio import asyncio
import binascii import binascii
import logging import logging
@ -70,21 +71,27 @@ class AbstractBlob:
'writers', 'writers',
'verified', 'verified',
'writing', 'writing',
'readers' 'readers',
'added_on',
'is_mine',
] ]
def __init__(self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None, def __init__(
self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None, blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
blob_directory: typing.Optional[str] = None): blob_directory: typing.Optional[str] = None, added_on: typing.Optional[int] = None, is_mine: bool = False,
):
self.loop = loop self.loop = loop
self.blob_hash = blob_hash self.blob_hash = blob_hash
self.length = length self.length = length
self.blob_completed_callback = blob_completed_callback self.blob_completed_callback = blob_completed_callback
self.blob_directory = blob_directory self.blob_directory = blob_directory
self.writers: typing.Dict[typing.Tuple[typing.Optional[str], typing.Optional[int]], HashBlobWriter] = {} self.writers: typing.Dict[typing.Tuple[typing.Optional[str], typing.Optional[int]], HashBlobWriter] = {}
self.verified: asyncio.Event = asyncio.Event(loop=self.loop) self.verified: asyncio.Event = asyncio.Event()
self.writing: asyncio.Event = asyncio.Event(loop=self.loop) self.writing: asyncio.Event = asyncio.Event()
self.readers: typing.List[typing.BinaryIO] = [] self.readers: typing.List[typing.BinaryIO] = []
self.added_on = added_on or time.time()
self.is_mine = is_mine
if not is_valid_blobhash(blob_hash): if not is_valid_blobhash(blob_hash):
raise InvalidBlobHashError(blob_hash) raise InvalidBlobHashError(blob_hash)
@ -110,7 +117,7 @@ class AbstractBlob:
if reader in self.readers: if reader in self.readers:
self.readers.remove(reader) self.readers.remove(reader)
def _write_blob(self, blob_bytes: bytes): def _write_blob(self, blob_bytes: bytes) -> asyncio.Task:
raise NotImplementedError() raise NotImplementedError()
def set_length(self, length) -> None: def set_length(self, length) -> None:
@ -181,34 +188,41 @@ class AbstractBlob:
@classmethod @classmethod
async def create_from_unencrypted( async def create_from_unencrypted(
cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes, cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
unencrypted: bytes, blob_num: int, unencrypted: bytes, blob_num: int, added_on: int, is_mine: bool,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None) -> BlobInfo: blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None,
) -> BlobInfo:
""" """
Create an encrypted BlobFile from plaintext bytes Create an encrypted BlobFile from plaintext bytes
""" """
blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted) blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted)
length = len(blob_bytes) length = len(blob_bytes)
blob = cls(loop, blob_hash, length, blob_completed_callback, blob_dir) blob = cls(loop, blob_hash, length, blob_completed_callback, blob_dir, added_on, is_mine)
writer = blob.get_blob_writer() writer = blob.get_blob_writer()
writer.write(blob_bytes) writer.write(blob_bytes)
await blob.verified.wait() await blob.verified.wait()
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash) return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), added_on, blob_hash, is_mine)
def save_verified_blob(self, verified_bytes: bytes): def save_verified_blob(self, verified_bytes: bytes):
if self.verified.is_set(): if self.verified.is_set():
return return
if self.is_writeable():
self._write_blob(verified_bytes) def update_events(_):
self.verified.set() self.verified.set()
self.writing.clear()
if self.is_writeable():
self.writing.set()
task = self._write_blob(verified_bytes)
task.add_done_callback(update_events)
if self.blob_completed_callback: if self.blob_completed_callback:
self.blob_completed_callback(self) task.add_done_callback(lambda _: self.blob_completed_callback(self))
def get_blob_writer(self, peer_address: typing.Optional[str] = None, def get_blob_writer(self, peer_address: typing.Optional[str] = None,
peer_port: typing.Optional[int] = None) -> HashBlobWriter: peer_port: typing.Optional[int] = None) -> HashBlobWriter:
if (peer_address, peer_port) in self.writers and not self.writers[(peer_address, peer_port)].closed(): if (peer_address, peer_port) in self.writers and not self.writers[(peer_address, peer_port)].closed():
raise OSError(f"attempted to download blob twice from {peer_address}:{peer_port}") raise OSError(f"attempted to download blob twice from {peer_address}:{peer_port}")
fut = asyncio.Future(loop=self.loop) fut = asyncio.Future()
writer = HashBlobWriter(self.blob_hash, self.get_length, fut) writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
self.writers[(peer_address, peer_port)] = writer self.writers[(peer_address, peer_port)] = writer
@ -242,11 +256,13 @@ class BlobBuffer(AbstractBlob):
""" """
An in-memory only blob An in-memory only blob
""" """
def __init__(self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None, def __init__(
self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None, blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
blob_directory: typing.Optional[str] = None): blob_directory: typing.Optional[str] = None, added_on: typing.Optional[int] = None, is_mine: bool = False
):
self._verified_bytes: typing.Optional[BytesIO] = None self._verified_bytes: typing.Optional[BytesIO] = None
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory) super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory, added_on, is_mine)
@contextlib.contextmanager @contextlib.contextmanager
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]: def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
@ -261,9 +277,11 @@ class BlobBuffer(AbstractBlob):
self.verified.clear() self.verified.clear()
def _write_blob(self, blob_bytes: bytes): def _write_blob(self, blob_bytes: bytes):
async def write():
if self._verified_bytes: if self._verified_bytes:
raise OSError("already have bytes for blob") raise OSError("already have bytes for blob")
self._verified_bytes = BytesIO(blob_bytes) self._verified_bytes = BytesIO(blob_bytes)
return self.loop.create_task(write())
def delete(self): def delete(self):
if self._verified_bytes: if self._verified_bytes:
@ -281,10 +299,12 @@ class BlobFile(AbstractBlob):
""" """
A blob existing on the local file system A blob existing on the local file system
""" """
def __init__(self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None, def __init__(
self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None, blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
blob_directory: typing.Optional[str] = None): blob_directory: typing.Optional[str] = None, added_on: typing.Optional[int] = None, is_mine: bool = False
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory) ):
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory, added_on, is_mine)
if not blob_directory or not os.path.isdir(blob_directory): if not blob_directory or not os.path.isdir(blob_directory):
raise OSError(f"invalid blob directory '{blob_directory}'") raise OSError(f"invalid blob directory '{blob_directory}'")
self.file_path = os.path.join(self.blob_directory, self.blob_hash) self.file_path = os.path.join(self.blob_directory, self.blob_hash)
@ -319,22 +339,28 @@ class BlobFile(AbstractBlob):
handle.close() handle.close()
def _write_blob(self, blob_bytes: bytes): def _write_blob(self, blob_bytes: bytes):
def _write_blob():
with open(self.file_path, 'wb') as f: with open(self.file_path, 'wb') as f:
f.write(blob_bytes) f.write(blob_bytes)
async def write_blob():
await self.loop.run_in_executor(None, _write_blob)
return self.loop.create_task(write_blob())
def delete(self): def delete(self):
super().delete()
if os.path.isfile(self.file_path): if os.path.isfile(self.file_path):
os.remove(self.file_path) os.remove(self.file_path)
return super().delete()
@classmethod @classmethod
async def create_from_unencrypted( async def create_from_unencrypted(
cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes, cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
unencrypted: bytes, blob_num: int, unencrypted: bytes, blob_num: int, added_on: float, is_mine: bool,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None
asyncio.Task]] = None) -> BlobInfo: ) -> BlobInfo:
if not blob_dir or not os.path.isdir(blob_dir): if not blob_dir or not os.path.isdir(blob_dir):
raise OSError(f"cannot create blob in directory: '{blob_dir}'") raise OSError(f"cannot create blob in directory: '{blob_dir}'")
return await super().create_from_unencrypted( return await super().create_from_unencrypted(
loop, blob_dir, key, iv, unencrypted, blob_num, blob_completed_callback loop, blob_dir, key, iv, unencrypted, blob_num, added_on, is_mine, blob_completed_callback
) )

View file

@ -7,13 +7,19 @@ class BlobInfo:
'blob_num', 'blob_num',
'length', 'length',
'iv', 'iv',
'added_on',
'is_mine'
] ]
def __init__(self, blob_num: int, length: int, iv: str, blob_hash: typing.Optional[str] = None): def __init__(
self, blob_num: int, length: int, iv: str, added_on,
blob_hash: typing.Optional[str] = None, is_mine=False):
self.blob_hash = blob_hash self.blob_hash = blob_hash
self.blob_num = blob_num self.blob_num = blob_num
self.length = length self.length = length
self.iv = iv self.iv = iv
self.added_on = added_on
self.is_mine = is_mine
def as_dict(self) -> typing.Dict: def as_dict(self) -> typing.Dict:
d = { d = {

View file

@ -2,7 +2,7 @@ import os
import typing import typing
import asyncio import asyncio
import logging import logging
from lbry.utils import LRUCache from lbry.utils import LRUCacheWithMetrics
from lbry.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob from lbry.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
from lbry.stream.descriptor import StreamDescriptor from lbry.stream.descriptor import StreamDescriptor
from lbry.connection_manager import ConnectionManager from lbry.connection_manager import ConnectionManager
@ -32,34 +32,34 @@ class BlobManager:
else self._node_data_store.completed_blobs else self._node_data_store.completed_blobs
self.blobs: typing.Dict[str, AbstractBlob] = {} self.blobs: typing.Dict[str, AbstractBlob] = {}
self.config = config self.config = config
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCache( self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCacheWithMetrics(
self.config.blob_lru_cache_size) self.config.blob_lru_cache_size)
self.connection_manager = ConnectionManager(loop) self.connection_manager = ConnectionManager(loop)
def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None): def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None, is_mine: bool = False):
if self.config.save_blobs or ( if self.config.save_blobs or (
is_valid_blobhash(blob_hash) and os.path.isfile(os.path.join(self.blob_dir, blob_hash))): is_valid_blobhash(blob_hash) and os.path.isfile(os.path.join(self.blob_dir, blob_hash))):
return BlobFile( return BlobFile(
self.loop, blob_hash, length, self.blob_completed, self.blob_dir self.loop, blob_hash, length, self.blob_completed, self.blob_dir, is_mine=is_mine
) )
return BlobBuffer( return BlobBuffer(
self.loop, blob_hash, length, self.blob_completed, self.blob_dir self.loop, blob_hash, length, self.blob_completed, self.blob_dir, is_mine=is_mine
) )
def get_blob(self, blob_hash, length: typing.Optional[int] = None): def get_blob(self, blob_hash, length: typing.Optional[int] = None, is_mine: bool = False):
if blob_hash in self.blobs: if blob_hash in self.blobs:
if self.config.save_blobs and isinstance(self.blobs[blob_hash], BlobBuffer): if self.config.save_blobs and isinstance(self.blobs[blob_hash], BlobBuffer):
buffer = self.blobs.pop(blob_hash) buffer = self.blobs.pop(blob_hash)
if blob_hash in self.completed_blob_hashes: if blob_hash in self.completed_blob_hashes:
self.completed_blob_hashes.remove(blob_hash) self.completed_blob_hashes.remove(blob_hash)
self.blobs[blob_hash] = self._get_blob(blob_hash, length) self.blobs[blob_hash] = self._get_blob(blob_hash, length, is_mine)
if buffer.is_readable(): if buffer.is_readable():
with buffer.reader_context() as reader: with buffer.reader_context() as reader:
self.blobs[blob_hash].write_blob(reader.read()) self.blobs[blob_hash].write_blob(reader.read())
if length and self.blobs[blob_hash].length is None: if length and self.blobs[blob_hash].length is None:
self.blobs[blob_hash].set_length(length) self.blobs[blob_hash].set_length(length)
else: else:
self.blobs[blob_hash] = self._get_blob(blob_hash, length) self.blobs[blob_hash] = self._get_blob(blob_hash, length, is_mine)
return self.blobs[blob_hash] return self.blobs[blob_hash]
def is_blob_verified(self, blob_hash: str, length: typing.Optional[int] = None) -> bool: def is_blob_verified(self, blob_hash: str, length: typing.Optional[int] = None) -> bool:
@ -83,6 +83,8 @@ class BlobManager:
to_add = await self.storage.sync_missing_blobs(in_blobfiles_dir) to_add = await self.storage.sync_missing_blobs(in_blobfiles_dir)
if to_add: if to_add:
self.completed_blob_hashes.update(to_add) self.completed_blob_hashes.update(to_add)
# check blobs that aren't set as finished but were seen on disk
await self.ensure_completed_blobs_status(in_blobfiles_dir - to_add)
if self.config.track_bandwidth: if self.config.track_bandwidth:
self.connection_manager.start() self.connection_manager.start()
return True return True
@ -105,13 +107,26 @@ class BlobManager:
if isinstance(blob, BlobFile): if isinstance(blob, BlobFile):
if blob.blob_hash not in self.completed_blob_hashes: if blob.blob_hash not in self.completed_blob_hashes:
self.completed_blob_hashes.add(blob.blob_hash) self.completed_blob_hashes.add(blob.blob_hash)
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=True)) return self.loop.create_task(self.storage.add_blobs(
(blob.blob_hash, blob.length, blob.added_on, blob.is_mine), finished=True)
)
else: else:
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=False)) return self.loop.create_task(self.storage.add_blobs(
(blob.blob_hash, blob.length, blob.added_on, blob.is_mine), finished=False)
)
def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]: async def ensure_completed_blobs_status(self, blob_hashes: typing.Iterable[str]):
"""Returns of the blobhashes_to_check, which are valid""" """Ensures that completed blobs from a given list of blob hashes are set as 'finished' in the database."""
return [blob_hash for blob_hash in blob_hashes if self.is_blob_verified(blob_hash)] to_add = []
for blob_hash in blob_hashes:
if not self.is_blob_verified(blob_hash):
continue
blob = self.get_blob(blob_hash)
to_add.append((blob.blob_hash, blob.length, blob.added_on, blob.is_mine))
if len(to_add) > 500:
await self.storage.add_blobs(*to_add, finished=True)
to_add.clear()
return await self.storage.add_blobs(*to_add, finished=True)
def delete_blob(self, blob_hash: str): def delete_blob(self, blob_hash: str):
if not is_valid_blobhash(blob_hash): if not is_valid_blobhash(blob_hash):

View file

@ -0,0 +1,77 @@
import asyncio
import logging
log = logging.getLogger(__name__)
class DiskSpaceManager:
def __init__(self, config, db, blob_manager, cleaning_interval=30 * 60, analytics=None):
self.config = config
self.db = db
self.blob_manager = blob_manager
self.cleaning_interval = cleaning_interval
self.running = False
self.task = None
self.analytics = analytics
self._used_space_bytes = None
async def get_free_space_mb(self, is_network_blob=False):
limit_mb = self.config.network_storage_limit if is_network_blob else self.config.blob_storage_limit
space_used_mb = await self.get_space_used_mb()
space_used_mb = space_used_mb['network_storage'] if is_network_blob else space_used_mb['content_storage']
return max(0, limit_mb - space_used_mb)
async def get_space_used_bytes(self):
self._used_space_bytes = await self.db.get_stored_blob_disk_usage()
return self._used_space_bytes
async def get_space_used_mb(self, cached=True):
cached = cached and self._used_space_bytes is not None
space_used_bytes = self._used_space_bytes if cached else await self.get_space_used_bytes()
return {key: int(value/1024.0/1024.0) for key, value in space_used_bytes.items()}
async def clean(self):
await self._clean(False)
await self._clean(True)
async def _clean(self, is_network_blob=False):
space_used_mb = await self.get_space_used_mb(cached=False)
if is_network_blob:
space_used_mb = space_used_mb['network_storage']
else:
space_used_mb = space_used_mb['content_storage'] + space_used_mb['private_storage']
storage_limit_mb = self.config.network_storage_limit if is_network_blob else self.config.blob_storage_limit
if self.analytics:
asyncio.create_task(
self.analytics.send_disk_space_used(space_used_mb, storage_limit_mb, is_network_blob)
)
delete = []
available = storage_limit_mb - space_used_mb
if storage_limit_mb == 0 if not is_network_blob else available >= 0:
return 0
for blob_hash, file_size, _ in await self.db.get_stored_blobs(is_mine=False, is_network_blob=is_network_blob):
delete.append(blob_hash)
available += int(file_size/1024.0/1024.0)
if available >= 0:
break
if delete:
await self.db.stop_all_files()
await self.blob_manager.delete_blobs(delete, delete_from_db=True)
self._used_space_bytes = None
return len(delete)
async def cleaning_loop(self):
while self.running:
await asyncio.sleep(self.cleaning_interval)
await self.clean()
async def start(self):
self.running = True
self.task = asyncio.create_task(self.cleaning_loop())
self.task.add_done_callback(lambda _: log.info("Stopping blob cleanup service."))
async def stop(self):
if self.running:
self.running = False
self.task.cancel()

View file

@ -32,7 +32,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
self.buf = b'' self.buf = b''
# this is here to handle the race when the downloader is closed right as response_fut gets a result # this is here to handle the race when the downloader is closed right as response_fut gets a result
self.closed = asyncio.Event(loop=self.loop) self.closed = asyncio.Event()
def data_received(self, data: bytes): def data_received(self, data: bytes):
if self.connection_manager: if self.connection_manager:
@ -111,7 +111,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
self.transport.write(msg) self.transport.write(msg)
if self.connection_manager: if self.connection_manager:
self.connection_manager.sent_data(f"{self.peer_address}:{self.peer_port}", len(msg)) self.connection_manager.sent_data(f"{self.peer_address}:{self.peer_port}", len(msg))
response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop) response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout)
availability_response = response.get_availability_response() availability_response = response.get_availability_response()
price_response = response.get_price_response() price_response = response.get_price_response()
blob_response = response.get_blob_response() blob_response = response.get_blob_response()
@ -151,7 +151,9 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
f" timeout in {self.peer_timeout}" f" timeout in {self.peer_timeout}"
log.debug(msg) log.debug(msg)
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}" msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop) await asyncio.wait_for(self.writer.finished, self.peer_timeout)
# wait for the io to finish
await self.blob.verified.wait()
log.info("%s at %fMB/s", msg, log.info("%s at %fMB/s", msg,
round((float(self._blob_bytes_received) / round((float(self._blob_bytes_received) /
float(time.perf_counter() - start_time)) / 1000000.0, 2)) float(time.perf_counter() - start_time)) / 1000000.0, 2))
@ -185,7 +187,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
try: try:
self._blob_bytes_received = 0 self._blob_bytes_received = 0
self.blob, self.writer = blob, blob.get_blob_writer(self.peer_address, self.peer_port) self.blob, self.writer = blob, blob.get_blob_writer(self.peer_address, self.peer_port)
self._response_fut = asyncio.Future(loop=self.loop) self._response_fut = asyncio.Future()
return await self._download_blob() return await self._download_blob()
except OSError: except OSError:
# i'm not sure how to fix this race condition - jack # i'm not sure how to fix this race condition - jack
@ -242,7 +244,7 @@ async def request_blob(loop: asyncio.AbstractEventLoop, blob: Optional['Abstract
try: try:
if not connected_protocol: if not connected_protocol:
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port), await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
peer_connect_timeout, loop=loop) peer_connect_timeout)
connected_protocol = protocol connected_protocol = protocol
if blob is None or blob.get_is_verified() or not blob.is_writeable(): if blob is None or blob.get_is_verified() or not blob.is_writeable():
# blob is None happens when we are just opening a connection # blob is None happens when we are just opening a connection

View file

@ -3,6 +3,7 @@ import typing
import logging import logging
from lbry.utils import cache_concurrent from lbry.utils import cache_concurrent
from lbry.blob_exchange.client import request_blob from lbry.blob_exchange.client import request_blob
from lbry.dht.node import get_kademlia_peers_from_hosts
if typing.TYPE_CHECKING: if typing.TYPE_CHECKING:
from lbry.conf import Config from lbry.conf import Config
from lbry.dht.node import Node from lbry.dht.node import Node
@ -29,7 +30,7 @@ class BlobDownloader:
self.failures: typing.Dict['KademliaPeer', int] = {} self.failures: typing.Dict['KademliaPeer', int] = {}
self.connection_failures: typing.Set['KademliaPeer'] = set() self.connection_failures: typing.Set['KademliaPeer'] = set()
self.connections: typing.Dict['KademliaPeer', 'BlobExchangeClientProtocol'] = {} self.connections: typing.Dict['KademliaPeer', 'BlobExchangeClientProtocol'] = {}
self.is_running = asyncio.Event(loop=self.loop) self.is_running = asyncio.Event()
def should_race_continue(self, blob: 'AbstractBlob'): def should_race_continue(self, blob: 'AbstractBlob'):
max_probes = self.config.max_connections_per_download * (1 if self.connections else 10) max_probes = self.config.max_connections_per_download * (1 if self.connections else 10)
@ -63,8 +64,8 @@ class BlobDownloader:
self.scores[peer] = bytes_received / elapsed if bytes_received and elapsed else 1 self.scores[peer] = bytes_received / elapsed if bytes_received and elapsed else 1
async def new_peer_or_finished(self): async def new_peer_or_finished(self):
active_tasks = list(self.active_connections.values()) + [asyncio.sleep(1)] active_tasks = list(self.active_connections.values()) + [asyncio.create_task(asyncio.sleep(1))]
await asyncio.wait(active_tasks, loop=self.loop, return_when='FIRST_COMPLETED') await asyncio.wait(active_tasks, return_when='FIRST_COMPLETED')
def cleanup_active(self): def cleanup_active(self):
if not self.active_connections and not self.connections: if not self.active_connections and not self.connections:
@ -87,7 +88,6 @@ class BlobDownloader:
if blob.get_is_verified(): if blob.get_is_verified():
return blob return blob
self.is_running.set() self.is_running.set()
tried_for_this_blob: typing.Set['KademliaPeer'] = set()
try: try:
while not blob.get_is_verified() and self.is_running.is_set(): while not blob.get_is_verified() and self.is_running.is_set():
batch: typing.Set['KademliaPeer'] = set(self.connections.keys()) batch: typing.Set['KademliaPeer'] = set(self.connections.keys())
@ -97,23 +97,14 @@ class BlobDownloader:
"%s running, %d peers, %d ignored, %d active, %s connections", blob_hash[:6], "%s running, %d peers, %d ignored, %d active, %s connections", blob_hash[:6],
len(batch), len(self.ignored), len(self.active_connections), len(self.connections) len(batch), len(self.ignored), len(self.active_connections), len(self.connections)
) )
re_add: typing.Set['KademliaPeer'] = set()
for peer in sorted(batch, key=lambda peer: self.scores.get(peer, 0), reverse=True): for peer in sorted(batch, key=lambda peer: self.scores.get(peer, 0), reverse=True):
if peer in self.ignored: if peer in self.ignored:
continue continue
if peer in tried_for_this_blob: if peer in self.active_connections or not self.should_race_continue(blob):
continue continue
if peer in self.active_connections:
if peer not in re_add:
re_add.add(peer)
continue
if not self.should_race_continue(blob):
break
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port) log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id)) t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id))
self.active_connections[peer] = t self.active_connections[peer] = t
tried_for_this_blob.add(peer)
if not re_add:
self.peer_queue.put_nowait(list(batch)) self.peer_queue.put_nowait(list(batch))
await self.new_peer_or_finished() await self.new_peer_or_finished()
self.cleanup_active() self.cleanup_active()
@ -133,11 +124,14 @@ class BlobDownloader:
protocol.close() protocol.close()
async def download_blob(loop, config: 'Config', blob_manager: 'BlobManager', node: 'Node', async def download_blob(loop, config: 'Config', blob_manager: 'BlobManager', dht_node: 'Node',
blob_hash: str) -> 'AbstractBlob': blob_hash: str) -> 'AbstractBlob':
search_queue = asyncio.Queue(loop=loop, maxsize=config.max_connections_per_download) search_queue = asyncio.Queue(maxsize=config.max_connections_per_download)
search_queue.put_nowait(blob_hash) search_queue.put_nowait(blob_hash)
peer_queue, accumulate_task = node.accumulate_peers(search_queue) peer_queue, accumulate_task = dht_node.accumulate_peers(search_queue)
fixed_peers = None if not config.fixed_peers else await get_kademlia_peers_from_hosts(config.fixed_peers)
if fixed_peers:
loop.call_later(config.fixed_peer_delay, peer_queue.put_nowait, fixed_peers)
downloader = BlobDownloader(loop, config, blob_manager, peer_queue) downloader = BlobDownloader(loop, config, blob_manager, peer_queue)
try: try:
return await downloader.download_blob(blob_hash) return await downloader.download_blob(blob_hash)

View file

@ -1,6 +1,7 @@
import asyncio import asyncio
import binascii import binascii
import logging import logging
import socket
import typing import typing
from json.decoder import JSONDecodeError from json.decoder import JSONDecodeError
from lbry.blob_exchange.serialization import BlobResponse, BlobRequest, blob_response_types from lbry.blob_exchange.serialization import BlobResponse, BlobRequest, blob_response_types
@ -24,19 +25,19 @@ class BlobServerProtocol(asyncio.Protocol):
self.idle_timeout = idle_timeout self.idle_timeout = idle_timeout
self.transfer_timeout = transfer_timeout self.transfer_timeout = transfer_timeout
self.server_task: typing.Optional[asyncio.Task] = None self.server_task: typing.Optional[asyncio.Task] = None
self.started_listening = asyncio.Event(loop=self.loop) self.started_listening = asyncio.Event()
self.buf = b'' self.buf = b''
self.transport: typing.Optional[asyncio.Transport] = None self.transport: typing.Optional[asyncio.Transport] = None
self.lbrycrd_address = lbrycrd_address self.lbrycrd_address = lbrycrd_address
self.peer_address_and_port: typing.Optional[str] = None self.peer_address_and_port: typing.Optional[str] = None
self.started_transfer = asyncio.Event(loop=self.loop) self.started_transfer = asyncio.Event()
self.transfer_finished = asyncio.Event(loop=self.loop) self.transfer_finished = asyncio.Event()
self.close_on_idle_task: typing.Optional[asyncio.Task] = None self.close_on_idle_task: typing.Optional[asyncio.Task] = None
async def close_on_idle(self): async def close_on_idle(self):
while self.transport: while self.transport:
try: try:
await asyncio.wait_for(self.started_transfer.wait(), self.idle_timeout, loop=self.loop) await asyncio.wait_for(self.started_transfer.wait(), self.idle_timeout)
except asyncio.TimeoutError: except asyncio.TimeoutError:
log.debug("closing idle connection from %s", self.peer_address_and_port) log.debug("closing idle connection from %s", self.peer_address_and_port)
return self.close() return self.close()
@ -100,7 +101,7 @@ class BlobServerProtocol(asyncio.Protocol):
log.debug("send %s to %s:%i", blob_hash, peer_address, peer_port) log.debug("send %s to %s:%i", blob_hash, peer_address, peer_port)
self.started_transfer.set() self.started_transfer.set()
try: try:
sent = await asyncio.wait_for(blob.sendfile(self), self.transfer_timeout, loop=self.loop) sent = await asyncio.wait_for(blob.sendfile(self), self.transfer_timeout)
if sent and sent > 0: if sent and sent > 0:
self.blob_manager.connection_manager.sent_data(self.peer_address_and_port, sent) self.blob_manager.connection_manager.sent_data(self.peer_address_and_port, sent)
log.info("sent %s (%i bytes) to %s:%i", blob_hash, sent, peer_address, peer_port) log.info("sent %s (%i bytes) to %s:%i", blob_hash, sent, peer_address, peer_port)
@ -137,7 +138,7 @@ class BlobServerProtocol(asyncio.Protocol):
try: try:
request = BlobRequest.deserialize(self.buf + data) request = BlobRequest.deserialize(self.buf + data)
self.buf = remainder self.buf = remainder
except JSONDecodeError: except (UnicodeDecodeError, JSONDecodeError):
log.error("request from %s is not valid json (%i bytes): %s", self.peer_address_and_port, log.error("request from %s is not valid json (%i bytes): %s", self.peer_address_and_port,
len(self.buf + data), '' if not data else binascii.hexlify(self.buf + data).decode()) len(self.buf + data), '' if not data else binascii.hexlify(self.buf + data).decode())
self.close() self.close()
@ -156,7 +157,7 @@ class BlobServer:
self.loop = loop self.loop = loop
self.blob_manager = blob_manager self.blob_manager = blob_manager
self.server_task: typing.Optional[asyncio.Task] = None self.server_task: typing.Optional[asyncio.Task] = None
self.started_listening = asyncio.Event(loop=self.loop) self.started_listening = asyncio.Event()
self.lbrycrd_address = lbrycrd_address self.lbrycrd_address = lbrycrd_address
self.idle_timeout = idle_timeout self.idle_timeout = idle_timeout
self.transfer_timeout = transfer_timeout self.transfer_timeout = transfer_timeout
@ -167,6 +168,13 @@ class BlobServer:
raise Exception("already running") raise Exception("already running")
async def _start_server(): async def _start_server():
# checking if the port is in use
# thx https://stackoverflow.com/a/52872579
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
if s.connect_ex(('localhost', port)) == 0:
# the port is already in use!
log.error("Failed to bind TCP %s:%d", interface, port)
server = await self.loop.create_server( server = await self.loop.create_server(
lambda: self.server_protocol_class(self.loop, self.blob_manager, self.lbrycrd_address, lambda: self.server_protocol_class(self.loop, self.blob_manager, self.lbrycrd_address,
self.idle_timeout, self.transfer_timeout), self.idle_timeout, self.transfer_timeout),

View file

@ -1,13 +1,12 @@
import os import os
import re import re
import sys import sys
import typing
import logging import logging
from typing import List, Dict, Tuple, Union, TypeVar, Generic, Optional
from argparse import ArgumentParser from argparse import ArgumentParser
from contextlib import contextmanager from contextlib import contextmanager
import yaml
from appdirs import user_data_dir, user_config_dir from appdirs import user_data_dir, user_config_dir
import yaml
from lbry.error import InvalidCurrencyError from lbry.error import InvalidCurrencyError
from lbry.dht import constants from lbry.dht import constants
from lbry.wallet.coinselection import STRATEGIES from lbry.wallet.coinselection import STRATEGIES
@ -16,7 +15,7 @@ log = logging.getLogger(__name__)
NOT_SET = type('NOT_SET', (object,), {}) # pylint: disable=invalid-name NOT_SET = type('NOT_SET', (object,), {}) # pylint: disable=invalid-name
T = typing.TypeVar('T') T = TypeVar('T')
CURRENCIES = { CURRENCIES = {
'BTC': {'type': 'crypto'}, 'BTC': {'type': 'crypto'},
@ -25,11 +24,11 @@ CURRENCIES = {
} }
class Setting(typing.Generic[T]): class Setting(Generic[T]):
def __init__(self, doc: str, default: typing.Optional[T] = None, def __init__(self, doc: str, default: Optional[T] = None,
previous_names: typing.Optional[typing.List[str]] = None, previous_names: Optional[List[str]] = None,
metavar: typing.Optional[str] = None): metavar: Optional[str] = None):
self.doc = doc self.doc = doc
self.default = default self.default = default
self.previous_names = previous_names or [] self.previous_names = previous_names or []
@ -46,7 +45,7 @@ class Setting(typing.Generic[T]):
def no_cli_name(self): def no_cli_name(self):
return f"--no-{self.name.replace('_', '-')}" return f"--no-{self.name.replace('_', '-')}"
def __get__(self, obj: typing.Optional['BaseConfig'], owner) -> T: def __get__(self, obj: Optional['BaseConfig'], owner) -> T:
if obj is None: if obj is None:
return self return self
for location in obj.search_order: for location in obj.search_order:
@ -54,7 +53,7 @@ class Setting(typing.Generic[T]):
return location[self.name] return location[self.name]
return self.default return self.default
def __set__(self, obj: 'BaseConfig', val: typing.Union[T, NOT_SET]): def __set__(self, obj: 'BaseConfig', val: Union[T, NOT_SET]):
if val == NOT_SET: if val == NOT_SET:
for location in obj.modify_order: for location in obj.modify_order:
if self.name in location: if self.name in location:
@ -64,6 +63,18 @@ class Setting(typing.Generic[T]):
for location in obj.modify_order: for location in obj.modify_order:
location[self.name] = val location[self.name] = val
def is_set(self, obj: 'BaseConfig') -> bool:
for location in obj.search_order:
if self.name in location:
return True
return False
def is_set_to_default(self, obj: 'BaseConfig') -> bool:
for location in obj.search_order:
if self.name in location:
return location[self.name] == self.default
return False
def validate(self, value): def validate(self, value):
raise NotImplementedError() raise NotImplementedError()
@ -88,7 +99,7 @@ class String(Setting[str]):
f"Setting '{self.name}' must be a string." f"Setting '{self.name}' must be a string."
# TODO: removes this after pylint starts to understand generics # TODO: removes this after pylint starts to understand generics
def __get__(self, obj: typing.Optional['BaseConfig'], owner) -> str: # pylint: disable=useless-super-delegation def __get__(self, obj: Optional['BaseConfig'], owner) -> str: # pylint: disable=useless-super-delegation
return super().__get__(obj, owner) return super().__get__(obj, owner)
@ -192,7 +203,7 @@ class MaxKeyFee(Setting[dict]):
) )
parser.add_argument( parser.add_argument(
self.no_cli_name, self.no_cli_name,
help=f"Disable maximum key fee check.", help="Disable maximum key fee check.",
dest=self.name, dest=self.name,
const=None, const=None,
action="store_const", action="store_const",
@ -201,7 +212,7 @@ class MaxKeyFee(Setting[dict]):
class StringChoice(String): class StringChoice(String):
def __init__(self, doc: str, valid_values: typing.List[str], default: str, *args, **kwargs): def __init__(self, doc: str, valid_values: List[str], default: str, *args, **kwargs):
super().__init__(doc, default, *args, **kwargs) super().__init__(doc, default, *args, **kwargs)
if not valid_values: if not valid_values:
raise ValueError("No valid values provided") raise ValueError("No valid values provided")
@ -274,17 +285,95 @@ class Strings(ListSetting):
f"'{self.name}' must be a string." f"'{self.name}' must be a string."
class KnownHubsList:
def __init__(self, config: 'Config' = None, file_name: str = 'known_hubs.yml'):
self.file_name = file_name
self.path = os.path.join(config.wallet_dir, self.file_name) if config else None
self.hubs: Dict[Tuple[str, int], Dict] = {}
if self.exists:
self.load()
@property
def exists(self):
return self.path and os.path.exists(self.path)
@property
def serialized(self) -> Dict[str, Dict]:
return {f"{host}:{port}": details for (host, port), details in self.hubs.items()}
def filter(self, match_none=False, **kwargs):
if not kwargs:
return self.hubs
result = {}
for hub, details in self.hubs.items():
for key, constraint in kwargs.items():
value = details.get(key)
if value == constraint or (match_none and value is None):
result[hub] = details
break
return result
def load(self):
if self.path:
with open(self.path, 'r') as known_hubs_file:
raw = known_hubs_file.read()
for hub, details in yaml.safe_load(raw).items():
self.set(hub, details)
def save(self):
if self.path:
with open(self.path, 'w') as known_hubs_file:
known_hubs_file.write(yaml.safe_dump(self.serialized, default_flow_style=False))
def set(self, hub: str, details: Dict):
if hub and hub.count(':') == 1:
host, port = hub.split(':')
hub_parts = (host, int(port))
if hub_parts not in self.hubs:
self.hubs[hub_parts] = details
return hub
def add_hubs(self, hubs: List[str]):
added = False
for hub in hubs:
if self.set(hub, {}) is not None:
added = True
return added
def items(self):
return self.hubs.items()
def __bool__(self):
return len(self) > 0
def __len__(self):
return self.hubs.__len__()
def __iter__(self):
return iter(self.hubs)
class EnvironmentAccess: class EnvironmentAccess:
PREFIX = 'LBRY_' PREFIX = 'LBRY_'
def __init__(self, environ: dict): def __init__(self, config: 'BaseConfig', environ: dict):
self.environ = environ self.configuration = config
self.data = {}
if environ:
self.load(environ)
def load(self, environ):
for setting in self.configuration.get_settings():
value = environ.get(f'{self.PREFIX}{setting.name.upper()}', NOT_SET)
if value != NOT_SET and not (isinstance(setting, ListSetting) and value is None):
self.data[setting.name] = setting.deserialize(value)
def __contains__(self, item: str): def __contains__(self, item: str):
return f'{self.PREFIX}{item.upper()}' in self.environ return item in self.data
def __getitem__(self, item: str): def __getitem__(self, item: str):
return self.environ[f'{self.PREFIX}{item.upper()}'] return self.data[item]
class ArgumentAccess: class ArgumentAccess:
@ -325,7 +414,7 @@ class ConfigFileAccess:
cls = type(self.configuration) cls = type(self.configuration)
with open(self.path, 'r') as config_file: with open(self.path, 'r') as config_file:
raw = config_file.read() raw = config_file.read()
serialized = yaml.load(raw) or {} serialized = yaml.safe_load(raw) or {}
for key, value in serialized.items(): for key, value in serialized.items():
attr = getattr(cls, key, None) attr = getattr(cls, key, None)
if attr is None: if attr is None:
@ -369,7 +458,7 @@ class ConfigFileAccess:
del self.data[key] del self.data[key]
TBC = typing.TypeVar('TBC', bound='BaseConfig') TBC = TypeVar('TBC', bound='BaseConfig')
class BaseConfig: class BaseConfig:
@ -443,7 +532,7 @@ class BaseConfig:
self.arguments = ArgumentAccess(self, args) self.arguments = ArgumentAccess(self, args)
def set_environment(self, environ=None): def set_environment(self, environ=None):
self.environment = EnvironmentAccess(environ or os.environ) self.environment = EnvironmentAccess(self, environ or os.environ)
def set_persisted(self, config_file_path=None): def set_persisted(self, config_file_path=None):
if config_file_path is None: if config_file_path is None:
@ -464,15 +553,22 @@ class BaseConfig:
class TranscodeConfig(BaseConfig): class TranscodeConfig(BaseConfig):
ffmpeg_folder = String('The path to ffmpeg and ffprobe', '') ffmpeg_path = String('A list of places to check for ffmpeg and ffprobe. '
f'$data_dir/ffmpeg/bin and $PATH are checked afterward. Separator: {os.pathsep}',
'', previous_names=['ffmpeg_folder'])
video_encoder = String('FFmpeg codec and parameters for the video encoding. ' video_encoder = String('FFmpeg codec and parameters for the video encoding. '
'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental', 'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental',
'libx264 -crf 18 -vf "format=yuv420p"') 'libx264 -crf 24 -preset faster -pix_fmt yuv420p')
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 5_000_000)
video_scaler = String('FFmpeg scaling parameters for reducing bitrate. '
'Example: -vf "scale=-2:720,fps=24" -maxrate 5M -bufsize 3M',
r'-vf "scale=if(gte(iw\,ih)\,min(1920\,iw)\,-2):if(lt(iw\,ih)\,min(1920\,ih)\,-2)" '
r'-maxrate 5500K -bufsize 5000K')
audio_encoder = String('FFmpeg codec and parameters for the audio encoding. ' audio_encoder = String('FFmpeg codec and parameters for the audio encoding. '
'Example: libopus -b:a 128k', 'Example: libopus -b:a 128k',
'aac -b:a 192k') 'aac -b:a 160k')
volume_filter = String('FFmpeg filter for audio normalization.', '-af loudnorm') volume_filter = String('FFmpeg filter for audio normalization. Exmple: -af loudnorm', '')
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', '240') volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', 240)
class CLIConfig(TranscodeConfig): class CLIConfig(TranscodeConfig):
@ -493,6 +589,9 @@ class CLIConfig(TranscodeConfig):
class Config(CLIConfig): class Config(CLIConfig):
jurisdiction = String("Limit interactions to wallet server in this jurisdiction.")
# directories # directories
data_dir = Path("Directory path to store blobs.", metavar='DIR') data_dir = Path("Directory path to store blobs.", metavar='DIR')
download_dir = Path( download_dir = Path(
@ -514,7 +613,7 @@ class Config(CLIConfig):
"ports or have firewall rules you likely want to disable this.", True "ports or have firewall rules you likely want to disable this.", True
) )
udp_port = Integer("UDP port for communicating on the LBRY DHT", 4444, previous_names=['dht_node_port']) udp_port = Integer("UDP port for communicating on the LBRY DHT", 4444, previous_names=['dht_node_port'])
tcp_port = Integer("TCP port to listen for incoming blob requests", 3333, previous_names=['peer_port']) tcp_port = Integer("TCP port to listen for incoming blob requests", 4444, previous_names=['peer_port'])
prometheus_port = Integer("Port to expose prometheus metrics (off by default)", 0) prometheus_port = Integer("Port to expose prometheus metrics (off by default)", 0)
network_interface = String("Interface to use for the DHT and blob exchange", '0.0.0.0') network_interface = String("Interface to use for the DHT and blob exchange", '0.0.0.0')
@ -523,17 +622,24 @@ class Config(CLIConfig):
"Routing table bucket index below which we always split the bucket if given a new key to add to it and " "Routing table bucket index below which we always split the bucket if given a new key to add to it and "
"the bucket is full. As this value is raised the depth of the routing table (and number of peers in it) " "the bucket is full. As this value is raised the depth of the routing table (and number of peers in it) "
"will increase. This setting is used by seed nodes, you probably don't want to change it during normal " "will increase. This setting is used by seed nodes, you probably don't want to change it during normal "
"use.", 1 "use.", 2
)
is_bootstrap_node = Toggle(
"When running as a bootstrap node, disable all logic related to balancing the routing table, so we can "
"add as many peers as possible and better help first-runs.", False
) )
# protocol timeouts # protocol timeouts
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0) download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0) blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
hub_timeout = Float("Timeout when making a hub request", 30.0)
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0) peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0)
node_rpc_timeout = Float("Timeout when making a DHT request", constants.RPC_TIMEOUT) node_rpc_timeout = Float("Timeout when making a DHT request", constants.RPC_TIMEOUT)
# blob announcement and download # blob announcement and download
save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True) save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True)
network_storage_limit = Integer("Disk space in MB to be allocated for helping the P2P network. 0 = disable", 0)
blob_storage_limit = Integer("Disk space in MB to be allocated for blob storage. 0 = no limit", 0)
blob_lru_cache_size = Integer( blob_lru_cache_size = Integer(
"LRU cache size for decrypted downloaded blobs used to minimize re-downloading the same blobs when " "LRU cache size for decrypted downloaded blobs used to minimize re-downloading the same blobs when "
"replying to a range request. Set to 0 to disable.", 32 "replying to a range request. Set to 0 to disable.", 32
@ -550,6 +656,7 @@ class Config(CLIConfig):
"Maximum number of peers to connect to while downloading a blob", 4, "Maximum number of peers to connect to while downloading a blob", 4,
previous_names=['max_connections_per_stream'] previous_names=['max_connections_per_stream']
) )
concurrent_hub_requests = Integer("Maximum number of concurrent hub requests", 32)
fixed_peer_delay = Float( fixed_peer_delay = Float(
"Amount of seconds before adding the reflector servers as potential peers to download from in case dht" "Amount of seconds before adding the reflector servers as potential peers to download from in case dht"
"peers are not found or are slow", 2.0 "peers are not found or are slow", 2.0
@ -558,6 +665,7 @@ class Config(CLIConfig):
"Don't download streams with fees exceeding this amount. When set to " "Don't download streams with fees exceeding this amount. When set to "
"null, the amount is unbounded.", {'currency': 'USD', 'amount': 50.0} "null, the amount is unbounded.", {'currency': 'USD', 'amount': 50.0}
) )
max_wallet_server_fee = String("Maximum daily LBC amount allowed as payment for wallet servers.", "0.0")
# reflector settings # reflector settings
reflect_streams = Toggle( reflect_streams = Toggle(
@ -569,42 +677,62 @@ class Config(CLIConfig):
) )
# servers # servers
reflector_servers = Servers("Reflector re-hosting servers", [ reflector_servers = Servers("Reflector re-hosting servers for mirroring publishes", [
('reflector.lbry.com', 5566) ('reflector.lbry.com', 5566)
]) ])
fixed_peers = Servers("Fixed peers to fall back to if none are found on P2P for a blob", [
('cdn.reflector.lbry.com', 5567)
])
tracker_servers = Servers("BitTorrent-compatible (BEP15) UDP trackers for helping P2P discovery", [
('tracker.lbry.com', 9252),
('tracker.lbry.grin.io', 9252),
('tracker.lbry.pigg.es', 9252),
('tracker.lizard.technology', 9252),
('s1.lbry.network', 9252),
])
lbryum_servers = Servers("SPV wallet servers", [ lbryum_servers = Servers("SPV wallet servers", [
('spv1.lbry.com', 50001), ('spv11.lbry.com', 50001),
('spv2.lbry.com', 50001), ('spv12.lbry.com', 50001),
('spv3.lbry.com', 50001), ('spv13.lbry.com', 50001),
('spv4.lbry.com', 50001), ('spv14.lbry.com', 50001),
('spv5.lbry.com', 50001), ('spv15.lbry.com', 50001),
('spv6.lbry.com', 50001), ('spv16.lbry.com', 50001),
('spv7.lbry.com', 50001), ('spv17.lbry.com', 50001),
('spv8.lbry.com', 50001), ('spv18.lbry.com', 50001),
('spv9.lbry.com', 50001), ('spv19.lbry.com', 50001),
('hub.lbry.grin.io', 50001),
('hub.lizard.technology', 50001),
('s1.lbry.network', 50001),
]) ])
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [ known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
('dht.lbry.grin.io', 4444), # Grin
('dht.lbry.madiator.com', 4444), # Madiator
('dht.lbry.pigg.es', 4444), # Pigges
('lbrynet1.lbry.com', 4444), # US EAST ('lbrynet1.lbry.com', 4444), # US EAST
('lbrynet2.lbry.com', 4444), # US WEST ('lbrynet2.lbry.com', 4444), # US WEST
('lbrynet3.lbry.com', 4444), # EU ('lbrynet3.lbry.com', 4444), # EU
('lbrynet4.lbry.com', 4444) # ASIA ('lbrynet4.lbry.com', 4444), # ASIA
('dht.lizard.technology', 4444), # Jack
('s2.lbry.network', 4444),
]) ])
comment_server = String("Comment server API URL", "https://comments.lbry.com/api")
# blockchain # blockchain
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main') blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
cache_time = Integer("Time to cache resolved claims", 150) # TODO: use this
# daemon # daemon
save_files = Toggle("Save downloaded files when calling `get` by default", True) save_files = Toggle("Save downloaded files when calling `get` by default", False)
components_to_skip = Strings("components which will be skipped during start-up of daemon", []) components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
share_usage_data = Toggle( share_usage_data = Toggle(
"Whether to share usage stats and diagnostic info with LBRY.", True, "Whether to share usage stats and diagnostic info with LBRY.", False,
previous_names=['upload_log', 'upload_log', 'share_debug_info'] previous_names=['upload_log', 'upload_log', 'share_debug_info']
) )
track_bandwidth = Toggle("Track bandwidth usage", True) track_bandwidth = Toggle("Track bandwidth usage", True)
allowed_origin = String(
"Allowed `Origin` header value for API request (sent by browser), use * to allow "
"all hosts; default is to only allow API requests with no `Origin` value.", "")
# media server # media server
streaming_server = String('Host name and port to serve streaming media over range requests', streaming_server = String('Host name and port to serve streaming media over range requests',
@ -614,7 +742,14 @@ class Config(CLIConfig):
coin_selection_strategy = StringChoice( coin_selection_strategy = StringChoice(
"Strategy to use when selecting UTXOs for a transaction", "Strategy to use when selecting UTXOs for a transaction",
STRATEGIES, "standard") STRATEGIES, "prefer_confirmed"
)
transaction_cache_size = Integer("Transaction cache size", 2 ** 17)
save_resolved_claims = Toggle(
"Save content claims to the database when they are resolved to keep file_list up to date, "
"only disable this if file_x commands are not needed", True
)
@property @property
def streaming_host(self): def streaming_host(self):
@ -627,6 +762,7 @@ class Config(CLIConfig):
def __init__(self, **kwargs): def __init__(self, **kwargs):
super().__init__(**kwargs) super().__init__(**kwargs)
self.set_default_paths() self.set_default_paths()
self.known_hubs = KnownHubsList(self)
def set_default_paths(self): def set_default_paths(self):
if 'darwin' in sys.platform.lower(): if 'darwin' in sys.platform.lower():
@ -648,7 +784,7 @@ class Config(CLIConfig):
return os.path.join(self.data_dir, 'lbrynet.log') return os.path.join(self.data_dir, 'lbrynet.log')
def get_windows_directories() -> typing.Tuple[str, str, str]: def get_windows_directories() -> Tuple[str, str, str]:
from lbry.winpaths import get_path, FOLDERID, UserHandle, \ from lbry.winpaths import get_path, FOLDERID, UserHandle, \
PathNotFoundException # pylint: disable=import-outside-toplevel PathNotFoundException # pylint: disable=import-outside-toplevel
@ -670,18 +806,19 @@ def get_windows_directories() -> typing.Tuple[str, str, str]:
return data_dir, lbryum_dir, download_dir return data_dir, lbryum_dir, download_dir
def get_darwin_directories() -> typing.Tuple[str, str, str]: def get_darwin_directories() -> Tuple[str, str, str]:
data_dir = user_data_dir('LBRY') data_dir = user_data_dir('LBRY')
lbryum_dir = os.path.expanduser('~/.lbryum') lbryum_dir = os.path.expanduser('~/.lbryum')
download_dir = os.path.expanduser('~/Downloads') download_dir = os.path.expanduser('~/Downloads')
return data_dir, lbryum_dir, download_dir return data_dir, lbryum_dir, download_dir
def get_linux_directories() -> typing.Tuple[str, str, str]: def get_linux_directories() -> Tuple[str, str, str]:
try: try:
with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg: with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg:
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()).group(1) down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read())
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir) if down_dir:
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir.group(1))
download_dir = re.sub('\"', '', down_dir) download_dir = re.sub('\"', '', down_dir)
except OSError: except OSError:
download_dir = os.getenv('XDG_DOWNLOAD_DIR') download_dir = os.getenv('XDG_DOWNLOAD_DIR')

View file

@ -67,7 +67,7 @@ class ConnectionManager:
while True: while True:
last = time.perf_counter() last = time.perf_counter()
await asyncio.sleep(0.1, loop=self.loop) await asyncio.sleep(0.1)
self._status['incoming_bps'].clear() self._status['incoming_bps'].clear()
self._status['outgoing_bps'].clear() self._status['outgoing_bps'].clear()
now = time.perf_counter() now = time.perf_counter()

View file

@ -1,6 +1,9 @@
import asyncio import asyncio
import typing import typing
import logging import logging
from prometheus_client import Counter, Gauge
if typing.TYPE_CHECKING: if typing.TYPE_CHECKING:
from lbry.dht.node import Node from lbry.dht.node import Node
from lbry.extras.daemon.storage import SQLiteStorage from lbry.extras.daemon.storage import SQLiteStorage
@ -9,45 +12,59 @@ log = logging.getLogger(__name__)
class BlobAnnouncer: class BlobAnnouncer:
announcements_sent_metric = Counter(
"announcements_sent", "Number of announcements sent and their respective status.", namespace="dht_node",
labelnames=("peers", "error"),
)
announcement_queue_size_metric = Gauge(
"announcement_queue_size", "Number of hashes waiting to be announced.", namespace="dht_node",
labelnames=("scope",)
)
def __init__(self, loop: asyncio.AbstractEventLoop, node: 'Node', storage: 'SQLiteStorage'): def __init__(self, loop: asyncio.AbstractEventLoop, node: 'Node', storage: 'SQLiteStorage'):
self.loop = loop self.loop = loop
self.node = node self.node = node
self.storage = storage self.storage = storage
self.announce_task: asyncio.Task = None self.announce_task: asyncio.Task = None
self.announce_queue: typing.List[str] = [] self.announce_queue: typing.List[str] = []
self._done = asyncio.Event()
self.announced = set()
async def _submit_announcement(self, blob_hash): async def _run_consumer(self):
while self.announce_queue:
try: try:
blob_hash = self.announce_queue.pop()
peers = len(await self.node.announce_blob(blob_hash)) peers = len(await self.node.announce_blob(blob_hash))
self.announcements_sent_metric.labels(peers=peers, error=False).inc()
if peers > 4: if peers > 4:
return blob_hash self.announced.add(blob_hash)
else: else:
log.debug("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers) log.debug("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
except Exception as err: except Exception as err:
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8 self.announcements_sent_metric.labels(peers=0, error=True).inc()
raise err
log.warning("error announcing %s: %s", blob_hash[:8], str(err)) log.warning("error announcing %s: %s", blob_hash[:8], str(err))
async def _announce(self, batch_size: typing.Optional[int] = 10): async def _announce(self, batch_size: typing.Optional[int] = 10):
while batch_size: while batch_size:
if not self.node.joined.is_set(): if not self.node.joined.is_set():
await self.node.joined.wait() await self.node.joined.wait()
await asyncio.sleep(60, loop=self.loop) await asyncio.sleep(60)
if not self.node.protocol.routing_table.get_peers(): if not self.node.protocol.routing_table.get_peers():
log.warning("No peers in DHT, announce round skipped") log.warning("No peers in DHT, announce round skipped")
continue continue
self.announce_queue.extend(await self.storage.get_blobs_to_announce()) self.announce_queue.extend(await self.storage.get_blobs_to_announce())
self.announcement_queue_size_metric.labels(scope="global").set(len(self.announce_queue))
log.debug("announcer task wake up, %d blobs to announce", len(self.announce_queue)) log.debug("announcer task wake up, %d blobs to announce", len(self.announce_queue))
while len(self.announce_queue) > 0: while len(self.announce_queue) > 0:
log.info("%i blobs to announce", len(self.announce_queue)) log.info("%i blobs to announce", len(self.announce_queue))
announced = await asyncio.gather(*[ await asyncio.gather(*[self._run_consumer() for _ in range(batch_size)])
self._submit_announcement( announced = list(filter(None, self.announced))
self.announce_queue.pop()) for _ in range(batch_size) if self.announce_queue
], loop=self.loop)
announced = list(filter(None, announced))
if announced: if announced:
await self.storage.update_last_announced_blobs(announced) await self.storage.update_last_announced_blobs(announced)
log.info("announced %i blobs", len(announced)) log.info("announced %i blobs", len(announced))
self.announced.clear()
self._done.set()
self._done.clear()
def start(self, batch_size: typing.Optional[int] = 10): def start(self, batch_size: typing.Optional[int] = 10):
assert not self.announce_task or self.announce_task.done(), "already running" assert not self.announce_task or self.announce_task.done(), "already running"
@ -56,3 +73,6 @@ class BlobAnnouncer:
def stop(self): def stop(self):
if self.announce_task and not self.announce_task.done(): if self.announce_task and not self.announce_task.done():
self.announce_task.cancel() self.announce_task.cancel()
def wait(self):
return self._done.wait()

View file

@ -20,7 +20,6 @@ MAYBE_PING_DELAY = 300 # 5 minutes
CHECK_REFRESH_INTERVAL = REFRESH_INTERVAL / 5 CHECK_REFRESH_INTERVAL = REFRESH_INTERVAL / 5
RPC_ID_LENGTH = 20 RPC_ID_LENGTH = 20
PROTOCOL_VERSION = 1 PROTOCOL_VERSION = 1
BOTTOM_OUT_LIMIT = 3
MSG_SIZE_LIMIT = 1400 MSG_SIZE_LIMIT = 1400

View file

@ -1,9 +1,11 @@
import logging import logging
import asyncio import asyncio
import typing import typing
import binascii
import socket import socket
from lbry.utils import resolve_host
from prometheus_client import Gauge
from lbry.utils import aclosing, resolve_host
from lbry.dht import constants from lbry.dht import constants
from lbry.dht.peer import make_kademlia_peer from lbry.dht.peer import make_kademlia_peer
from lbry.dht.protocol.distance import Distance from lbry.dht.protocol.distance import Distance
@ -18,20 +20,32 @@ log = logging.getLogger(__name__)
class Node: class Node:
storing_peers_metric = Gauge(
"storing_peers", "Number of peers storing blobs announced to this node", namespace="dht_node",
labelnames=("scope",),
)
stored_blob_with_x_bytes_colliding = Gauge(
"stored_blobs_x_bytes_colliding", "Number of blobs with at least X bytes colliding with this node id prefix",
namespace="dht_node", labelnames=("amount",)
)
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int, def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int,
internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: float = constants.RPC_TIMEOUT, internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: float = constants.RPC_TIMEOUT,
split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX, split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX, is_bootstrap_node: bool = False,
storage: typing.Optional['SQLiteStorage'] = None): storage: typing.Optional['SQLiteStorage'] = None):
self.loop = loop self.loop = loop
self.internal_udp_port = internal_udp_port self.internal_udp_port = internal_udp_port
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port, rpc_timeout, self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port, rpc_timeout,
split_buckets_under_index) split_buckets_under_index, is_bootstrap_node)
self.listening_port: asyncio.DatagramTransport = None self.listening_port: asyncio.DatagramTransport = None
self.joined = asyncio.Event(loop=self.loop) self.joined = asyncio.Event()
self._join_task: asyncio.Task = None self._join_task: asyncio.Task = None
self._refresh_task: asyncio.Task = None self._refresh_task: asyncio.Task = None
self._storage = storage self._storage = storage
@property
def stored_blob_hashes(self):
return self.protocol.data_store.keys()
async def refresh_node(self, force_once=False): async def refresh_node(self, force_once=False):
while True: while True:
# remove peers with expired blob announcements from the datastore # remove peers with expired blob announcements from the datastore
@ -41,17 +55,21 @@ class Node:
# add all peers in the routing table # add all peers in the routing table
total_peers.extend(self.protocol.routing_table.get_peers()) total_peers.extend(self.protocol.routing_table.get_peers())
# add all the peers who have announced blobs to us # add all the peers who have announced blobs to us
total_peers.extend(self.protocol.data_store.get_storing_contacts()) storing_peers = self.protocol.data_store.get_storing_contacts()
self.storing_peers_metric.labels("global").set(len(storing_peers))
total_peers.extend(storing_peers)
counts = {0: 0, 1: 0, 2: 0}
node_id = self.protocol.node_id
for blob_hash in self.protocol.data_store.keys():
bytes_colliding = 0 if blob_hash[0] != node_id[0] else 2 if blob_hash[1] == node_id[1] else 1
counts[bytes_colliding] += 1
self.stored_blob_with_x_bytes_colliding.labels(amount=0).set(counts[0])
self.stored_blob_with_x_bytes_colliding.labels(amount=1).set(counts[1])
self.stored_blob_with_x_bytes_colliding.labels(amount=2).set(counts[2])
# get ids falling in the midpoint of each bucket that hasn't been recently updated # get ids falling in the midpoint of each bucket that hasn't been recently updated
node_ids = self.protocol.routing_table.get_refresh_list(0, True) node_ids = self.protocol.routing_table.get_refresh_list(0, True)
# if we have 3 or fewer populated buckets get two random ids in the range of each to try and
# populate/split the buckets further
buckets_with_contacts = self.protocol.routing_table.buckets_with_contacts()
if buckets_with_contacts <= 3:
for i in range(buckets_with_contacts):
node_ids.append(self.protocol.routing_table.random_id_in_bucket_range(i))
node_ids.append(self.protocol.routing_table.random_id_in_bucket_range(i))
if self.protocol.routing_table.get_peers(): if self.protocol.routing_table.get_peers():
# if we have node ids to look up, perform the iterative search until we have k results # if we have node ids to look up, perform the iterative search until we have k results
@ -61,7 +79,7 @@ class Node:
else: else:
if force_once: if force_once:
break break
fut = asyncio.Future(loop=self.loop) fut = asyncio.Future()
self.loop.call_later(constants.REFRESH_INTERVAL // 4, fut.set_result, None) self.loop.call_later(constants.REFRESH_INTERVAL // 4, fut.set_result, None)
await fut await fut
continue continue
@ -75,12 +93,12 @@ class Node:
if force_once: if force_once:
break break
fut = asyncio.Future(loop=self.loop) fut = asyncio.Future()
self.loop.call_later(constants.REFRESH_INTERVAL, fut.set_result, None) self.loop.call_later(constants.REFRESH_INTERVAL, fut.set_result, None)
await fut await fut
async def announce_blob(self, blob_hash: str) -> typing.List[bytes]: async def announce_blob(self, blob_hash: str) -> typing.List[bytes]:
hash_value = binascii.unhexlify(blob_hash.encode()) hash_value = bytes.fromhex(blob_hash)
assert len(hash_value) == constants.HASH_LENGTH assert len(hash_value) == constants.HASH_LENGTH
peers = await self.peer_search(hash_value) peers = await self.peer_search(hash_value)
@ -90,12 +108,12 @@ class Node:
for peer in peers: for peer in peers:
log.debug("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port) log.debug("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
stored_to_tup = await asyncio.gather( stored_to_tup = await asyncio.gather(
*(self.protocol.store_to_peer(hash_value, peer) for peer in peers), loop=self.loop *(self.protocol.store_to_peer(hash_value, peer) for peer in peers)
) )
stored_to = [node_id for node_id, contacted in stored_to_tup if contacted] stored_to = [node_id for node_id, contacted in stored_to_tup if contacted]
if stored_to: if stored_to:
log.debug( log.debug(
"Stored %s to %i of %i attempted peers", binascii.hexlify(hash_value).decode()[:8], "Stored %s to %i of %i attempted peers", hash_value.hex()[:8],
len(stored_to), len(peers) len(stored_to), len(peers)
) )
else: else:
@ -164,38 +182,35 @@ class Node:
for address, udp_port in known_node_urls or [] for address, udp_port in known_node_urls or []
])) ]))
except socket.gaierror: except socket.gaierror:
await asyncio.sleep(30, loop=self.loop) await asyncio.sleep(30)
continue continue
self.protocol.peer_manager.reset() self.protocol.peer_manager.reset()
self.protocol.ping_queue.enqueue_maybe_ping(*seed_peers, delay=0.0) self.protocol.ping_queue.enqueue_maybe_ping(*seed_peers, delay=0.0)
await self.peer_search(self.protocol.node_id, shortlist=seed_peers, count=32) await self.peer_search(self.protocol.node_id, shortlist=seed_peers, count=32)
await asyncio.sleep(1, loop=self.loop) await asyncio.sleep(1)
def start(self, interface: str, known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None): def start(self, interface: str, known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
self._join_task = self.loop.create_task(self.join_network(interface, known_node_urls)) self._join_task = self.loop.create_task(self.join_network(interface, known_node_urls))
def get_iterative_node_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None, def get_iterative_node_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
bottom_out_limit: int = constants.BOTTOM_OUT_LIMIT,
max_results: int = constants.K) -> IterativeNodeFinder: max_results: int = constants.K) -> IterativeNodeFinder:
shortlist = shortlist or self.protocol.routing_table.find_close_peers(key)
return IterativeNodeFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol, return IterativeNodeFinder(self.loop, self.protocol, key, max_results, shortlist)
key, bottom_out_limit, max_results, None, shortlist)
def get_iterative_value_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None, def get_iterative_value_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
bottom_out_limit: int = 40,
max_results: int = -1) -> IterativeValueFinder: max_results: int = -1) -> IterativeValueFinder:
shortlist = shortlist or self.protocol.routing_table.find_close_peers(key)
return IterativeValueFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol, return IterativeValueFinder(self.loop, self.protocol, key, max_results, shortlist)
key, bottom_out_limit, max_results, None, shortlist)
async def peer_search(self, node_id: bytes, count=constants.K, max_results=constants.K * 2, async def peer_search(self, node_id: bytes, count=constants.K, max_results=constants.K * 2,
bottom_out_limit=20, shortlist: typing.Optional[typing.List['KademliaPeer']] = None shortlist: typing.Optional[typing.List['KademliaPeer']] = None
) -> typing.List['KademliaPeer']: ) -> typing.List['KademliaPeer']:
peers = [] peers = []
async for iteration_peers in self.get_iterative_node_finder( async with aclosing(self.get_iterative_node_finder(
node_id, shortlist=shortlist, bottom_out_limit=bottom_out_limit, max_results=max_results): node_id, shortlist=shortlist, max_results=max_results)) as node_finder:
async for iteration_peers in node_finder:
peers.extend(iteration_peers) peers.extend(iteration_peers)
distance = Distance(node_id) distance = Distance(node_id)
peers.sort(key=lambda peer: distance(peer.node_id)) peers.sort(key=lambda peer: distance(peer.node_id))
@ -222,8 +237,8 @@ class Node:
# prioritize peers who reply to a dht ping first # prioritize peers who reply to a dht ping first
# this minimizes attempting to make tcp connections that won't work later to dead or unreachable peers # this minimizes attempting to make tcp connections that won't work later to dead or unreachable peers
async with aclosing(self.get_iterative_value_finder(bytes.fromhex(blob_hash))) as value_finder:
async for results in self.get_iterative_value_finder(binascii.unhexlify(blob_hash.encode())): async for results in value_finder:
to_put = [] to_put = []
for peer in results: for peer in results:
if peer.address == self.protocol.external_ip and self.protocol.peer_port == peer.tcp_port: if peer.address == self.protocol.external_ip and self.protocol.peer_port == peer.tcp_port:
@ -256,5 +271,12 @@ class Node:
def accumulate_peers(self, search_queue: asyncio.Queue, def accumulate_peers(self, search_queue: asyncio.Queue,
peer_queue: typing.Optional[asyncio.Queue] = None peer_queue: typing.Optional[asyncio.Queue] = None
) -> typing.Tuple[asyncio.Queue, asyncio.Task]: ) -> typing.Tuple[asyncio.Queue, asyncio.Task]:
queue = peer_queue or asyncio.Queue(loop=self.loop) queue = peer_queue or asyncio.Queue()
return queue, self.loop.create_task(self._accumulate_peers_for_value(search_queue, queue)) return queue, self.loop.create_task(self._accumulate_peers_for_value(search_queue, queue))
async def get_kademlia_peers_from_hosts(peer_list: typing.List[typing.Tuple[str, int]]) -> typing.List['KademliaPeer']:
peer_address_list = [(await resolve_host(url, port, proto='tcp'), port) for url, port in peer_list]
kademlia_peer_list = [make_kademlia_peer(None, address, None, tcp_port=port, allow_localhost=True)
for address, port in peer_address_list]
return kademlia_peer_list

View file

@ -1,18 +1,21 @@
import typing import typing
import asyncio import asyncio
import logging import logging
import ipaddress
from binascii import hexlify
from dataclasses import dataclass, field from dataclasses import dataclass, field
from functools import lru_cache from functools import lru_cache
from prometheus_client import Gauge
from lbry.utils import is_valid_public_ipv4 as _is_valid_public_ipv4, LRUCache
from lbry.dht import constants from lbry.dht import constants
from lbry.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address from lbry.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
ALLOW_LOCALHOST = False
CACHE_SIZE = 16384
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@lru_cache(1024) @lru_cache(CACHE_SIZE)
def make_kademlia_peer(node_id: typing.Optional[bytes], address: typing.Optional[str], def make_kademlia_peer(node_id: typing.Optional[bytes], address: typing.Optional[str],
udp_port: typing.Optional[int] = None, udp_port: typing.Optional[int] = None,
tcp_port: typing.Optional[int] = None, tcp_port: typing.Optional[int] = None,
@ -20,40 +23,32 @@ def make_kademlia_peer(node_id: typing.Optional[bytes], address: typing.Optional
return KademliaPeer(address, node_id, udp_port, tcp_port=tcp_port, allow_localhost=allow_localhost) return KademliaPeer(address, node_id, udp_port, tcp_port=tcp_port, allow_localhost=allow_localhost)
# the ipaddress module does not show these subnets as reserved
CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
ALLOW_LOCALHOST = False
def is_valid_public_ipv4(address, allow_localhost: bool = False): def is_valid_public_ipv4(address, allow_localhost: bool = False):
allow_localhost = bool(allow_localhost or ALLOW_LOCALHOST) allow_localhost = bool(allow_localhost or ALLOW_LOCALHOST)
try: return _is_valid_public_ipv4(address, allow_localhost)
parsed_ip = ipaddress.ip_address(address)
if parsed_ip.is_loopback and allow_localhost:
return True
return not any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local,
parsed_ip.is_loopback, parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private,
parsed_ip.is_reserved,
CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
except ipaddress.AddressValueError:
return False
class PeerManager: class PeerManager:
peer_manager_keys_metric = Gauge(
"peer_manager_keys", "Number of keys tracked by PeerManager dicts (sum)", namespace="dht_node",
labelnames=("scope",)
)
def __init__(self, loop: asyncio.AbstractEventLoop): def __init__(self, loop: asyncio.AbstractEventLoop):
self._loop = loop self._loop = loop
self._rpc_failures: typing.Dict[ self._rpc_failures: typing.Dict[
typing.Tuple[str, int], typing.Tuple[typing.Optional[float], typing.Optional[float]] typing.Tuple[str, int], typing.Tuple[typing.Optional[float], typing.Optional[float]]
] = {} ] = LRUCache(CACHE_SIZE)
self._last_replied: typing.Dict[typing.Tuple[str, int], float] = {} self._last_replied: typing.Dict[typing.Tuple[str, int], float] = LRUCache(CACHE_SIZE)
self._last_sent: typing.Dict[typing.Tuple[str, int], float] = {} self._last_sent: typing.Dict[typing.Tuple[str, int], float] = LRUCache(CACHE_SIZE)
self._last_requested: typing.Dict[typing.Tuple[str, int], float] = {} self._last_requested: typing.Dict[typing.Tuple[str, int], float] = LRUCache(CACHE_SIZE)
self._node_id_mapping: typing.Dict[typing.Tuple[str, int], bytes] = {} self._node_id_mapping: typing.Dict[typing.Tuple[str, int], bytes] = LRUCache(CACHE_SIZE)
self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = {} self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = LRUCache(CACHE_SIZE)
self._node_tokens: typing.Dict[bytes, (float, bytes)] = {} self._node_tokens: typing.Dict[bytes, (float, bytes)] = LRUCache(CACHE_SIZE)
def count_cache_keys(self):
return len(self._rpc_failures) + len(self._last_replied) + len(self._last_sent) + len(
self._last_requested) + len(self._node_id_mapping) + len(self._node_id_reverse_mapping) + len(
self._node_tokens)
def reset(self): def reset(self):
for statistic in (self._rpc_failures, self._last_replied, self._last_sent, self._last_requested): for statistic in (self._rpc_failures, self._last_replied, self._last_sent, self._last_requested):
@ -103,6 +98,10 @@ class PeerManager:
self._node_id_mapping.pop(self._node_id_reverse_mapping.pop(node_id)) self._node_id_mapping.pop(self._node_id_reverse_mapping.pop(node_id))
self._node_id_mapping[(address, udp_port)] = node_id self._node_id_mapping[(address, udp_port)] = node_id
self._node_id_reverse_mapping[node_id] = (address, udp_port) self._node_id_reverse_mapping[node_id] = (address, udp_port)
self.peer_manager_keys_metric.labels("global").set(self.count_cache_keys())
def get_node_id_for_endpoint(self, address, port):
return self._node_id_mapping.get((address, port))
def prune(self): # TODO: periodically call this def prune(self): # TODO: periodically call this
now = self._loop.time() now = self._loop.time()
@ -154,7 +153,8 @@ class PeerManager:
def peer_is_good(self, peer: 'KademliaPeer'): def peer_is_good(self, peer: 'KademliaPeer'):
return self.contact_triple_is_good(peer.node_id, peer.address, peer.udp_port) return self.contact_triple_is_good(peer.node_id, peer.address, peer.udp_port)
def decode_tcp_peer_from_compact_address(self, compact_address: bytes) -> 'KademliaPeer': # pylint: disable=no-self-use
def decode_tcp_peer_from_compact_address(compact_address: bytes) -> 'KademliaPeer': # pylint: disable=no-self-use
node_id, address, tcp_port = decode_compact_address(compact_address) node_id, address, tcp_port = decode_compact_address(compact_address)
return make_kademlia_peer(node_id, address, udp_port=None, tcp_port=tcp_port) return make_kademlia_peer(node_id, address, udp_port=None, tcp_port=tcp_port)
@ -171,11 +171,11 @@ class KademliaPeer:
def __post_init__(self): def __post_init__(self):
if self._node_id is not None: if self._node_id is not None:
if not len(self._node_id) == constants.HASH_LENGTH: if not len(self._node_id) == constants.HASH_LENGTH:
raise ValueError("invalid node_id: {}".format(hexlify(self._node_id).decode())) raise ValueError("invalid node_id: {}".format(self._node_id.hex()))
if self.udp_port is not None and not 1 <= self.udp_port <= 65535: if self.udp_port is not None and not 1024 <= self.udp_port <= 65535:
raise ValueError("invalid udp port") raise ValueError(f"invalid udp port: {self.address}:{self.udp_port}")
if self.tcp_port is not None and not 1 <= self.tcp_port <= 65535: if self.tcp_port is not None and not 1024 <= self.tcp_port <= 65535:
raise ValueError("invalid tcp port") raise ValueError(f"invalid tcp port: {self.address}:{self.tcp_port}")
if not is_valid_public_ipv4(self.address, self.allow_localhost): if not is_valid_public_ipv4(self.address, self.allow_localhost):
raise ValueError(f"invalid ip address: '{self.address}'") raise ValueError(f"invalid ip address: '{self.address}'")
@ -194,3 +194,6 @@ class KademliaPeer:
def compact_ip(self): def compact_ip(self):
return make_compact_ip(self.address) return make_compact_ip(self.address)
def __str__(self):
return f"{self.__class__.__name__}({self.node_id.hex()[:8]}@{self.address}:{self.udp_port}-{self.tcp_port})"

View file

@ -16,6 +16,12 @@ class DictDataStore:
self._peer_manager = peer_manager self._peer_manager = peer_manager
self.completed_blobs: typing.Set[str] = set() self.completed_blobs: typing.Set[str] = set()
def keys(self):
return self._data_store.keys()
def __len__(self):
return self._data_store.__len__()
def removed_expired_peers(self): def removed_expired_peers(self):
now = self.loop.time() now = self.loop.time()
keys = list(self._data_store.keys()) keys = list(self._data_store.keys())

View file

@ -1,18 +1,17 @@
import asyncio import asyncio
from binascii import hexlify
from itertools import chain from itertools import chain
from collections import defaultdict from collections import defaultdict, OrderedDict
from collections.abc import AsyncIterator
import typing import typing
import logging import logging
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from lbry.dht import constants from lbry.dht import constants
from lbry.dht.error import RemoteException, TransportNotConnected from lbry.dht.error import RemoteException, TransportNotConnected
from lbry.dht.protocol.distance import Distance from lbry.dht.protocol.distance import Distance
from lbry.dht.peer import make_kademlia_peer from lbry.dht.peer import make_kademlia_peer, decode_tcp_peer_from_compact_address
from lbry.dht.serialization.datagram import PAGE_KEY from lbry.dht.serialization.datagram import PAGE_KEY
if TYPE_CHECKING: if TYPE_CHECKING:
from lbry.dht.protocol.routing_table import TreeRoutingTable
from lbry.dht.protocol.protocol import KademliaProtocol from lbry.dht.protocol.protocol import KademliaProtocol
from lbry.dht.peer import PeerManager, KademliaPeer from lbry.dht.peer import PeerManager, KademliaPeer
@ -27,6 +26,15 @@ class FindResponse:
def get_close_triples(self) -> typing.List[typing.Tuple[bytes, str, int]]: def get_close_triples(self) -> typing.List[typing.Tuple[bytes, str, int]]:
raise NotImplementedError() raise NotImplementedError()
def get_close_kademlia_peers(self, peer_info) -> typing.Generator[typing.Iterator['KademliaPeer'], None, None]:
for contact_triple in self.get_close_triples():
node_id, address, udp_port = contact_triple
try:
yield make_kademlia_peer(node_id, address, udp_port)
except ValueError:
log.warning("misbehaving peer %s:%i returned peer with reserved ip %s:%i", peer_info.address,
peer_info.udp_port, address, udp_port)
class FindNodeResponse(FindResponse): class FindNodeResponse(FindResponse):
def __init__(self, key: bytes, close_triples: typing.List[typing.Tuple[bytes, str, int]]): def __init__(self, key: bytes, close_triples: typing.List[typing.Tuple[bytes, str, int]]):
@ -57,57 +65,33 @@ class FindValueResponse(FindResponse):
return [(node_id, address.decode(), port) for node_id, address, port in self.close_triples] return [(node_id, address.decode(), port) for node_id, address, port in self.close_triples]
def get_shortlist(routing_table: 'TreeRoutingTable', key: bytes, class IterativeFinder(AsyncIterator):
shortlist: typing.Optional[typing.List['KademliaPeer']]) -> typing.List['KademliaPeer']: def __init__(self, loop: asyncio.AbstractEventLoop,
""" protocol: 'KademliaProtocol', key: bytes,
If not provided, initialize the shortlist of peers to probe to the (up to) k closest peers in the routing table max_results: typing.Optional[int] = constants.K,
:param routing_table: a TreeRoutingTable
:param key: a 48 byte hash
:param shortlist: optional manually provided shortlist, this is done during bootstrapping when there are no
peers in the routing table. During bootstrap the shortlist is set to be the seed nodes.
"""
if len(key) != constants.HASH_LENGTH:
raise ValueError("invalid key length: %i" % len(key))
return shortlist or routing_table.find_close_peers(key)
class IterativeFinder:
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager',
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.K,
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
shortlist: typing.Optional[typing.List['KademliaPeer']] = None): shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
if len(key) != constants.HASH_LENGTH: if len(key) != constants.HASH_LENGTH:
raise ValueError("invalid key length: %i" % len(key)) raise ValueError("invalid key length: %i" % len(key))
self.loop = loop self.loop = loop
self.peer_manager = peer_manager self.peer_manager = protocol.peer_manager
self.routing_table = routing_table
self.protocol = protocol self.protocol = protocol
self.key = key self.key = key
self.bottom_out_limit = bottom_out_limit self.max_results = max(constants.K, max_results)
self.max_results = max_results
self.exclude = exclude or []
self.active: typing.Set['KademliaPeer'] = set() self.active: typing.Dict['KademliaPeer', int] = OrderedDict() # peer: distance, sorted
self.contacted: typing.Set['KademliaPeer'] = set() self.contacted: typing.Set['KademliaPeer'] = set()
self.distance = Distance(key) self.distance = Distance(key)
self.closest_peer: typing.Optional['KademliaPeer'] = None self.iteration_queue = asyncio.Queue()
self.prev_closest_peer: typing.Optional['KademliaPeer'] = None
self.iteration_queue = asyncio.Queue(loop=self.loop) self.running_probes: typing.Dict['KademliaPeer', asyncio.Task] = {}
self.running_probes: typing.Set[asyncio.Task] = set()
self.iteration_count = 0 self.iteration_count = 0
self.bottom_out_count = 0
self.running = False self.running = False
self.tasks: typing.List[asyncio.Task] = [] self.tasks: typing.List[asyncio.Task] = []
self.delayed_calls: typing.List[asyncio.Handle] = [] for peer in shortlist:
for peer in get_shortlist(routing_table, key, shortlist):
if peer.node_id: if peer.node_id:
self._add_active(peer) self._add_active(peer, force=True)
else: else:
# seed nodes # seed nodes
self._schedule_probe(peer) self._schedule_probe(peer)
@ -139,66 +123,79 @@ class IterativeFinder:
""" """
return [] return []
def _is_closer(self, peer: 'KademliaPeer') -> bool: def _add_active(self, peer, force=False):
return not self.closest_peer or self.distance.is_closer(peer.node_id, self.closest_peer.node_id) if not force and self.peer_manager.peer_is_good(peer) is False:
return
def _add_active(self, peer): if peer in self.contacted:
return
if peer not in self.active and peer.node_id and peer.node_id != self.protocol.node_id: if peer not in self.active and peer.node_id and peer.node_id != self.protocol.node_id:
self.active.add(peer) self.active[peer] = self.distance(peer.node_id)
if self._is_closer(peer): self.active = OrderedDict(sorted(self.active.items(), key=lambda item: item[1]))
self.prev_closest_peer = self.closest_peer
self.closest_peer = peer
async def _handle_probe_result(self, peer: 'KademliaPeer', response: FindResponse): async def _handle_probe_result(self, peer: 'KademliaPeer', response: FindResponse):
self._add_active(peer) self._add_active(peer)
for contact_triple in response.get_close_triples(): for new_peer in response.get_close_kademlia_peers(peer):
node_id, address, udp_port = contact_triple self._add_active(new_peer)
try:
self._add_active(make_kademlia_peer(node_id, address, udp_port))
except ValueError:
log.warning("misbehaving peer %s:%i returned peer with reserved ip %s:%i", peer.address,
peer.udp_port, address, udp_port)
self.check_result_ready(response) self.check_result_ready(response)
self._log_state(reason="check result")
def _reset_closest(self, peer):
if peer in self.active:
del self.active[peer]
async def _send_probe(self, peer: 'KademliaPeer'): async def _send_probe(self, peer: 'KademliaPeer'):
try: try:
response = await self.send_probe(peer) response = await self.send_probe(peer)
except asyncio.TimeoutError: except asyncio.TimeoutError:
self.active.discard(peer) self._reset_closest(peer)
return return
except asyncio.CancelledError:
log.debug("%s[%x] cancelled probe",
type(self).__name__, id(self))
raise
except ValueError as err: except ValueError as err:
log.warning(str(err)) log.warning(str(err))
self.active.discard(peer) self._reset_closest(peer)
return return
except TransportNotConnected: except TransportNotConnected:
return self.aclose() await self._aclose(reason="not connected")
return
except RemoteException: except RemoteException:
self._reset_closest(peer)
return return
return await self._handle_probe_result(peer, response) return await self._handle_probe_result(peer, response)
async def _search_round(self): def _search_round(self):
""" """
Send up to constants.alpha (5) probes to closest active peers Send up to constants.alpha (5) probes to closest active peers
""" """
added = 0 added = 0
to_probe = list(self.active - self.contacted) for index, peer in enumerate(self.active.keys()):
to_probe.sort(key=lambda peer: self.distance(self.key)) if index == 0:
for peer in to_probe: log.debug("%s[%x] closest to probe: %s",
if added >= constants.ALPHA: type(self).__name__, id(self),
peer.node_id.hex()[:8])
if peer in self.contacted:
continue
if len(self.running_probes) >= constants.ALPHA:
break
if index > (constants.K + len(self.running_probes)):
break break
origin_address = (peer.address, peer.udp_port) origin_address = (peer.address, peer.udp_port)
if origin_address in self.exclude:
continue
if peer.node_id == self.protocol.node_id: if peer.node_id == self.protocol.node_id:
continue continue
if origin_address == (self.protocol.external_ip, self.protocol.udp_port): if origin_address == (self.protocol.external_ip, self.protocol.udp_port):
continue continue
self._schedule_probe(peer) self._schedule_probe(peer)
added += 1 added += 1
log.debug("running %d probes", len(self.running_probes)) log.debug("%s[%x] running %d probes for key %s",
type(self).__name__, id(self),
len(self.running_probes), self.key.hex()[:8])
if not added and not self.running_probes: if not added and not self.running_probes:
log.debug("search for %s exhausted", hexlify(self.key)[:8]) log.debug("%s[%x] search for %s exhausted",
type(self).__name__, id(self),
self.key.hex()[:8])
self.search_exhausted() self.search_exhausted()
def _schedule_probe(self, peer: 'KademliaPeer'): def _schedule_probe(self, peer: 'KademliaPeer'):
@ -207,33 +204,24 @@ class IterativeFinder:
t = self.loop.create_task(self._send_probe(peer)) t = self.loop.create_task(self._send_probe(peer))
def callback(_): def callback(_):
self.running_probes.difference_update({ self.running_probes.pop(peer, None)
probe for probe in self.running_probes if probe.done() or probe == t if self.running:
}) self._search_round()
if not self.running_probes:
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
t.add_done_callback(callback) t.add_done_callback(callback)
self.running_probes.add(t) self.running_probes[peer] = t
async def _search_task(self, delay: typing.Optional[float] = constants.ITERATIVE_LOOKUP_DELAY): def _log_state(self, reason="?"):
try: log.debug("%s[%x] [%s] %s: %i active nodes %i contacted %i produced %i queued",
if self.running: type(self).__name__, id(self), self.key.hex()[:8],
await self._search_round() reason, len(self.active), len(self.contacted),
if self.running: self.iteration_count, self.iteration_queue.qsize())
self.delayed_calls.append(self.loop.call_later(delay, self._search))
except (asyncio.CancelledError, StopAsyncIteration, TransportNotConnected):
if self.running:
self.loop.call_soon(self.aclose)
def _search(self):
self.tasks.append(self.loop.create_task(self._search_task()))
def __aiter__(self): def __aiter__(self):
if self.running: if self.running:
raise Exception("already running") raise Exception("already running")
self.running = True self.running = True
self._search() self.loop.call_soon(self._search_round)
return self return self
async def __anext__(self) -> typing.List['KademliaPeer']: async def __anext__(self) -> typing.List['KademliaPeer']:
@ -246,47 +234,57 @@ class IterativeFinder:
raise StopAsyncIteration raise StopAsyncIteration
self.iteration_count += 1 self.iteration_count += 1
return result return result
except (asyncio.CancelledError, StopAsyncIteration): except asyncio.CancelledError:
self.loop.call_soon(self.aclose) await self._aclose(reason="cancelled")
raise
except StopAsyncIteration:
await self._aclose(reason="no more results")
raise raise
def aclose(self): async def _aclose(self, reason="?"):
log.debug("%s[%x] [%s] shutdown because %s: %i active nodes %i contacted %i produced %i queued",
type(self).__name__, id(self), self.key.hex()[:8],
reason, len(self.active), len(self.contacted),
self.iteration_count, self.iteration_queue.qsize())
self.running = False self.running = False
self.iteration_queue.put_nowait(None) self.iteration_queue.put_nowait(None)
for task in chain(self.tasks, self.running_probes, self.delayed_calls): for task in chain(self.tasks, self.running_probes.values()):
task.cancel() task.cancel()
self.tasks.clear() self.tasks.clear()
self.running_probes.clear() self.running_probes.clear()
self.delayed_calls.clear()
async def aclose(self):
if self.running:
await self._aclose(reason="aclose")
log.debug("%s[%x] [%s] async close completed",
type(self).__name__, id(self), self.key.hex()[:8])
class IterativeNodeFinder(IterativeFinder): class IterativeNodeFinder(IterativeFinder):
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', def __init__(self, loop: asyncio.AbstractEventLoop,
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes, protocol: 'KademliaProtocol', key: bytes,
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.K, max_results: typing.Optional[int] = constants.K,
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
shortlist: typing.Optional[typing.List['KademliaPeer']] = None): shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
super().__init__(loop, peer_manager, routing_table, protocol, key, bottom_out_limit, max_results, exclude, super().__init__(loop, protocol, key, max_results, shortlist)
shortlist)
self.yielded_peers: typing.Set['KademliaPeer'] = set() self.yielded_peers: typing.Set['KademliaPeer'] = set()
async def send_probe(self, peer: 'KademliaPeer') -> FindNodeResponse: async def send_probe(self, peer: 'KademliaPeer') -> FindNodeResponse:
log.debug("probing %s:%d %s", peer.address, peer.udp_port, hexlify(peer.node_id)[:8] if peer.node_id else '') log.debug("probe %s:%d (%s) for NODE %s",
peer.address, peer.udp_port, peer.node_id.hex()[:8] if peer.node_id else '', self.key.hex()[:8])
response = await self.protocol.get_rpc_peer(peer).find_node(self.key) response = await self.protocol.get_rpc_peer(peer).find_node(self.key)
return FindNodeResponse(self.key, response) return FindNodeResponse(self.key, response)
def search_exhausted(self): def search_exhausted(self):
self.put_result(self.active, finish=True) self.put_result(self.active.keys(), finish=True)
def put_result(self, from_iter: typing.Iterable['KademliaPeer'], finish=False): def put_result(self, from_iter: typing.Iterable['KademliaPeer'], finish=False):
not_yet_yielded = [ not_yet_yielded = [
peer for peer in from_iter peer for peer in from_iter
if peer not in self.yielded_peers if peer not in self.yielded_peers
and peer.node_id != self.protocol.node_id and peer.node_id != self.protocol.node_id
and self.peer_manager.peer_is_good(peer) is not False and self.peer_manager.peer_is_good(peer) is True # return only peers who answered
] ]
not_yet_yielded.sort(key=lambda peer: self.distance(peer.node_id)) not_yet_yielded.sort(key=lambda peer: self.distance(peer.node_id))
to_yield = not_yet_yielded[:min(constants.K, len(not_yet_yielded))] to_yield = not_yet_yielded[:max(constants.K, self.max_results)]
if to_yield: if to_yield:
self.yielded_peers.update(to_yield) self.yielded_peers.update(to_yield)
self.iteration_queue.put_nowait(to_yield) self.iteration_queue.put_nowait(to_yield)
@ -298,27 +296,15 @@ class IterativeNodeFinder(IterativeFinder):
if found: if found:
log.debug("found") log.debug("found")
return self.put_result(self.active, finish=True) return self.put_result(self.active.keys(), finish=True)
if self.prev_closest_peer and self.closest_peer and not self._is_closer(self.prev_closest_peer):
# log.info("improving, %i %i %i %i %i", len(self.shortlist), len(self.active), len(self.contacted),
# self.bottom_out_count, self.iteration_count)
self.bottom_out_count = 0
elif self.prev_closest_peer and self.closest_peer:
self.bottom_out_count += 1
log.info("bottom out %i %i %i", len(self.active), len(self.contacted), self.bottom_out_count)
if self.bottom_out_count >= self.bottom_out_limit or self.iteration_count >= self.bottom_out_limit:
log.info("limit hit")
self.put_result(self.active, True)
class IterativeValueFinder(IterativeFinder): class IterativeValueFinder(IterativeFinder):
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', def __init__(self, loop: asyncio.AbstractEventLoop,
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes, protocol: 'KademliaProtocol', key: bytes,
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.K, max_results: typing.Optional[int] = constants.K,
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
shortlist: typing.Optional[typing.List['KademliaPeer']] = None): shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
super().__init__(loop, peer_manager, routing_table, protocol, key, bottom_out_limit, max_results, exclude, super().__init__(loop, protocol, key, max_results, shortlist)
shortlist)
self.blob_peers: typing.Set['KademliaPeer'] = set() self.blob_peers: typing.Set['KademliaPeer'] = set()
# this tracks the index of the most recent page we requested from each peer # this tracks the index of the most recent page we requested from each peer
self.peer_pages: typing.DefaultDict['KademliaPeer', int] = defaultdict(int) self.peer_pages: typing.DefaultDict['KademliaPeer', int] = defaultdict(int)
@ -326,6 +312,8 @@ class IterativeValueFinder(IterativeFinder):
self.discovered_peers: typing.Dict['KademliaPeer', typing.Set['KademliaPeer']] = defaultdict(set) self.discovered_peers: typing.Dict['KademliaPeer', typing.Set['KademliaPeer']] = defaultdict(set)
async def send_probe(self, peer: 'KademliaPeer') -> FindValueResponse: async def send_probe(self, peer: 'KademliaPeer') -> FindValueResponse:
log.debug("probe %s:%d (%s) for VALUE %s",
peer.address, peer.udp_port, peer.node_id.hex()[:8], self.key.hex()[:8])
page = self.peer_pages[peer] page = self.peer_pages[peer]
response = await self.protocol.get_rpc_peer(peer).find_value(self.key, page=page) response = await self.protocol.get_rpc_peer(peer).find_value(self.key, page=page)
parsed = FindValueResponse(self.key, response) parsed = FindValueResponse(self.key, response)
@ -335,7 +323,7 @@ class IterativeValueFinder(IterativeFinder):
decoded_peers = set() decoded_peers = set()
for compact_addr in parsed.found_compact_addresses: for compact_addr in parsed.found_compact_addresses:
try: try:
decoded_peers.add(self.peer_manager.decode_tcp_peer_from_compact_address(compact_addr)) decoded_peers.add(decode_tcp_peer_from_compact_address(compact_addr))
except ValueError: except ValueError:
log.warning("misbehaving peer %s:%i returned invalid peer for blob", log.warning("misbehaving peer %s:%i returned invalid peer for blob",
peer.address, peer.udp_port) peer.address, peer.udp_port)
@ -347,7 +335,6 @@ class IterativeValueFinder(IterativeFinder):
already_known + len(parsed.found_compact_addresses)) already_known + len(parsed.found_compact_addresses))
if len(self.discovered_peers[peer]) != already_known + len(parsed.found_compact_addresses): if len(self.discovered_peers[peer]) != already_known + len(parsed.found_compact_addresses):
log.warning("misbehaving peer %s:%i returned duplicate peers for blob", peer.address, peer.udp_port) log.warning("misbehaving peer %s:%i returned duplicate peers for blob", peer.address, peer.udp_port)
parsed.found_compact_addresses.clear()
elif len(parsed.found_compact_addresses) >= constants.K and self.peer_pages[peer] < parsed.pages: elif len(parsed.found_compact_addresses) >= constants.K and self.peer_pages[peer] < parsed.pages:
# the peer returned a full page and indicates it has more # the peer returned a full page and indicates it has more
self.peer_pages[peer] += 1 self.peer_pages[peer] += 1
@ -358,26 +345,15 @@ class IterativeValueFinder(IterativeFinder):
def check_result_ready(self, response: FindValueResponse): def check_result_ready(self, response: FindValueResponse):
if response.found: if response.found:
blob_peers = [self.peer_manager.decode_tcp_peer_from_compact_address(compact_addr) blob_peers = [decode_tcp_peer_from_compact_address(compact_addr)
for compact_addr in response.found_compact_addresses] for compact_addr in response.found_compact_addresses]
to_yield = [] to_yield = []
self.bottom_out_count = 0
for blob_peer in blob_peers: for blob_peer in blob_peers:
if blob_peer not in self.blob_peers: if blob_peer not in self.blob_peers:
self.blob_peers.add(blob_peer) self.blob_peers.add(blob_peer)
to_yield.append(blob_peer) to_yield.append(blob_peer)
if to_yield: if to_yield:
# log.info("found %i new peers for blob", len(to_yield))
self.iteration_queue.put_nowait(to_yield) self.iteration_queue.put_nowait(to_yield)
# if self.max_results and len(self.blob_peers) >= self.max_results:
# log.info("enough blob peers found")
# if not self.finished.is_set():
# self.finished.set()
elif self.prev_closest_peer and self.closest_peer:
self.bottom_out_count += 1
if self.bottom_out_count >= self.bottom_out_limit:
log.info("blob peer search bottomed out")
self.iteration_queue.put_nowait(None)
def get_initial_result(self) -> typing.List['KademliaPeer']: def get_initial_result(self) -> typing.List['KademliaPeer']:
if self.protocol.data_store.has_peers_for_blob(self.key): if self.protocol.data_store.has_peers_for_blob(self.key):

View file

@ -3,13 +3,16 @@ import socket
import functools import functools
import hashlib import hashlib
import asyncio import asyncio
import time
import typing import typing
import binascii
import random import random
from asyncio.protocols import DatagramProtocol from asyncio.protocols import DatagramProtocol
from asyncio.transports import DatagramTransport from asyncio.transports import DatagramTransport
from prometheus_client import Gauge, Counter, Histogram
from lbry.dht import constants from lbry.dht import constants
from lbry.dht.serialization.bencoding import DecodeError
from lbry.dht.serialization.datagram import decode_datagram, ErrorDatagram, ResponseDatagram, RequestDatagram from lbry.dht.serialization.datagram import decode_datagram, ErrorDatagram, ResponseDatagram, RequestDatagram
from lbry.dht.serialization.datagram import RESPONSE_TYPE, ERROR_TYPE, PAGE_KEY from lbry.dht.serialization.datagram import RESPONSE_TYPE, ERROR_TYPE, PAGE_KEY
from lbry.dht.error import RemoteException, TransportNotConnected from lbry.dht.error import RemoteException, TransportNotConnected
@ -30,6 +33,11 @@ OLD_PROTOCOL_ERRORS = {
class KademliaRPC: class KademliaRPC:
stored_blob_metric = Gauge(
"stored_blobs", "Number of blobs announced by other peers", namespace="dht_node",
labelnames=("scope",),
)
def __init__(self, protocol: 'KademliaProtocol', loop: asyncio.AbstractEventLoop, peer_port: int = 3333): def __init__(self, protocol: 'KademliaProtocol', loop: asyncio.AbstractEventLoop, peer_port: int = 3333):
self.protocol = protocol self.protocol = protocol
self.loop = loop self.loop = loop
@ -61,6 +69,7 @@ class KademliaRPC:
self.protocol.data_store.add_peer_to_blob( self.protocol.data_store.add_peer_to_blob(
rpc_contact, blob_hash rpc_contact, blob_hash
) )
self.stored_blob_metric.labels("global").set(len(self.protocol.data_store))
return b'OK' return b'OK'
def find_node(self, rpc_contact: 'KademliaPeer', key: bytes) -> typing.List[typing.Tuple[bytes, str, int]]: def find_node(self, rpc_contact: 'KademliaPeer', key: bytes) -> typing.List[typing.Tuple[bytes, str, int]]:
@ -96,7 +105,7 @@ class KademliaRPC:
if not rpc_contact.tcp_port or peer.compact_address_tcp() != rpc_contact.compact_address_tcp() if not rpc_contact.tcp_port or peer.compact_address_tcp() != rpc_contact.compact_address_tcp()
] ]
# if we don't have k storing peers to return and we have this hash locally, include our contact information # if we don't have k storing peers to return and we have this hash locally, include our contact information
if len(peers) < constants.K and binascii.hexlify(key).decode() in self.protocol.data_store.completed_blobs: if len(peers) < constants.K and key.hex() in self.protocol.data_store.completed_blobs:
peers.append(self.compact_address()) peers.append(self.compact_address())
if not peers: if not peers:
response[PAGE_KEY] = 0 response[PAGE_KEY] = 0
@ -209,6 +218,10 @@ class PingQueue:
def running(self): def running(self):
return self._running return self._running
@property
def busy(self):
return self._running and (any(self._running_pings) or any(self._pending_contacts))
def enqueue_maybe_ping(self, *peers: 'KademliaPeer', delay: typing.Optional[float] = None): def enqueue_maybe_ping(self, *peers: 'KademliaPeer', delay: typing.Optional[float] = None):
delay = delay if delay is not None else self._default_delay delay = delay if delay is not None else self._default_delay
now = self._loop.time() now = self._loop.time()
@ -220,7 +233,7 @@ class PingQueue:
async def ping_task(): async def ping_task():
try: try:
if self._protocol.peer_manager.peer_is_good(peer): if self._protocol.peer_manager.peer_is_good(peer):
if peer not in self._protocol.routing_table.get_peers(): if not self._protocol.routing_table.get_peer(peer.node_id):
self._protocol.add_peer(peer) self._protocol.add_peer(peer)
return return
await self._protocol.get_rpc_peer(peer).ping() await self._protocol.get_rpc_peer(peer).ping()
@ -240,7 +253,7 @@ class PingQueue:
del self._pending_contacts[peer] del self._pending_contacts[peer]
self.maybe_ping(peer) self.maybe_ping(peer)
break break
await asyncio.sleep(1, loop=self._loop) await asyncio.sleep(1)
def start(self): def start(self):
assert not self._running assert not self._running
@ -259,9 +272,33 @@ class PingQueue:
class KademliaProtocol(DatagramProtocol): class KademliaProtocol(DatagramProtocol):
request_sent_metric = Counter(
"request_sent", "Number of requests send from DHT RPC protocol", namespace="dht_node",
labelnames=("method",),
)
request_success_metric = Counter(
"request_success", "Number of successful requests", namespace="dht_node",
labelnames=("method",),
)
request_error_metric = Counter(
"request_error", "Number of errors returned from request to other peers", namespace="dht_node",
labelnames=("method",),
)
HISTOGRAM_BUCKETS = (
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 3.0, 3.5, 4.0, 4.50, 5.0, 5.50, 6.0, float('inf')
)
response_time_metric = Histogram(
"response_time", "Response times of DHT RPC requests", namespace="dht_node", buckets=HISTOGRAM_BUCKETS,
labelnames=("method",)
)
received_request_metric = Counter(
"received_request", "Number of received DHT RPC requests", namespace="dht_node",
labelnames=("method",),
)
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', node_id: bytes, external_ip: str, def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', node_id: bytes, external_ip: str,
udp_port: int, peer_port: int, rpc_timeout: float = constants.RPC_TIMEOUT, udp_port: int, peer_port: int, rpc_timeout: float = constants.RPC_TIMEOUT,
split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX): split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX, is_boostrap_node: bool = False):
self.peer_manager = peer_manager self.peer_manager = peer_manager
self.loop = loop self.loop = loop
self.node_id = node_id self.node_id = node_id
@ -276,15 +313,16 @@ class KademliaProtocol(DatagramProtocol):
self.transport: DatagramTransport = None self.transport: DatagramTransport = None
self.old_token_secret = constants.generate_id() self.old_token_secret = constants.generate_id()
self.token_secret = constants.generate_id() self.token_secret = constants.generate_id()
self.routing_table = TreeRoutingTable(self.loop, self.peer_manager, self.node_id, split_buckets_under_index) self.routing_table = TreeRoutingTable(
self.loop, self.peer_manager, self.node_id, split_buckets_under_index, is_bootstrap_node=is_boostrap_node)
self.data_store = DictDataStore(self.loop, self.peer_manager) self.data_store = DictDataStore(self.loop, self.peer_manager)
self.ping_queue = PingQueue(self.loop, self) self.ping_queue = PingQueue(self.loop, self)
self.node_rpc = KademliaRPC(self, self.loop, self.peer_port) self.node_rpc = KademliaRPC(self, self.loop, self.peer_port)
self.rpc_timeout = rpc_timeout self.rpc_timeout = rpc_timeout
self._split_lock = asyncio.Lock(loop=self.loop) self._split_lock = asyncio.Lock()
self._to_remove: typing.Set['KademliaPeer'] = set() self._to_remove: typing.Set['KademliaPeer'] = set()
self._to_add: typing.Set['KademliaPeer'] = set() self._to_add: typing.Set['KademliaPeer'] = set()
self._wakeup_routing_task = asyncio.Event(loop=self.loop) self._wakeup_routing_task = asyncio.Event()
self.maintaing_routing_task: typing.Optional[asyncio.Task] = None self.maintaing_routing_task: typing.Optional[asyncio.Task] = None
@functools.lru_cache(128) @functools.lru_cache(128)
@ -323,72 +361,10 @@ class KademliaProtocol(DatagramProtocol):
return args, {} return args, {}
async def _add_peer(self, peer: 'KademliaPeer'): async def _add_peer(self, peer: 'KademliaPeer'):
if not peer.node_id: async def probe(some_peer: 'KademliaPeer'):
log.warning("Tried adding a peer with no node id!") rpc_peer = self.get_rpc_peer(some_peer)
return False await rpc_peer.ping()
for my_peer in self.routing_table.get_peers(): return await self.routing_table.add_peer(peer, probe)
if (my_peer.address, my_peer.udp_port) == (peer.address, peer.udp_port) and my_peer.node_id != peer.node_id:
self.routing_table.remove_peer(my_peer)
self.routing_table.join_buckets()
bucket_index = self.routing_table.kbucket_index(peer.node_id)
if self.routing_table.buckets[bucket_index].add_peer(peer):
return True
# The bucket is full; see if it can be split (by checking if its range includes the host node's node_id)
if self.routing_table.should_split(bucket_index, peer.node_id):
self.routing_table.split_bucket(bucket_index)
# Retry the insertion attempt
result = await self._add_peer(peer)
self.routing_table.join_buckets()
return result
else:
# We can't split the k-bucket
#
# The 13 page kademlia paper specifies that the least recently contacted node in the bucket
# shall be pinged. If it fails to reply it is replaced with the new contact. If the ping is successful
# the new contact is ignored and not added to the bucket (sections 2.2 and 2.4).
#
# A reasonable extension to this is BEP 0005, which extends the above:
#
# Not all nodes that we learn about are equal. Some are "good" and some are not.
# Many nodes using the DHT are able to send queries and receive responses,
# but are not able to respond to queries from other nodes. It is important that
# each node's routing table must contain only known good nodes. A good node is
# a node has responded to one of our queries within the last 15 minutes. A node
# is also good if it has ever responded to one of our queries and has sent us a
# query within the last 15 minutes. After 15 minutes of inactivity, a node becomes
# questionable. Nodes become bad when they fail to respond to multiple queries
# in a row. Nodes that we know are good are given priority over nodes with unknown status.
#
# When there are bad or questionable nodes in the bucket, the least recent is selected for
# potential replacement (BEP 0005). When all nodes in the bucket are fresh, the head (least recent)
# contact is selected as described in section 2.2 of the kademlia paper. In both cases the new contact
# is ignored if the pinged node replies.
not_good_contacts = self.routing_table.buckets[bucket_index].get_bad_or_unknown_peers()
not_recently_replied = []
for my_peer in not_good_contacts:
last_replied = self.peer_manager.get_last_replied(my_peer.address, my_peer.udp_port)
if not last_replied or last_replied + 60 < self.loop.time():
not_recently_replied.append(my_peer)
if not_recently_replied:
to_replace = not_recently_replied[0]
else:
to_replace = self.routing_table.buckets[bucket_index].peers[0]
last_replied = self.peer_manager.get_last_replied(to_replace.address, to_replace.udp_port)
if last_replied and last_replied + 60 > self.loop.time():
return False
log.debug("pinging %s:%s", to_replace.address, to_replace.udp_port)
try:
to_replace_rpc = self.get_rpc_peer(to_replace)
await to_replace_rpc.ping()
return False
except asyncio.TimeoutError:
log.debug("Replacing dead contact in bucket %i: %s:%i with %s:%i ", bucket_index,
to_replace.address, to_replace.udp_port, peer.address, peer.udp_port)
if to_replace in self.routing_table.buckets[bucket_index]:
self.routing_table.buckets[bucket_index].remove_peer(to_replace)
return await self._add_peer(peer)
def add_peer(self, peer: 'KademliaPeer'): def add_peer(self, peer: 'KademliaPeer'):
if peer.node_id == self.node_id: if peer.node_id == self.node_id:
@ -406,16 +382,15 @@ class KademliaProtocol(DatagramProtocol):
async with self._split_lock: async with self._split_lock:
peer = self._to_remove.pop() peer = self._to_remove.pop()
self.routing_table.remove_peer(peer) self.routing_table.remove_peer(peer)
self.routing_table.join_buckets()
while self._to_add: while self._to_add:
async with self._split_lock: async with self._split_lock:
await self._add_peer(self._to_add.pop()) await self._add_peer(self._to_add.pop())
await asyncio.gather(self._wakeup_routing_task.wait(), asyncio.sleep(.1, loop=self.loop), loop=self.loop) await asyncio.gather(self._wakeup_routing_task.wait(), asyncio.sleep(.1))
self._wakeup_routing_task.clear() self._wakeup_routing_task.clear()
def _handle_rpc(self, sender_contact: 'KademliaPeer', message: RequestDatagram): def _handle_rpc(self, sender_contact: 'KademliaPeer', message: RequestDatagram):
assert sender_contact.node_id != self.node_id, (binascii.hexlify(sender_contact.node_id)[:8].decode(), assert sender_contact.node_id != self.node_id, (sender_contact.node_id.hex()[:8],
binascii.hexlify(self.node_id)[:8].decode()) self.node_id.hex()[:8])
method = message.method method = message.method
if method not in [b'ping', b'store', b'findNode', b'findValue']: if method not in [b'ping', b'store', b'findNode', b'findValue']:
raise AttributeError('Invalid method: %s' % message.method.decode()) raise AttributeError('Invalid method: %s' % message.method.decode())
@ -447,11 +422,15 @@ class KademliaProtocol(DatagramProtocol):
def handle_request_datagram(self, address: typing.Tuple[str, int], request_datagram: RequestDatagram): def handle_request_datagram(self, address: typing.Tuple[str, int], request_datagram: RequestDatagram):
# This is an RPC method request # This is an RPC method request
self.received_request_metric.labels(method=request_datagram.method).inc()
self.peer_manager.report_last_requested(address[0], address[1]) self.peer_manager.report_last_requested(address[0], address[1])
try:
peer = self.routing_table.get_peer(request_datagram.node_id) peer = self.routing_table.get_peer(request_datagram.node_id)
except IndexError: if not peer:
try:
peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1]) peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1])
except ValueError as err:
log.warning("error replying to %s: %s", address[0], str(err))
return
try: try:
self._handle_rpc(peer, request_datagram) self._handle_rpc(peer, request_datagram)
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it # if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
@ -551,12 +530,12 @@ class KademliaProtocol(DatagramProtocol):
address[0], address[1], OLD_PROTOCOL_ERRORS[error_datagram.response] address[0], address[1], OLD_PROTOCOL_ERRORS[error_datagram.response]
) )
def datagram_received(self, datagram: bytes, address: typing.Tuple[str, int]) -> None: # pylint: disable=arguments-differ def datagram_received(self, datagram: bytes, address: typing.Tuple[str, int]) -> None: # pylint: disable=arguments-renamed
try: try:
message = decode_datagram(datagram) message = decode_datagram(datagram)
except (ValueError, TypeError): except (ValueError, TypeError, DecodeError):
self.peer_manager.report_failure(address[0], address[1]) self.peer_manager.report_failure(address[0], address[1])
log.warning("Couldn't decode dht datagram from %s: %s", address, binascii.hexlify(datagram).decode()) log.warning("Couldn't decode dht datagram from %s: %s", address, datagram.hex())
return return
if isinstance(message, RequestDatagram): if isinstance(message, RequestDatagram):
@ -571,14 +550,19 @@ class KademliaProtocol(DatagramProtocol):
self._send(peer, request) self._send(peer, request)
response_fut = self.sent_messages[request.rpc_id][1] response_fut = self.sent_messages[request.rpc_id][1]
try: try:
self.request_sent_metric.labels(method=request.method).inc()
start = time.perf_counter()
response = await asyncio.wait_for(response_fut, self.rpc_timeout) response = await asyncio.wait_for(response_fut, self.rpc_timeout)
self.response_time_metric.labels(method=request.method).observe(time.perf_counter() - start)
self.peer_manager.report_last_replied(peer.address, peer.udp_port) self.peer_manager.report_last_replied(peer.address, peer.udp_port)
self.request_success_metric.labels(method=request.method).inc()
return response return response
except asyncio.CancelledError: except asyncio.CancelledError:
if not response_fut.done(): if not response_fut.done():
response_fut.cancel() response_fut.cancel()
raise raise
except (asyncio.TimeoutError, RemoteException): except (asyncio.TimeoutError, RemoteException):
self.request_error_metric.labels(method=request.method).inc()
self.peer_manager.report_failure(peer.address, peer.udp_port) self.peer_manager.report_failure(peer.address, peer.udp_port)
if self.peer_manager.peer_is_good(peer) is False: if self.peer_manager.peer_is_good(peer) is False:
self.remove_peer(peer) self.remove_peer(peer)
@ -598,7 +582,7 @@ class KademliaProtocol(DatagramProtocol):
if len(data) > constants.MSG_SIZE_LIMIT: if len(data) > constants.MSG_SIZE_LIMIT:
log.warning("cannot send datagram larger than %i bytes (packet is %i bytes)", log.warning("cannot send datagram larger than %i bytes (packet is %i bytes)",
constants.MSG_SIZE_LIMIT, len(data)) constants.MSG_SIZE_LIMIT, len(data))
log.debug("Packet is too large to send: %s", binascii.hexlify(data[:3500]).decode()) log.debug("Packet is too large to send: %s", data[:3500].hex())
raise ValueError( raise ValueError(
f"cannot send datagram larger than {constants.MSG_SIZE_LIMIT} bytes (packet is {len(data)} bytes)" f"cannot send datagram larger than {constants.MSG_SIZE_LIMIT} bytes (packet is {len(data)} bytes)"
) )
@ -658,13 +642,13 @@ class KademliaProtocol(DatagramProtocol):
res = await self.get_rpc_peer(peer).store(hash_value) res = await self.get_rpc_peer(peer).store(hash_value)
if res != b"OK": if res != b"OK":
raise ValueError(res) raise ValueError(res)
log.debug("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer) log.debug("Stored %s to %s", hash_value.hex()[:8], peer)
return peer.node_id, True return peer.node_id, True
try: try:
return await __store() return await __store()
except asyncio.TimeoutError: except asyncio.TimeoutError:
log.debug("Timeout while storing blob_hash %s at %s", binascii.hexlify(hash_value).decode()[:8], peer) log.debug("Timeout while storing blob_hash %s at %s", hash_value.hex()[:8], peer)
return peer.node_id, False return peer.node_id, False
except ValueError as err: except ValueError as err:
log.error("Unexpected response: %s", err) log.error("Unexpected response: %s", err)

View file

@ -4,7 +4,11 @@ import logging
import typing import typing
import itertools import itertools
from prometheus_client import Gauge
from lbry import utils
from lbry.dht import constants from lbry.dht import constants
from lbry.dht.error import RemoteException
from lbry.dht.protocol.distance import Distance from lbry.dht.protocol.distance import Distance
if typing.TYPE_CHECKING: if typing.TYPE_CHECKING:
from lbry.dht.peer import KademliaPeer, PeerManager from lbry.dht.peer import KademliaPeer, PeerManager
@ -13,10 +17,20 @@ log = logging.getLogger(__name__)
class KBucket: class KBucket:
""" Description - later
""" """
Kademlia K-bucket implementation.
"""
peer_in_routing_table_metric = Gauge(
"peers_in_routing_table", "Number of peers on routing table", namespace="dht_node",
labelnames=("scope",)
)
peer_with_x_bit_colliding_metric = Gauge(
"peer_x_bit_colliding", "Number of peers with at least X bits colliding with this node id",
namespace="dht_node", labelnames=("amount",)
)
def __init__(self, peer_manager: 'PeerManager', range_min: int, range_max: int, node_id: bytes): def __init__(self, peer_manager: 'PeerManager', range_min: int, range_max: int,
node_id: bytes, capacity: int = constants.K):
""" """
@param range_min: The lower boundary for the range in the n-bit ID @param range_min: The lower boundary for the range in the n-bit ID
space covered by this k-bucket space covered by this k-bucket
@ -24,12 +38,12 @@ class KBucket:
covered by this k-bucket covered by this k-bucket
""" """
self._peer_manager = peer_manager self._peer_manager = peer_manager
self.last_accessed = 0
self.range_min = range_min self.range_min = range_min
self.range_max = range_max self.range_max = range_max
self.peers: typing.List['KademliaPeer'] = [] self.peers: typing.List['KademliaPeer'] = []
self._node_id = node_id self._node_id = node_id
self._distance_to_self = Distance(node_id) self._distance_to_self = Distance(node_id)
self.capacity = capacity
def add_peer(self, peer: 'KademliaPeer') -> bool: def add_peer(self, peer: 'KademliaPeer') -> bool:
""" Add contact to _contact list in the right order. This will move the """ Add contact to _contact list in the right order. This will move the
@ -50,24 +64,25 @@ class KBucket:
self.peers.append(peer) self.peers.append(peer)
return True return True
else: else:
for i in range(len(self.peers)): for i, _ in enumerate(self.peers):
local_peer = self.peers[i] local_peer = self.peers[i]
if local_peer.node_id == peer.node_id: if local_peer.node_id == peer.node_id:
self.peers.remove(local_peer) self.peers.remove(local_peer)
self.peers.append(peer) self.peers.append(peer)
return True return True
if len(self.peers) < constants.K: if len(self.peers) < self.capacity:
self.peers.append(peer) self.peers.append(peer)
self.peer_in_routing_table_metric.labels("global").inc()
bits_colliding = utils.get_colliding_prefix_bits(peer.node_id, self._node_id)
self.peer_with_x_bit_colliding_metric.labels(amount=bits_colliding).inc()
return True return True
else: else:
return False return False
# raise BucketFull("No space in bucket to insert contact")
def get_peer(self, node_id: bytes) -> 'KademliaPeer': def get_peer(self, node_id: bytes) -> 'KademliaPeer':
for peer in self.peers: for peer in self.peers:
if peer.node_id == node_id: if peer.node_id == node_id:
return peer return peer
raise IndexError(node_id)
def get_peers(self, count=-1, exclude_contact=None, sort_distance_to=None) -> typing.List['KademliaPeer']: def get_peers(self, count=-1, exclude_contact=None, sort_distance_to=None) -> typing.List['KademliaPeer']:
""" Returns a list containing up to the first count number of contacts """ Returns a list containing up to the first count number of contacts
@ -124,6 +139,9 @@ class KBucket:
def remove_peer(self, peer: 'KademliaPeer') -> None: def remove_peer(self, peer: 'KademliaPeer') -> None:
self.peers.remove(peer) self.peers.remove(peer)
self.peer_in_routing_table_metric.labels("global").dec()
bits_colliding = utils.get_colliding_prefix_bits(peer.node_id, self._node_id)
self.peer_with_x_bit_colliding_metric.labels(amount=bits_colliding).dec()
def key_in_range(self, key: bytes) -> bool: def key_in_range(self, key: bytes) -> bool:
""" Tests whether the specified key (i.e. node ID) is in the range """ Tests whether the specified key (i.e. node ID) is in the range
@ -161,24 +179,36 @@ class TreeRoutingTable:
version of the Kademlia paper, in section 2.4. It does, however, use the version of the Kademlia paper, in section 2.4. It does, however, use the
ping RPC-based k-bucket eviction algorithm described in section 2.2 of ping RPC-based k-bucket eviction algorithm described in section 2.2 of
that paper. that paper.
BOOTSTRAP MODE: if set to True, we always add all peers. This is so a
bootstrap node does not get a bias towards its own node id and replies are
the best it can provide (joining peer knows its neighbors immediately).
Over time, this will need to be optimized so we use the disk as holding
everything in memory won't be feasible anymore.
See: https://github.com/bittorrent/bootstrap-dht
""" """
bucket_in_routing_table_metric = Gauge(
"buckets_in_routing_table", "Number of buckets on routing table", namespace="dht_node",
labelnames=("scope",)
)
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', parent_node_id: bytes, def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', parent_node_id: bytes,
split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX): split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX, is_bootstrap_node: bool = False):
self._loop = loop self._loop = loop
self._peer_manager = peer_manager self._peer_manager = peer_manager
self._parent_node_id = parent_node_id self._parent_node_id = parent_node_id
self._split_buckets_under_index = split_buckets_under_index self._split_buckets_under_index = split_buckets_under_index
self.buckets: typing.List[KBucket] = [ self.buckets: typing.List[KBucket] = [
KBucket( KBucket(
self._peer_manager, range_min=0, range_max=2 ** constants.HASH_BITS, node_id=self._parent_node_id self._peer_manager, range_min=0, range_max=2 ** constants.HASH_BITS, node_id=self._parent_node_id,
capacity=1 << 32 if is_bootstrap_node else constants.K
) )
] ]
def get_peers(self) -> typing.List['KademliaPeer']: def get_peers(self) -> typing.List['KademliaPeer']:
return list(itertools.chain.from_iterable(map(lambda bucket: bucket.peers, self.buckets))) return list(itertools.chain.from_iterable(map(lambda bucket: bucket.peers, self.buckets)))
def should_split(self, bucket_index: int, to_add: bytes) -> bool: def _should_split(self, bucket_index: int, to_add: bytes) -> bool:
# https://stackoverflow.com/questions/32129978/highly-unbalanced-kademlia-routing-table/32187456#32187456 # https://stackoverflow.com/questions/32129978/highly-unbalanced-kademlia-routing-table/32187456#32187456
if bucket_index < self._split_buckets_under_index: if bucket_index < self._split_buckets_under_index:
return True return True
@ -203,39 +233,32 @@ class TreeRoutingTable:
return [] return []
def get_peer(self, contact_id: bytes) -> 'KademliaPeer': def get_peer(self, contact_id: bytes) -> 'KademliaPeer':
""" return self.buckets[self._kbucket_index(contact_id)].get_peer(contact_id)
@raise IndexError: No contact with the specified contact ID is known
by this node
"""
return self.buckets[self.kbucket_index(contact_id)].get_peer(contact_id)
def get_refresh_list(self, start_index: int = 0, force: bool = False) -> typing.List[bytes]: def get_refresh_list(self, start_index: int = 0, force: bool = False) -> typing.List[bytes]:
bucket_index = start_index
refresh_ids = [] refresh_ids = []
now = int(self._loop.time()) for offset, _ in enumerate(self.buckets[start_index:]):
for bucket in self.buckets[start_index:]: refresh_ids.append(self._midpoint_id_in_bucket_range(start_index + offset))
if force or now - bucket.last_accessed >= constants.REFRESH_INTERVAL: # if we have 3 or fewer populated buckets get two random ids in the range of each to try and
to_search = self.midpoint_id_in_bucket_range(bucket_index) # populate/split the buckets further
refresh_ids.append(to_search) buckets_with_contacts = self.buckets_with_contacts()
bucket_index += 1 if buckets_with_contacts <= 3:
for i in range(buckets_with_contacts):
refresh_ids.append(self._random_id_in_bucket_range(i))
refresh_ids.append(self._random_id_in_bucket_range(i))
return refresh_ids return refresh_ids
def remove_peer(self, peer: 'KademliaPeer') -> None: def remove_peer(self, peer: 'KademliaPeer') -> None:
if not peer.node_id: if not peer.node_id:
return return
bucket_index = self.kbucket_index(peer.node_id) bucket_index = self._kbucket_index(peer.node_id)
try: try:
self.buckets[bucket_index].remove_peer(peer) self.buckets[bucket_index].remove_peer(peer)
self._join_buckets()
except ValueError: except ValueError:
return return
def touch_kbucket(self, key: bytes) -> None: def _kbucket_index(self, key: bytes) -> int:
self.touch_kbucket_by_index(self.kbucket_index(key))
def touch_kbucket_by_index(self, bucket_index: int):
self.buckets[bucket_index].last_accessed = int(self._loop.time())
def kbucket_index(self, key: bytes) -> int:
i = 0 i = 0
for bucket in self.buckets: for bucket in self.buckets:
if bucket.key_in_range(key): if bucket.key_in_range(key):
@ -244,19 +267,19 @@ class TreeRoutingTable:
i += 1 i += 1
return i return i
def random_id_in_bucket_range(self, bucket_index: int) -> bytes: def _random_id_in_bucket_range(self, bucket_index: int) -> bytes:
random_id = int(random.randrange(self.buckets[bucket_index].range_min, self.buckets[bucket_index].range_max)) random_id = int(random.randrange(self.buckets[bucket_index].range_min, self.buckets[bucket_index].range_max))
return Distance( return Distance(
self._parent_node_id self._parent_node_id
)(random_id.to_bytes(constants.HASH_LENGTH, 'big')).to_bytes(constants.HASH_LENGTH, 'big') )(random_id.to_bytes(constants.HASH_LENGTH, 'big')).to_bytes(constants.HASH_LENGTH, 'big')
def midpoint_id_in_bucket_range(self, bucket_index: int) -> bytes: def _midpoint_id_in_bucket_range(self, bucket_index: int) -> bytes:
half = int((self.buckets[bucket_index].range_max - self.buckets[bucket_index].range_min) // 2) half = int((self.buckets[bucket_index].range_max - self.buckets[bucket_index].range_min) // 2)
return Distance(self._parent_node_id)( return Distance(self._parent_node_id)(
int(self.buckets[bucket_index].range_min + half).to_bytes(constants.HASH_LENGTH, 'big') int(self.buckets[bucket_index].range_min + half).to_bytes(constants.HASH_LENGTH, 'big')
).to_bytes(constants.HASH_LENGTH, 'big') ).to_bytes(constants.HASH_LENGTH, 'big')
def split_bucket(self, old_bucket_index: int) -> None: def _split_bucket(self, old_bucket_index: int) -> None:
""" Splits the specified k-bucket into two new buckets which together """ Splits the specified k-bucket into two new buckets which together
cover the same range in the key/ID space cover the same range in the key/ID space
@ -279,8 +302,9 @@ class TreeRoutingTable:
# ...and remove them from the old bucket # ...and remove them from the old bucket
for contact in new_bucket.peers: for contact in new_bucket.peers:
old_bucket.remove_peer(contact) old_bucket.remove_peer(contact)
self.bucket_in_routing_table_metric.labels("global").set(len(self.buckets))
def join_buckets(self): def _join_buckets(self):
if len(self.buckets) == 1: if len(self.buckets) == 1:
return return
to_pop = [i for i, bucket in enumerate(self.buckets) if len(bucket) == 0] to_pop = [i for i, bucket in enumerate(self.buckets) if len(bucket) == 0]
@ -302,14 +326,8 @@ class TreeRoutingTable:
elif can_go_higher: elif can_go_higher:
self.buckets[bucket_index_to_pop + 1].range_min = bucket.range_min self.buckets[bucket_index_to_pop + 1].range_min = bucket.range_min
self.buckets.remove(bucket) self.buckets.remove(bucket)
return self.join_buckets() self.bucket_in_routing_table_metric.labels("global").set(len(self.buckets))
return self._join_buckets()
def contact_in_routing_table(self, address_tuple: typing.Tuple[str, int]) -> bool:
for bucket in self.buckets:
for contact in bucket.get_peers(sort_distance_to=False):
if address_tuple[0] == contact.address and address_tuple[1] == contact.udp_port:
return True
return False
def buckets_with_contacts(self) -> int: def buckets_with_contacts(self) -> int:
count = 0 count = 0
@ -317,3 +335,70 @@ class TreeRoutingTable:
if len(bucket) > 0: if len(bucket) > 0:
count += 1 count += 1
return count return count
async def add_peer(self, peer: 'KademliaPeer', probe: typing.Callable[['KademliaPeer'], typing.Awaitable]):
if not peer.node_id:
log.warning("Tried adding a peer with no node id!")
return False
for my_peer in self.get_peers():
if (my_peer.address, my_peer.udp_port) == (peer.address, peer.udp_port) and my_peer.node_id != peer.node_id:
self.remove_peer(my_peer)
self._join_buckets()
bucket_index = self._kbucket_index(peer.node_id)
if self.buckets[bucket_index].add_peer(peer):
return True
# The bucket is full; see if it can be split (by checking if its range includes the host node's node_id)
if self._should_split(bucket_index, peer.node_id):
self._split_bucket(bucket_index)
# Retry the insertion attempt
result = await self.add_peer(peer, probe)
self._join_buckets()
return result
else:
# We can't split the k-bucket
#
# The 13 page kademlia paper specifies that the least recently contacted node in the bucket
# shall be pinged. If it fails to reply it is replaced with the new contact. If the ping is successful
# the new contact is ignored and not added to the bucket (sections 2.2 and 2.4).
#
# A reasonable extension to this is BEP 0005, which extends the above:
#
# Not all nodes that we learn about are equal. Some are "good" and some are not.
# Many nodes using the DHT are able to send queries and receive responses,
# but are not able to respond to queries from other nodes. It is important that
# each node's routing table must contain only known good nodes. A good node is
# a node has responded to one of our queries within the last 15 minutes. A node
# is also good if it has ever responded to one of our queries and has sent us a
# query within the last 15 minutes. After 15 minutes of inactivity, a node becomes
# questionable. Nodes become bad when they fail to respond to multiple queries
# in a row. Nodes that we know are good are given priority over nodes with unknown status.
#
# When there are bad or questionable nodes in the bucket, the least recent is selected for
# potential replacement (BEP 0005). When all nodes in the bucket are fresh, the head (least recent)
# contact is selected as described in section 2.2 of the kademlia paper. In both cases the new contact
# is ignored if the pinged node replies.
not_good_contacts = self.buckets[bucket_index].get_bad_or_unknown_peers()
not_recently_replied = []
for my_peer in not_good_contacts:
last_replied = self._peer_manager.get_last_replied(my_peer.address, my_peer.udp_port)
if not last_replied or last_replied + 60 < self._loop.time():
not_recently_replied.append(my_peer)
if not_recently_replied:
to_replace = not_recently_replied[0]
else:
to_replace = self.buckets[bucket_index].peers[0]
last_replied = self._peer_manager.get_last_replied(to_replace.address, to_replace.udp_port)
if last_replied and last_replied + 60 > self._loop.time():
return False
log.debug("pinging %s:%s", to_replace.address, to_replace.udp_port)
try:
await probe(to_replace)
return False
except (asyncio.TimeoutError, RemoteException):
log.debug("Replacing dead contact in bucket %i: %s:%i with %s:%i ", bucket_index,
to_replace.address, to_replace.udp_port, peer.address, peer.udp_port)
if to_replace in self.buckets[bucket_index]:
self.buckets[bucket_index].remove_peer(to_replace)
return await self.add_peer(peer, probe)

View file

@ -144,7 +144,7 @@ class ErrorDatagram(KademliaDatagramBase):
self.response = response.decode() self.response = response.decode()
def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]: def _decode_datagram(datagram: bytes):
msg_types = { msg_types = {
REQUEST_TYPE: RequestDatagram, REQUEST_TYPE: RequestDatagram,
RESPONSE_TYPE: ResponseDatagram, RESPONSE_TYPE: ResponseDatagram,
@ -152,26 +152,36 @@ def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDa
} }
primitive: typing.Dict = bdecode(datagram) primitive: typing.Dict = bdecode(datagram)
if primitive[0] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
datagram_type = primitive[0] # pylint: disable=unsubscriptable-object converted = {
str(k).encode() if not isinstance(k, bytes) else k: v for k, v in primitive.items()
}
if converted[b'0'] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
datagram_type = converted[b'0'] # pylint: disable=unsubscriptable-object
else: else:
raise ValueError("invalid datagram type") raise ValueError("invalid datagram type")
datagram_class = msg_types[datagram_type] datagram_class = msg_types[datagram_type]
decoded = { decoded = {
k: primitive[i] # pylint: disable=unsubscriptable-object k: converted[str(i).encode()] # pylint: disable=unsubscriptable-object
for i, k in enumerate(datagram_class.required_fields) for i, k in enumerate(datagram_class.required_fields)
if i in primitive # pylint: disable=unsupported-membership-test if str(i).encode() in converted # pylint: disable=unsupported-membership-test
} }
for i, _ in enumerate(OPTIONAL_FIELDS): for i, _ in enumerate(OPTIONAL_FIELDS):
if i + OPTIONAL_ARG_OFFSET in primitive: if str(i + OPTIONAL_ARG_OFFSET).encode() in converted:
decoded[i + OPTIONAL_ARG_OFFSET] = primitive[i + OPTIONAL_ARG_OFFSET] decoded[i + OPTIONAL_ARG_OFFSET] = converted[str(i + OPTIONAL_ARG_OFFSET).encode()]
return decoded, datagram_class
def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]:
decoded, datagram_class = _decode_datagram(datagram)
return datagram_class(**decoded) return datagram_class(**decoded)
def make_compact_ip(address: str) -> bytearray: def make_compact_ip(address: str) -> bytearray:
compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), address.split('.'), bytearray()) compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), address.split('.'), bytearray())
if len(compact_ip) != 4: if len(compact_ip) != 4:
raise ValueError(f"invalid IPv4 length") raise ValueError("invalid IPv4 length")
return compact_ip return compact_ip
@ -180,7 +190,7 @@ def make_compact_address(node_id: bytes, address: str, port: int) -> bytearray:
if not 0 < port < 65536: if not 0 < port < 65536:
raise ValueError(f'Invalid port: {port}') raise ValueError(f'Invalid port: {port}')
if len(node_id) != constants.HASH_BITS // 8: if len(node_id) != constants.HASH_BITS // 8:
raise ValueError(f"invalid node node_id length") raise ValueError("invalid node node_id length")
return compact_ip + port.to_bytes(2, 'big') + node_id return compact_ip + port.to_bytes(2, 'big') + node_id
@ -191,5 +201,5 @@ def decode_compact_address(compact_address: bytes) -> typing.Tuple[bytes, str, i
if not 0 < port < 65536: if not 0 < port < 65536:
raise ValueError(f'Invalid port: {port}') raise ValueError(f'Invalid port: {port}')
if len(node_id) != constants.HASH_BITS // 8: if len(node_id) != constants.HASH_BITS // 8:
raise ValueError(f"invalid node node_id length") raise ValueError("invalid node node_id length")
return node_id, address, port return node_id, address, port

View file

@ -34,6 +34,11 @@ Code | Name | Message
**11x** | InputValue(ValueError) | Invalid argument value provided to command. **11x** | InputValue(ValueError) | Invalid argument value provided to command.
111 | GenericInputValue | The value '{value}' for argument '{argument}' is not valid. 111 | GenericInputValue | The value '{value}' for argument '{argument}' is not valid.
112 | InputValueIsNone | None or null is not valid value for argument '{argument}'. 112 | InputValueIsNone | None or null is not valid value for argument '{argument}'.
113 | ConflictingInputValue | Only '{first_argument}' or '{second_argument}' is allowed, not both.
114 | InputStringIsBlank | {argument} cannot be blank.
115 | EmptyPublishedFile | Cannot publish empty file: {file_path}
116 | MissingPublishedFile | File does not exist: {file_path}
117 | InvalidStreamURL | Invalid LBRY stream URL: '{url}' -- When an URL cannot be downloaded, such as '@Channel/' or a collection
**2xx** | Configuration | Configuration errors. **2xx** | Configuration | Configuration errors.
201 | ConfigWrite | Cannot write configuration file '{path}'. -- When writing the default config fails on startup, such as due to permission issues. 201 | ConfigWrite | Cannot write configuration file '{path}'. -- When writing the default config fails on startup, such as due to permission issues.
202 | ConfigRead | Cannot find provided configuration file '{path}'. -- Can't open the config file user provided via command line args. 202 | ConfigRead | Cannot find provided configuration file '{path}'. -- Can't open the config file user provided via command line args.
@ -51,12 +56,22 @@ Code | Name | Message
405 | ChannelKeyNotFound | Channel signing key not found. 405 | ChannelKeyNotFound | Channel signing key not found.
406 | ChannelKeyInvalid | Channel signing key is out of date. -- For example, channel was updated but you don't have the updated key. 406 | ChannelKeyInvalid | Channel signing key is out of date. -- For example, channel was updated but you don't have the updated key.
407 | DataDownload | Failed to download blob. *generic* 407 | DataDownload | Failed to download blob. *generic*
408 | PrivateKeyNotFound | Couldn't find private key for {key} '{value}'.
410 | Resolve | Failed to resolve '{url}'. 410 | Resolve | Failed to resolve '{url}'.
411 | ResolveTimeout | Failed to resolve '{url}' within the timeout. 411 | ResolveTimeout | Failed to resolve '{url}' within the timeout.
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{censor_id}'. 411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{censor_id}'.
420 | KeyFeeAboveMaxAllowed | {message} 420 | KeyFeeAboveMaxAllowed | {message}
421 | InvalidPassword | Password is invalid. 421 | InvalidPassword | Password is invalid.
422 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version. 422 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version.
423 | TooManyClaimSearchParameters | {key} cant have more than {limit} items.
424 | AlreadyPurchased | You already have a purchase for claim_id '{claim_id_hex}'. Use --allow-duplicate-purchase flag to override.
431 | ServerPaymentInvalidAddress | Invalid address from wallet server: '{address}' - skipping payment round.
432 | ServerPaymentWalletLocked | Cannot spend funds with locked wallet, skipping payment round.
433 | ServerPaymentFeeAboveMaxAllowed | Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.
434 | WalletNotLoaded | Wallet {wallet_id} is not loaded.
435 | WalletAlreadyLoaded | Wallet {wallet_path} is already loaded.
436 | WalletNotFound | Wallet not found at {wallet_path}.
437 | WalletAlreadyExists | Wallet {wallet_path} already exists, use `wallet_add` to load it.
**5xx** | Blob | **Blobs** **5xx** | Blob | **Blobs**
500 | BlobNotFound | Blob not found. 500 | BlobNotFound | Blob not found.
501 | BlobPermissionDenied | Permission denied to read blob. 501 | BlobPermissionDenied | Permission denied to read blob.

View file

@ -1,4 +1,4 @@
from .base import BaseError from .base import BaseError, claim_id
class UserInputError(BaseError): class UserInputError(BaseError):
@ -16,18 +16,22 @@ class CommandError(UserInputError):
class CommandDoesNotExistError(CommandError): class CommandDoesNotExistError(CommandError):
def __init__(self, command): def __init__(self, command):
self.command = command
super().__init__(f"Command '{command}' does not exist.") super().__init__(f"Command '{command}' does not exist.")
class CommandDeprecatedError(CommandError): class CommandDeprecatedError(CommandError):
def __init__(self, command): def __init__(self, command):
self.command = command
super().__init__(f"Command '{command}' is deprecated.") super().__init__(f"Command '{command}' is deprecated.")
class CommandInvalidArgumentError(CommandError): class CommandInvalidArgumentError(CommandError):
def __init__(self, argument, command): def __init__(self, argument, command):
self.argument = argument
self.command = command
super().__init__(f"Invalid argument '{argument}' to command '{command}'.") super().__init__(f"Invalid argument '{argument}' to command '{command}'.")
@ -37,6 +41,7 @@ class CommandTemporarilyUnavailableError(CommandError):
""" """
def __init__(self, command): def __init__(self, command):
self.command = command
super().__init__(f"Command '{command}' is temporarily unavailable.") super().__init__(f"Command '{command}' is temporarily unavailable.")
@ -46,6 +51,7 @@ class CommandPermanentlyUnavailableError(CommandError):
""" """
def __init__(self, command): def __init__(self, command):
self.command = command
super().__init__(f"Command '{command}' is permanently unavailable.") super().__init__(f"Command '{command}' is permanently unavailable.")
@ -58,15 +64,57 @@ class InputValueError(UserInputError, ValueError):
class GenericInputValueError(InputValueError): class GenericInputValueError(InputValueError):
def __init__(self, value, argument): def __init__(self, value, argument):
self.value = value
self.argument = argument
super().__init__(f"The value '{value}' for argument '{argument}' is not valid.") super().__init__(f"The value '{value}' for argument '{argument}' is not valid.")
class InputValueIsNoneError(InputValueError): class InputValueIsNoneError(InputValueError):
def __init__(self, argument): def __init__(self, argument):
self.argument = argument
super().__init__(f"None or null is not valid value for argument '{argument}'.") super().__init__(f"None or null is not valid value for argument '{argument}'.")
class ConflictingInputValueError(InputValueError):
def __init__(self, first_argument, second_argument):
self.first_argument = first_argument
self.second_argument = second_argument
super().__init__(f"Only '{first_argument}' or '{second_argument}' is allowed, not both.")
class InputStringIsBlankError(InputValueError):
def __init__(self, argument):
self.argument = argument
super().__init__(f"{argument} cannot be blank.")
class EmptyPublishedFileError(InputValueError):
def __init__(self, file_path):
self.file_path = file_path
super().__init__(f"Cannot publish empty file: {file_path}")
class MissingPublishedFileError(InputValueError):
def __init__(self, file_path):
self.file_path = file_path
super().__init__(f"File does not exist: {file_path}")
class InvalidStreamURLError(InputValueError):
"""
When an URL cannot be downloaded, such as '@Channel/' or a collection
"""
def __init__(self, url):
self.url = url
super().__init__(f"Invalid LBRY stream URL: '{url}'")
class ConfigurationError(BaseError): class ConfigurationError(BaseError):
""" """
Configuration errors. Configuration errors.
@ -79,6 +127,7 @@ class ConfigWriteError(ConfigurationError):
""" """
def __init__(self, path): def __init__(self, path):
self.path = path
super().__init__(f"Cannot write configuration file '{path}'.") super().__init__(f"Cannot write configuration file '{path}'.")
@ -88,6 +137,7 @@ class ConfigReadError(ConfigurationError):
""" """
def __init__(self, path): def __init__(self, path):
self.path = path
super().__init__(f"Cannot find provided configuration file '{path}'.") super().__init__(f"Cannot find provided configuration file '{path}'.")
@ -97,18 +147,21 @@ class ConfigParseError(ConfigurationError):
""" """
def __init__(self, path): def __init__(self, path):
self.path = path
super().__init__(f"Failed to parse the configuration file '{path}'.") super().__init__(f"Failed to parse the configuration file '{path}'.")
class ConfigMissingError(ConfigurationError): class ConfigMissingError(ConfigurationError):
def __init__(self, path): def __init__(self, path):
self.path = path
super().__init__(f"Configuration file '{path}' is missing setting that has no default / fallback.") super().__init__(f"Configuration file '{path}' is missing setting that has no default / fallback.")
class ConfigInvalidError(ConfigurationError): class ConfigInvalidError(ConfigurationError):
def __init__(self, path): def __init__(self, path):
self.path = path
super().__init__(f"Configuration file '{path}' has setting with invalid value.") super().__init__(f"Configuration file '{path}' has setting with invalid value.")
@ -185,27 +238,41 @@ class DataDownloadError(WalletError):
super().__init__("Failed to download blob. *generic*") super().__init__("Failed to download blob. *generic*")
class PrivateKeyNotFoundError(WalletError):
def __init__(self, key, value):
self.key = key
self.value = value
super().__init__(f"Couldn't find private key for {key} '{value}'.")
class ResolveError(WalletError): class ResolveError(WalletError):
def __init__(self, url): def __init__(self, url):
self.url = url
super().__init__(f"Failed to resolve '{url}'.") super().__init__(f"Failed to resolve '{url}'.")
class ResolveTimeoutError(WalletError): class ResolveTimeoutError(WalletError):
def __init__(self, url): def __init__(self, url):
self.url = url
super().__init__(f"Failed to resolve '{url}' within the timeout.") super().__init__(f"Failed to resolve '{url}' within the timeout.")
class ResolveCensoredError(WalletError): class ResolveCensoredError(WalletError):
def __init__(self, url, censor_id): def __init__(self, url, censor_id, censor_row):
self.url = url
self.censor_id = censor_id
self.censor_row = censor_row
super().__init__(f"Resolve of '{url}' was censored by channel with claim id '{censor_id}'.") super().__init__(f"Resolve of '{url}' was censored by channel with claim id '{censor_id}'.")
class KeyFeeAboveMaxAllowedError(WalletError): class KeyFeeAboveMaxAllowedError(WalletError):
def __init__(self, message): def __init__(self, message):
self.message = message
super().__init__(f"{message}") super().__init__(f"{message}")
@ -218,9 +285,78 @@ class InvalidPasswordError(WalletError):
class IncompatibleWalletServerError(WalletError): class IncompatibleWalletServerError(WalletError):
def __init__(self, server, port): def __init__(self, server, port):
self.server = server
self.port = port
super().__init__(f"'{server}:{port}' has an incompatibly old version.") super().__init__(f"'{server}:{port}' has an incompatibly old version.")
class TooManyClaimSearchParametersError(WalletError):
def __init__(self, key, limit):
self.key = key
self.limit = limit
super().__init__(f"{key} cant have more than {limit} items.")
class AlreadyPurchasedError(WalletError):
"""
allow-duplicate-purchase flag to override.
"""
def __init__(self, claim_id_hex):
self.claim_id_hex = claim_id_hex
super().__init__(f"You already have a purchase for claim_id '{claim_id_hex}'. Use")
class ServerPaymentInvalidAddressError(WalletError):
def __init__(self, address):
self.address = address
super().__init__(f"Invalid address from wallet server: '{address}' - skipping payment round.")
class ServerPaymentWalletLockedError(WalletError):
def __init__(self):
super().__init__("Cannot spend funds with locked wallet, skipping payment round.")
class ServerPaymentFeeAboveMaxAllowedError(WalletError):
def __init__(self, daily_fee, max_fee):
self.daily_fee = daily_fee
self.max_fee = max_fee
super().__init__(f"Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.")
class WalletNotLoadedError(WalletError):
def __init__(self, wallet_id):
self.wallet_id = wallet_id
super().__init__(f"Wallet {wallet_id} is not loaded.")
class WalletAlreadyLoadedError(WalletError):
def __init__(self, wallet_path):
self.wallet_path = wallet_path
super().__init__(f"Wallet {wallet_path} is already loaded.")
class WalletNotFoundError(WalletError):
def __init__(self, wallet_path):
self.wallet_path = wallet_path
super().__init__(f"Wallet not found at {wallet_path}.")
class WalletAlreadyExistsError(WalletError):
def __init__(self, wallet_path):
self.wallet_path = wallet_path
super().__init__(f"Wallet {wallet_path} already exists, use `wallet_add` to load it.")
class BlobError(BaseError): class BlobError(BaseError):
""" """
**Blobs** **Blobs**
@ -278,30 +414,35 @@ class DownloadCancelledError(BlobError):
class DownloadSDTimeoutError(BlobError): class DownloadSDTimeoutError(BlobError):
def __init__(self, download): def __init__(self, download):
self.download = download
super().__init__(f"Failed to download sd blob {download} within timeout.") super().__init__(f"Failed to download sd blob {download} within timeout.")
class DownloadDataTimeoutError(BlobError): class DownloadDataTimeoutError(BlobError):
def __init__(self, download): def __init__(self, download):
self.download = download
super().__init__(f"Failed to download data blobs for sd hash {download} within timeout.") super().__init__(f"Failed to download data blobs for sd hash {download} within timeout.")
class InvalidStreamDescriptorError(BlobError): class InvalidStreamDescriptorError(BlobError):
def __init__(self, message): def __init__(self, message):
self.message = message
super().__init__(f"{message}") super().__init__(f"{message}")
class InvalidDataError(BlobError): class InvalidDataError(BlobError):
def __init__(self, message): def __init__(self, message):
self.message = message
super().__init__(f"{message}") super().__init__(f"{message}")
class InvalidBlobHashError(BlobError): class InvalidBlobHashError(BlobError):
def __init__(self, message): def __init__(self, message):
self.message = message
super().__init__(f"{message}") super().__init__(f"{message}")
@ -314,12 +455,14 @@ class ComponentError(BaseError):
class ComponentStartConditionNotMetError(ComponentError): class ComponentStartConditionNotMetError(ComponentError):
def __init__(self, components): def __init__(self, components):
self.components = components
super().__init__(f"Unresolved dependencies for: {components}") super().__init__(f"Unresolved dependencies for: {components}")
class ComponentsNotStartedError(ComponentError): class ComponentsNotStartedError(ComponentError):
def __init__(self, message): def __init__(self, message):
self.message = message
super().__init__(f"{message}") super().__init__(f"{message}")
@ -332,16 +475,20 @@ class CurrencyExchangeError(BaseError):
class InvalidExchangeRateResponseError(CurrencyExchangeError): class InvalidExchangeRateResponseError(CurrencyExchangeError):
def __init__(self, source, reason): def __init__(self, source, reason):
self.source = source
self.reason = reason
super().__init__(f"Failed to get exchange rate from {source}: {reason}") super().__init__(f"Failed to get exchange rate from {source}: {reason}")
class CurrencyConversionError(CurrencyExchangeError): class CurrencyConversionError(CurrencyExchangeError):
def __init__(self, message): def __init__(self, message):
self.message = message
super().__init__(f"{message}") super().__init__(f"{message}")
class InvalidCurrencyError(CurrencyExchangeError): class InvalidCurrencyError(CurrencyExchangeError):
def __init__(self, currency): def __init__(self, currency):
self.currency = currency
super().__init__(f"Invalid currency: {currency} is not a supported currency.") super().__init__(f"Invalid currency: {currency} is not a supported currency.")

View file

@ -1,2 +1,9 @@
from binascii import hexlify
def claim_id(claim_hash):
return hexlify(claim_hash[::-1]).decode()
class BaseError(Exception): class BaseError(Exception):
pass pass

View file

@ -13,10 +13,12 @@ class {name}({parents}):{doc}
""" """
INIT = """ INIT = """
def __init__({args}): def __init__({args}):{fields}
super().__init__({format}"{message}") super().__init__({format}"{message}")
""" """
FUNCTIONS = ['claim_id']
class ErrorClass: class ErrorClass:
@ -50,10 +52,20 @@ class ErrorClass:
def get_arguments(self): def get_arguments(self):
args = ['self'] args = ['self']
for arg in re.findall('{([a-z0-1_]+)}', self.message): for arg in re.findall('{([a-z0-1_()]+)}', self.message):
for func in FUNCTIONS:
if arg.startswith(f'{func}('):
arg = arg[len(f'{func}('):-1]
break
args.append(arg) args.append(arg)
return args return args
@staticmethod
def get_fields(args):
if len(args) > 1:
return ''.join(f'\n{INDENT*2}self.{field} = {field}' for field in args[1:])
return ''
@staticmethod @staticmethod
def get_doc_string(doc): def get_doc_string(doc):
if doc: if doc:
@ -69,7 +81,8 @@ class ErrorClass:
args = self.get_arguments() args = self.get_arguments()
if self.is_leaf: if self.is_leaf:
out.write((CLASS + INIT).format( out.write((CLASS + INIT).format(
name=self.class_name, parents=', '.join(parents), args=', '.join(args), name=self.class_name, parents=', '.join(parents),
args=', '.join(args), fields=self.get_fields(args),
message=self.message, doc=self.get_doc_string(self.comment), format='f' if len(args) > 1 else '' message=self.message, doc=self.get_doc_string(self.comment), format='f' if len(args) > 1 else ''
)) ))
else: else:
@ -102,7 +115,7 @@ def find_parent(stack, child):
def generate(out): def generate(out):
out.write('from .base import BaseError\n') out.write(f"from .base import BaseError, {', '.join(FUNCTIONS)}\n")
stack = {} stack = {}
for error in get_errors(): for error in get_errors():
error.render(out, find_parent(stack, error)) error.render(out, find_parent(stack, error))

View file

@ -3,7 +3,6 @@ import sys
import shutil import shutil
import signal import signal
import pathlib import pathlib
import platform
import json import json
import asyncio import asyncio
import argparse import argparse
@ -15,7 +14,6 @@ from aiohttp.web import GracefulExit
from docopt import docopt from docopt import docopt
from lbry import __version__ as lbrynet_version from lbry import __version__ as lbrynet_version
from lbry.extras.daemon.loggly_handler import get_loggly_handler
from lbry.extras.daemon.daemon import Daemon from lbry.extras.daemon.daemon import Daemon
from lbry.conf import Config, CLIConfig from lbry.conf import Config, CLIConfig
@ -103,7 +101,7 @@ class ArgumentParser(argparse.ArgumentParser):
self._optionals.title = 'Options' self._optionals.title = 'Options'
if group_name is None: if group_name is None:
self.epilog = ( self.epilog = (
f"Run 'lbrynet COMMAND --help' for more information on a command or group." "Run 'lbrynet COMMAND --help' for more information on a command or group."
) )
else: else:
self.epilog = ( self.epilog = (
@ -228,6 +226,9 @@ def get_argument_parser():
def ensure_directory_exists(path: str): def ensure_directory_exists(path: str):
if not os.path.isdir(path): if not os.path.isdir(path):
pathlib.Path(path).mkdir(parents=True, exist_ok=True) pathlib.Path(path).mkdir(parents=True, exist_ok=True)
use_effective_ids = os.access in os.supports_effective_ids
if not os.access(path, os.W_OK, effective_ids=use_effective_ids):
raise PermissionError(f"The following directory is not writable: {path}")
LOG_MODULES = 'lbry', 'aioupnp' LOG_MODULES = 'lbry', 'aioupnp'
@ -256,16 +257,8 @@ def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config
else: else:
logger.getChild('lbry').setLevel(logging.DEBUG) logger.getChild('lbry').setLevel(logging.DEBUG)
if conf.share_usage_data:
loggly_handler = get_loggly_handler()
loggly_handler.setLevel(logging.ERROR)
logger.getChild('lbry').addHandler(loggly_handler)
def run_daemon(args: argparse.Namespace, conf: Config): def run_daemon(args: argparse.Namespace, conf: Config):
if sys.version_info < (3, 8) and platform.system() == "Windows":
# TODO: remove after we move to requiring Python 3.8
asyncio.set_event_loop(asyncio.ProactorEventLoop())
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
if args.verbose is not None: if args.verbose is not None:
loop.set_debug(True) loop.set_debug(True)

View file

@ -18,6 +18,7 @@ DOWNLOAD_STARTED = 'Download Started'
DOWNLOAD_ERRORED = 'Download Errored' DOWNLOAD_ERRORED = 'Download Errored'
DOWNLOAD_FINISHED = 'Download Finished' DOWNLOAD_FINISHED = 'Download Finished'
HEARTBEAT = 'Heartbeat' HEARTBEAT = 'Heartbeat'
DISK_SPACE = 'Disk Space'
CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon
NEW_CHANNEL = 'New Channel' NEW_CHANNEL = 'New Channel'
CREDITS_SENT = 'Credits Sent' CREDITS_SENT = 'Credits Sent'
@ -66,7 +67,7 @@ def _download_properties(conf: Config, external_ip: str, resolve_duration: float
"node_rpc_timeout": conf.node_rpc_timeout, "node_rpc_timeout": conf.node_rpc_timeout,
"peer_connect_timeout": conf.peer_connect_timeout, "peer_connect_timeout": conf.peer_connect_timeout,
"blob_download_timeout": conf.blob_download_timeout, "blob_download_timeout": conf.blob_download_timeout,
"use_fixed_peers": len(conf.reflector_servers) > 0, "use_fixed_peers": len(conf.fixed_peers) > 0,
"fixed_peer_delay": fixed_peer_delay, "fixed_peer_delay": fixed_peer_delay,
"added_fixed_peers": added_fixed_peers, "added_fixed_peers": added_fixed_peers,
"active_peer_count": active_peer_count, "active_peer_count": active_peer_count,
@ -110,7 +111,6 @@ class AnalyticsManager:
self.cookies = {} self.cookies = {}
self.url = ANALYTICS_ENDPOINT self.url = ANALYTICS_ENDPOINT
self._write_key = utils.deobfuscate(ANALYTICS_TOKEN) self._write_key = utils.deobfuscate(ANALYTICS_TOKEN)
self._enabled = conf.share_usage_data
self._tracked_data = collections.defaultdict(list) self._tracked_data = collections.defaultdict(list)
self.context = _make_context(system_info.get_platform()) self.context = _make_context(system_info.get_platform())
self.installation_id = installation_id self.installation_id = installation_id
@ -118,20 +118,24 @@ class AnalyticsManager:
self.task: typing.Optional[asyncio.Task] = None self.task: typing.Optional[asyncio.Task] = None
self.external_ip: typing.Optional[str] = None self.external_ip: typing.Optional[str] = None
@property
def enabled(self):
return self.conf.share_usage_data
@property @property
def is_started(self): def is_started(self):
return self.task is not None return self.task is not None
async def start(self): async def start(self):
if self._enabled and self.task is None: if self.task is None:
self.external_ip = await utils.get_external_ip()
self.task = asyncio.create_task(self.run()) self.task = asyncio.create_task(self.run())
async def run(self): async def run(self):
while True: while True:
if self.enabled:
self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
await self._send_heartbeat() await self._send_heartbeat()
await asyncio.sleep(1800) await asyncio.sleep(1800)
self.external_ip = await utils.get_external_ip()
def stop(self): def stop(self):
if self.task is not None and not self.task.done(): if self.task is not None and not self.task.done():
@ -154,7 +158,7 @@ class AnalyticsManager:
async def track(self, event: typing.Dict): async def track(self, event: typing.Dict):
"""Send a single tracking event""" """Send a single tracking event"""
if self._enabled: if self.enabled:
log.debug('Sending track event: %s', event) log.debug('Sending track event: %s', event)
await self._post(event) await self._post(event)
@ -166,6 +170,15 @@ class AnalyticsManager:
}) })
) )
async def send_disk_space_used(self, storage_used, storage_limit, is_from_network_quota):
await self.track(
self._event(DISK_SPACE, {
'used': storage_used,
'limit': storage_limit,
'from_network_quota': is_from_network_quota
})
)
async def send_server_startup(self): async def send_server_startup(self):
await self.track(self._event(SERVER_STARTUP)) await self.track(self._event(SERVER_STARTUP))

View file

@ -1,5 +1,5 @@
from lbry.conf import Config
from lbry.extras.cli import execute_command from lbry.extras.cli import execute_command
from lbry.conf import Config
def daemon_rpc(conf: Config, method: str, **kwargs): def daemon_rpc(conf: Config, method: str, **kwargs):

View file

@ -1,66 +0,0 @@
import logging
import time
import hashlib
import binascii
import ecdsa
from lbry import utils
from lbry.crypto.hash import sha256
from lbry.wallet.transaction import Output
log = logging.getLogger(__name__)
def get_encoded_signature(signature):
signature = signature.encode() if isinstance(signature, str) else signature
r = int(signature[:int(len(signature) / 2)], 16)
s = int(signature[int(len(signature) / 2):], 16)
return ecdsa.util.sigencode_der(r, s, len(signature) * 4)
def cid2hash(claim_id: str) -> bytes:
return binascii.unhexlify(claim_id.encode())[::-1]
def is_comment_signed_by_channel(comment: dict, channel: Output, abandon=False):
if isinstance(channel, Output):
try:
signing_field = comment['comment_id'] if abandon else comment['comment']
pieces = [
comment['signing_ts'].encode(),
cid2hash(comment['channel_id']),
signing_field.encode()
]
return Output.is_signature_valid(
get_encoded_signature(comment['signature']),
sha256(b''.join(pieces)),
channel.claim.channel.public_key_bytes
)
except KeyError:
pass
return False
def sign_comment(comment: dict, channel: Output, abandon=False):
timestamp = str(int(time.time()))
signing_field = comment['comment_id'] if abandon else comment['comment']
pieces = [timestamp.encode(), channel.claim_hash, signing_field.encode()]
digest = sha256(b''.join(pieces))
signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
comment.update({
'signature': binascii.hexlify(signature).decode(),
'signing_ts': timestamp
})
async def jsonrpc_post(url: str, method: str, params: dict = None, **kwargs) -> any:
params = params or {}
params.update(kwargs)
json_body = {'jsonrpc': '2.0', 'id': None, 'method': method, 'params': params}
async with utils.aiohttp_request('POST', url, json=json_body) as response:
try:
result = await response.json()
return result['result'] if 'result' in result else result
except Exception as cte:
log.exception('Unable to decode response from server: %s', cte)
return await response.text()

View file

@ -37,7 +37,7 @@ class Component(metaclass=ComponentType):
def running(self): def running(self):
return self._running return self._running
async def get_status(self): async def get_status(self): # pylint: disable=no-self-use
return return
async def start(self): async def start(self):

View file

@ -42,7 +42,7 @@ class ComponentManager:
self.analytics_manager = analytics_manager self.analytics_manager = analytics_manager
self.component_classes = {} self.component_classes = {}
self.components = set() self.components = set()
self.started = asyncio.Event(loop=self.loop) self.started = asyncio.Event()
self.peer_manager = peer_manager or PeerManager(asyncio.get_event_loop_policy().get_event_loop()) self.peer_manager = peer_manager or PeerManager(asyncio.get_event_loop_policy().get_event_loop())
for component_name, component_class in self.default_component_classes.items(): for component_name, component_class in self.default_component_classes.items():
@ -118,7 +118,7 @@ class ComponentManager:
component._setup() for component in stage if not component.running component._setup() for component in stage if not component.running
] ]
if needing_start: if needing_start:
await asyncio.wait(needing_start) await asyncio.wait(map(asyncio.create_task, needing_start))
self.started.set() self.started.set()
async def stop(self): async def stop(self):
@ -131,7 +131,7 @@ class ComponentManager:
component._stop() for component in stage if component.running component._stop() for component in stage if component.running
] ]
if needing_stop: if needing_stop:
await asyncio.wait(needing_stop) await asyncio.wait(map(asyncio.create_task, needing_stop))
def all_components_running(self, *component_names): def all_components_running(self, *component_names):
""" """
@ -158,11 +158,14 @@ class ComponentManager:
for component in self.components for component in self.components
} }
def get_component(self, component_name): def get_actual_component(self, component_name):
for component in self.components: for component in self.components:
if component.component_name == component_name: if component.component_name == component_name:
return component.component return component
raise NameError(component_name) raise NameError(component_name)
def get_component(self, component_name):
return self.get_actual_component(component_name).component
def has_component(self, component_name): def has_component(self, component_name):
return any(component for component in self.components if component_name == component.component_name) return any(component for component in self.components if component_name == component.component_name)

View file

@ -4,6 +4,7 @@ import asyncio
import logging import logging
import binascii import binascii
import typing import typing
import base58 import base58
from aioupnp import __version__ as aioupnp_version from aioupnp import __version__ as aioupnp_version
@ -15,12 +16,19 @@ from lbry.dht.node import Node
from lbry.dht.peer import is_valid_public_ipv4 from lbry.dht.peer import is_valid_public_ipv4
from lbry.dht.blob_announcer import BlobAnnouncer from lbry.dht.blob_announcer import BlobAnnouncer
from lbry.blob.blob_manager import BlobManager from lbry.blob.blob_manager import BlobManager
from lbry.blob.disk_space_manager import DiskSpaceManager
from lbry.blob_exchange.server import BlobServer from lbry.blob_exchange.server import BlobServer
from lbry.stream.background_downloader import BackgroundDownloader
from lbry.stream.stream_manager import StreamManager from lbry.stream.stream_manager import StreamManager
from lbry.file.file_manager import FileManager
from lbry.extras.daemon.component import Component from lbry.extras.daemon.component import Component
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
from lbry.extras.daemon.storage import SQLiteStorage from lbry.extras.daemon.storage import SQLiteStorage
from lbry.torrent.torrent_manager import TorrentManager
from lbry.wallet import WalletManager from lbry.wallet import WalletManager
from lbry.wallet.usage_payment import WalletServerPayer
from lbry.torrent.tracker import TrackerClient
from lbry.torrent.session import TorrentSession
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -29,12 +37,17 @@ log = logging.getLogger(__name__)
DATABASE_COMPONENT = "database" DATABASE_COMPONENT = "database"
BLOB_COMPONENT = "blob_manager" BLOB_COMPONENT = "blob_manager"
WALLET_COMPONENT = "wallet" WALLET_COMPONENT = "wallet"
WALLET_SERVER_PAYMENTS_COMPONENT = "wallet_server_payments"
DHT_COMPONENT = "dht" DHT_COMPONENT = "dht"
HASH_ANNOUNCER_COMPONENT = "hash_announcer" HASH_ANNOUNCER_COMPONENT = "hash_announcer"
STREAM_MANAGER_COMPONENT = "stream_manager" FILE_MANAGER_COMPONENT = "file_manager"
DISK_SPACE_COMPONENT = "disk_space"
BACKGROUND_DOWNLOADER_COMPONENT = "background_downloader"
PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server" PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
UPNP_COMPONENT = "upnp" UPNP_COMPONENT = "upnp"
EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager" EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager"
TRACKER_ANNOUNCER_COMPONENT = "tracker_announcer_component"
LIBTORRENT_COMPONENT = "libtorrent_component"
class DatabaseComponent(Component): class DatabaseComponent(Component):
@ -50,7 +63,7 @@ class DatabaseComponent(Component):
@staticmethod @staticmethod
def get_current_db_revision(): def get_current_db_revision():
return 14 return 15
@property @property
def revision_filename(self): def revision_filename(self):
@ -110,15 +123,17 @@ class WalletComponent(Component):
async def get_status(self): async def get_status(self):
if self.wallet_manager is None: if self.wallet_manager is None:
return return
session_pool = self.wallet_manager.ledger.network.session_pool is_connected = self.wallet_manager.ledger.network.is_connected
sessions = session_pool.sessions sessions = []
connected = None connected = None
if self.wallet_manager.ledger.network.client: if is_connected:
addr_and_port = self.wallet_manager.ledger.network.client.server_address_and_port addr, port = self.wallet_manager.ledger.network.client.server
if addr_and_port: connected = f"{addr}:{port}"
connected = f"{addr_and_port[0]}:{addr_and_port[1]}" sessions.append(self.wallet_manager.ledger.network.client)
result = { result = {
'connected': connected, 'connected': connected,
'connected_features': self.wallet_manager.ledger.network.server_features,
'servers': [ 'servers': [
{ {
'host': session.server[0], 'host': session.server[0],
@ -127,8 +142,8 @@ class WalletComponent(Component):
'availability': session.available, 'availability': session.available,
} for session in sessions } for session in sessions
], ],
'known_servers': len(sessions), 'known_servers': len(self.wallet_manager.ledger.network.known_hubs),
'available_servers': len(list(session_pool.available_sessions)) 'available_servers': 1 if is_connected else 0
} }
if self.wallet_manager.ledger.network.remote_height: if self.wallet_manager.ledger.network.remote_height:
@ -140,7 +155,7 @@ class WalletComponent(Component):
progress = min(max(math.ceil(float(download_height) / float(target_height) * 100), 0), 100) progress = min(max(math.ceil(float(download_height) / float(target_height) * 100), 0), 100)
else: else:
progress = 100 progress = 100
best_hash = self.wallet_manager.get_best_blockhash() best_hash = await self.wallet_manager.get_best_blockhash()
result.update({ result.update({
'headers_synchronization_progress': progress, 'headers_synchronization_progress': progress,
'blocks': max(local_height, 0), 'blocks': max(local_height, 0),
@ -160,6 +175,34 @@ class WalletComponent(Component):
self.wallet_manager = None self.wallet_manager = None
class WalletServerPaymentsComponent(Component):
component_name = WALLET_SERVER_PAYMENTS_COMPONENT
depends_on = [WALLET_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
self.usage_payment_service = WalletServerPayer(
max_fee=self.conf.max_wallet_server_fee, analytics_manager=self.component_manager.analytics_manager,
)
@property
def component(self) -> typing.Optional[WalletServerPayer]:
return self.usage_payment_service
async def start(self):
wallet_manager = self.component_manager.get_component(WALLET_COMPONENT)
await self.usage_payment_service.start(wallet_manager.ledger, wallet_manager.default_wallet)
async def stop(self):
await self.usage_payment_service.stop()
async def get_status(self):
return {
'max_fee': self.usage_payment_service.max_fee,
'running': self.usage_payment_service.running
}
class BlobComponent(Component): class BlobComponent(Component):
component_name = BLOB_COMPONENT component_name = BLOB_COMPONENT
depends_on = [DATABASE_COMPONENT] depends_on = [DATABASE_COMPONENT]
@ -236,7 +279,7 @@ class DHTComponent(Component):
external_ip = upnp_component.external_ip external_ip = upnp_component.external_ip
storage = self.component_manager.get_component(DATABASE_COMPONENT) storage = self.component_manager.get_component(DATABASE_COMPONENT)
if not external_ip: if not external_ip:
external_ip = await utils.get_external_ip() external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
if not external_ip: if not external_ip:
log.warning("failed to get external ip") log.warning("failed to get external ip")
@ -250,6 +293,7 @@ class DHTComponent(Component):
peer_port=self.external_peer_port, peer_port=self.external_peer_port,
rpc_timeout=self.conf.node_rpc_timeout, rpc_timeout=self.conf.node_rpc_timeout,
split_buckets_under_index=self.conf.split_buckets_under_index, split_buckets_under_index=self.conf.split_buckets_under_index,
is_bootstrap_node=self.conf.is_bootstrap_node,
storage=storage storage=storage
) )
self.dht_node.start(self.conf.network_interface, self.conf.known_dht_nodes) self.dht_node.start(self.conf.network_interface, self.conf.known_dht_nodes)
@ -288,23 +332,23 @@ class HashAnnouncerComponent(Component):
} }
class StreamManagerComponent(Component): class FileManagerComponent(Component):
component_name = STREAM_MANAGER_COMPONENT component_name = FILE_MANAGER_COMPONENT
depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT] depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT]
def __init__(self, component_manager): def __init__(self, component_manager):
super().__init__(component_manager) super().__init__(component_manager)
self.stream_manager: typing.Optional[StreamManager] = None self.file_manager: typing.Optional[FileManager] = None
@property @property
def component(self) -> typing.Optional[StreamManager]: def component(self) -> typing.Optional[FileManager]:
return self.stream_manager return self.file_manager
async def get_status(self): async def get_status(self):
if not self.stream_manager: if not self.file_manager:
return return
return { return {
'managed_files': len(self.stream_manager.streams), 'managed_files': len(self.file_manager.get_filtered()),
} }
async def start(self): async def start(self):
@ -315,14 +359,148 @@ class StreamManagerComponent(Component):
if self.component_manager.has_component(DHT_COMPONENT) else None if self.component_manager.has_component(DHT_COMPONENT) else None
log.info('Starting the file manager') log.info('Starting the file manager')
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
self.stream_manager = StreamManager( self.file_manager = FileManager(
loop, self.conf, blob_manager, wallet, storage, node, self.component_manager.analytics_manager loop, self.conf, wallet, storage, self.component_manager.analytics_manager
) )
await self.stream_manager.start() self.file_manager.source_managers['stream'] = StreamManager(
loop, self.conf, blob_manager, wallet, storage, node,
)
if self.component_manager.has_component(LIBTORRENT_COMPONENT):
torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT)
self.file_manager.source_managers['torrent'] = TorrentManager(
loop, self.conf, torrent, storage, self.component_manager.analytics_manager
)
await self.file_manager.start()
log.info('Done setting up file manager') log.info('Done setting up file manager')
async def stop(self): async def stop(self):
self.stream_manager.stop() await self.file_manager.stop()
class BackgroundDownloaderComponent(Component):
MIN_PREFIX_COLLIDING_BITS = 8
component_name = BACKGROUND_DOWNLOADER_COMPONENT
depends_on = [DATABASE_COMPONENT, BLOB_COMPONENT, DISK_SPACE_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
self.background_task: typing.Optional[asyncio.Task] = None
self.download_loop_delay_seconds = 60
self.ongoing_download: typing.Optional[asyncio.Task] = None
self.space_manager: typing.Optional[DiskSpaceManager] = None
self.blob_manager: typing.Optional[BlobManager] = None
self.background_downloader: typing.Optional[BackgroundDownloader] = None
self.dht_node: typing.Optional[Node] = None
self.space_available: typing.Optional[int] = None
@property
def is_busy(self):
return bool(self.ongoing_download and not self.ongoing_download.done())
@property
def component(self) -> 'BackgroundDownloaderComponent':
return self
async def get_status(self):
return {'running': self.background_task is not None and not self.background_task.done(),
'available_free_space_mb': self.space_available,
'ongoing_download': self.is_busy}
async def download_blobs_in_background(self):
while True:
self.space_available = await self.space_manager.get_free_space_mb(True)
if not self.is_busy and self.space_available > 10:
self._download_next_close_blob_hash()
await asyncio.sleep(self.download_loop_delay_seconds)
def _download_next_close_blob_hash(self):
node_id = self.dht_node.protocol.node_id
for blob_hash in self.dht_node.stored_blob_hashes:
if blob_hash.hex() in self.blob_manager.completed_blob_hashes:
continue
if utils.get_colliding_prefix_bits(node_id, blob_hash) >= self.MIN_PREFIX_COLLIDING_BITS:
self.ongoing_download = asyncio.create_task(self.background_downloader.download_blobs(blob_hash.hex()))
return
async def start(self):
self.space_manager: DiskSpaceManager = self.component_manager.get_component(DISK_SPACE_COMPONENT)
if not self.component_manager.has_component(DHT_COMPONENT):
return
self.dht_node = self.component_manager.get_component(DHT_COMPONENT)
self.blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
storage = self.component_manager.get_component(DATABASE_COMPONENT)
self.background_downloader = BackgroundDownloader(self.conf, storage, self.blob_manager, self.dht_node)
self.background_task = asyncio.create_task(self.download_blobs_in_background())
async def stop(self):
if self.ongoing_download and not self.ongoing_download.done():
self.ongoing_download.cancel()
if self.background_task:
self.background_task.cancel()
class DiskSpaceComponent(Component):
component_name = DISK_SPACE_COMPONENT
depends_on = [DATABASE_COMPONENT, BLOB_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
self.disk_space_manager: typing.Optional[DiskSpaceManager] = None
@property
def component(self) -> typing.Optional[DiskSpaceManager]:
return self.disk_space_manager
async def get_status(self):
if self.disk_space_manager:
space_used = await self.disk_space_manager.get_space_used_mb(cached=True)
return {
'total_used_mb': space_used['total'],
'published_blobs_storage_used_mb': space_used['private_storage'],
'content_blobs_storage_used_mb': space_used['content_storage'],
'seed_blobs_storage_used_mb': space_used['network_storage'],
'running': self.disk_space_manager.running,
}
return {'space_used': '0', 'network_seeding_space_used': '0', 'running': False}
async def start(self):
db = self.component_manager.get_component(DATABASE_COMPONENT)
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
self.disk_space_manager = DiskSpaceManager(
self.conf, db, blob_manager,
analytics=self.component_manager.analytics_manager
)
await self.disk_space_manager.start()
async def stop(self):
await self.disk_space_manager.stop()
class TorrentComponent(Component):
component_name = LIBTORRENT_COMPONENT
def __init__(self, component_manager):
super().__init__(component_manager)
self.torrent_session = None
@property
def component(self) -> typing.Optional[TorrentSession]:
return self.torrent_session
async def get_status(self):
if not self.torrent_session:
return
return {
'running': True, # TODO: what to return here?
}
async def start(self):
self.torrent_session = TorrentSession(asyncio.get_event_loop(), None)
await self.torrent_session.bind() # TODO: specify host/port
async def stop(self):
if self.torrent_session:
await self.torrent_session.pause()
class PeerProtocolServerComponent(Component): class PeerProtocolServerComponent(Component):
@ -373,7 +551,7 @@ class UPnPComponent(Component):
while True: while True:
if now: if now:
await self._maintain_redirects() await self._maintain_redirects()
await asyncio.sleep(360, loop=self.component_manager.loop) await asyncio.sleep(360)
async def _maintain_redirects(self): async def _maintain_redirects(self):
# setup the gateway if necessary # setup the gateway if necessary
@ -382,8 +560,6 @@ class UPnPComponent(Component):
self.upnp = await UPnP.discover(loop=self.component_manager.loop) self.upnp = await UPnP.discover(loop=self.component_manager.loop)
log.info("found upnp gateway: %s", self.upnp.gateway.manufacturer_string) log.info("found upnp gateway: %s", self.upnp.gateway.manufacturer_string)
except Exception as err: except Exception as err:
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
raise
log.warning("upnp discovery failed: %s", err) log.warning("upnp discovery failed: %s", err)
self.upnp = None self.upnp = None
@ -398,11 +574,15 @@ class UPnPComponent(Component):
pass pass
if external_ip and not is_valid_public_ipv4(external_ip): if external_ip and not is_valid_public_ipv4(external_ip):
log.warning("UPnP returned a private/reserved ip - %s, checking lbry.com fallback", external_ip) log.warning("UPnP returned a private/reserved ip - %s, checking lbry.com fallback", external_ip)
external_ip = await utils.get_external_ip() external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
if self.external_ip and self.external_ip != external_ip: if self.external_ip and self.external_ip != external_ip:
log.info("external ip changed from %s to %s", self.external_ip, external_ip) log.info("external ip changed from %s to %s", self.external_ip, external_ip)
if external_ip: if external_ip:
self.external_ip = external_ip self.external_ip = external_ip
dht_component = self.component_manager.get_component(DHT_COMPONENT)
if dht_component:
dht_node = dht_component.component
dht_node.protocol.external_ip = external_ip
# assert self.external_ip is not None # TODO: handle going/starting offline # assert self.external_ip is not None # TODO: handle going/starting offline
if not self.upnp_redirects and self.upnp: # setup missing redirects if not self.upnp_redirects and self.upnp: # setup missing redirects
@ -456,13 +636,15 @@ class UPnPComponent(Component):
async def start(self): async def start(self):
log.info("detecting external ip") log.info("detecting external ip")
if not self.use_upnp: if not self.use_upnp:
self.external_ip = await utils.get_external_ip() self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
return return
success = False success = False
await self._maintain_redirects() await self._maintain_redirects()
if self.upnp: if self.upnp:
if not self.upnp_redirects and not all([x in self.component_manager.skip_components for x in if not self.upnp_redirects and not all(
(DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT)]): x in self.component_manager.skip_components
for x in (DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT)
):
log.error("failed to setup upnp") log.error("failed to setup upnp")
else: else:
success = True success = True
@ -471,9 +653,9 @@ class UPnPComponent(Component):
else: else:
log.error("failed to setup upnp") log.error("failed to setup upnp")
if not self.external_ip: if not self.external_ip:
self.external_ip = await utils.get_external_ip() self.external_ip, probed_url = await utils.get_external_ip(self.conf.lbryum_servers)
if self.external_ip: if self.external_ip:
log.info("detected external ip using lbry.com fallback") log.info("detected external ip using %s fallback", probed_url)
if self.component_manager.analytics_manager: if self.component_manager.analytics_manager:
self.component_manager.loop.create_task( self.component_manager.loop.create_task(
self.component_manager.analytics_manager.send_upnp_setup_success_fail( self.component_manager.analytics_manager.send_upnp_setup_success_fail(
@ -489,7 +671,7 @@ class UPnPComponent(Component):
log.info("Removing upnp redirects: %s", self.upnp_redirects) log.info("Removing upnp redirects: %s", self.upnp_redirects)
await asyncio.wait([ await asyncio.wait([
self.upnp.delete_port_mapping(port, protocol) for protocol, port in self.upnp_redirects.items() self.upnp.delete_port_mapping(port, protocol) for protocol, port in self.upnp_redirects.items()
], loop=self.component_manager.loop) ])
if self._maintain_redirects_task and not self._maintain_redirects_task.done(): if self._maintain_redirects_task and not self._maintain_redirects_task.done():
self._maintain_redirects_task.cancel() self._maintain_redirects_task.cancel()
@ -520,3 +702,49 @@ class ExchangeRateManagerComponent(Component):
async def stop(self): async def stop(self):
self.exchange_rate_manager.stop() self.exchange_rate_manager.stop()
class TrackerAnnouncerComponent(Component):
component_name = TRACKER_ANNOUNCER_COMPONENT
depends_on = [FILE_MANAGER_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
self.file_manager = None
self.announce_task = None
self.tracker_client: typing.Optional[TrackerClient] = None
@property
def component(self):
return self.tracker_client
@property
def running(self):
return self._running and self.announce_task and not self.announce_task.done()
async def announce_forever(self):
while True:
sleep_seconds = 60.0
announce_sd_hashes = []
for file in self.file_manager.get_filtered():
if not file.downloader:
continue
announce_sd_hashes.append(bytes.fromhex(file.sd_hash))
await self.tracker_client.announce_many(*announce_sd_hashes)
await asyncio.sleep(sleep_seconds)
async def start(self):
node = self.component_manager.get_component(DHT_COMPONENT) \
if self.component_manager.has_component(DHT_COMPONENT) else None
node_id = node.protocol.node_id if node else None
self.tracker_client = TrackerClient(node_id, self.conf.tcp_port, lambda: self.conf.tracker_servers)
await self.tracker_client.start()
self.file_manager = self.component_manager.get_component(FILE_MANAGER_COMPONENT)
self.announce_task = asyncio.create_task(self.announce_forever())
async def stop(self):
self.file_manager = None
if self.announce_task and not self.announce_task.done():
self.announce_task.cancel()
self.announce_task = None
self.tracker_client.stop()

File diff suppressed because it is too large Load diff

View file

@ -2,9 +2,10 @@ import json
import time import time
import asyncio import asyncio
import logging import logging
from statistics import median
from decimal import Decimal from decimal import Decimal
from typing import Optional, Iterable, Type from typing import Optional, Iterable, Type
from aiohttp.client_exceptions import ContentTypeError from aiohttp.client_exceptions import ContentTypeError, ClientConnectionError
from lbry.error import InvalidExchangeRateResponseError, CurrencyConversionError from lbry.error import InvalidExchangeRateResponseError, CurrencyConversionError
from lbry.utils import aiohttp_request from lbry.utils import aiohttp_request
from lbry.wallet.dewies import lbc_to_dewies from lbry.wallet.dewies import lbc_to_dewies
@ -58,9 +59,12 @@ class MarketFeed:
raise NotImplementedError() raise NotImplementedError()
async def get_response(self): async def get_response(self):
async with aiohttp_request('get', self.url, params=self.params, timeout=self.request_timeout) as response: async with aiohttp_request(
'get', self.url, params=self.params,
timeout=self.request_timeout, headers={"User-Agent": "lbrynet"}
) as response:
try: try:
self._last_response = await response.json() self._last_response = await response.json(content_type=None)
except ContentTypeError as e: except ContentTypeError as e:
self._last_response = {} self._last_response = {}
log.warning("Could not parse exchange rate response from %s: %s", self.name, e.message) log.warning("Could not parse exchange rate response from %s: %s", self.name, e.message)
@ -75,18 +79,21 @@ class MarketFeed:
log.debug("Saving rate update %f for %s from %s", rate, self.market, self.name) log.debug("Saving rate update %f for %s from %s", rate, self.market, self.name)
self.rate = ExchangeRate(self.market, rate, int(time.time())) self.rate = ExchangeRate(self.market, rate, int(time.time()))
self.last_check = time.time() self.last_check = time.time()
self.event.set()
return self.rate return self.rate
except asyncio.CancelledError:
raise
except asyncio.TimeoutError: except asyncio.TimeoutError:
log.warning("Timed out fetching exchange rate from %s.", self.name) log.warning("Timed out fetching exchange rate from %s.", self.name)
except json.JSONDecodeError as e: except json.JSONDecodeError as e:
log.warning("Could not parse exchange rate response from %s: %s", self.name, e.doc) msg = e.doc if '<html>' not in e.doc else 'unexpected content type.'
log.warning("Could not parse exchange rate response from %s: %s", self.name, msg)
log.debug(e.doc)
except InvalidExchangeRateResponseError as e: except InvalidExchangeRateResponseError as e:
log.warning(str(e)) log.warning(str(e))
except ClientConnectionError as e:
log.warning("Error trying to connect to exchange rate %s: %s", self.name, str(e))
except Exception as e: except Exception as e:
log.exception("Exchange rate error (%s from %s):", self.market, self.name) log.exception("Exchange rate error (%s from %s):", self.market, self.name)
finally:
self.event.set()
async def keep_updated(self): async def keep_updated(self):
while True: while True:
@ -104,70 +111,92 @@ class MarketFeed:
self.event.clear() self.event.clear()
class BittrexFeed(MarketFeed): class BaseBittrexFeed(MarketFeed):
name = "Bittrex" name = "Bittrex"
market = "BTCLBC" market = None
url = "https://bittrex.com/api/v1.1/public/getmarkethistory" url = None
params = {'market': 'BTC-LBC', 'count': 50}
fee = 0.0025 fee = 0.0025
def get_rate_from_response(self, json_response):
if 'lastTradeRate' not in json_response:
raise InvalidExchangeRateResponseError(self.name, 'result not found')
return 1.0 / float(json_response['lastTradeRate'])
class BittrexBTCFeed(BaseBittrexFeed):
market = "BTCLBC"
url = "https://api.bittrex.com/v3/markets/LBC-BTC/ticker"
class BittrexUSDFeed(BaseBittrexFeed):
market = "USDLBC"
url = "https://api.bittrex.com/v3/markets/LBC-USD/ticker"
class BaseCoinExFeed(MarketFeed):
name = "CoinEx"
market = None
url = None
def get_rate_from_response(self, json_response):
if 'data' not in json_response or \
'ticker' not in json_response['data'] or \
'last' not in json_response['data']['ticker']:
raise InvalidExchangeRateResponseError(self.name, 'result not found')
return 1.0 / float(json_response['data']['ticker']['last'])
class CoinExBTCFeed(BaseCoinExFeed):
market = "BTCLBC"
url = "https://api.coinex.com/v1/market/ticker?market=LBCBTC"
class CoinExUSDFeed(BaseCoinExFeed):
market = "USDLBC"
url = "https://api.coinex.com/v1/market/ticker?market=LBCUSDT"
class BaseHotbitFeed(MarketFeed):
name = "hotbit"
market = None
url = "https://api.hotbit.io/api/v1/market.last"
def get_rate_from_response(self, json_response): def get_rate_from_response(self, json_response):
if 'result' not in json_response: if 'result' not in json_response:
raise InvalidExchangeRateResponseError(self.name, 'result not found') raise InvalidExchangeRateResponseError(self.name, 'result not found')
trades = json_response['result'] return 1.0 / float(json_response['result'])
if len(trades) == 0:
raise InvalidExchangeRateResponseError(self.name, 'trades not found')
totals = sum([i['Total'] for i in trades])
qtys = sum([i['Quantity'] for i in trades])
if totals <= 0 or qtys <= 0:
raise InvalidExchangeRateResponseError(self.name, 'quantities were not positive')
vwap = totals / qtys
return float(1.0 / vwap)
class LBRYFeed(MarketFeed): class HotbitBTCFeed(BaseHotbitFeed):
name = "lbry.com"
market = "BTCLBC" market = "BTCLBC"
url = "https://api.lbry.com/lbc/exchange_rate" params = {"market": "LBC/BTC"}
def get_rate_from_response(self, json_response):
if 'data' not in json_response:
raise InvalidExchangeRateResponseError(self.name, 'result not found')
return 1.0 / json_response['data']['lbc_btc']
class LBRYBTCFeed(LBRYFeed): class HotbitUSDFeed(BaseHotbitFeed):
market = "USDBTC" market = "USDLBC"
params = {"market": "LBC/USDT"}
def get_rate_from_response(self, json_response):
if 'data' not in json_response:
raise InvalidExchangeRateResponseError(self.name, 'result not found')
return 1.0 / json_response['data']['btc_usd']
class CryptonatorFeed(MarketFeed): class UPbitBTCFeed(MarketFeed):
name = "cryptonator.com" name = "UPbit"
market = "BTCLBC" market = "BTCLBC"
url = "https://api.cryptonator.com/api/ticker/btc-lbc" url = "https://api.upbit.com/v1/ticker"
params = {"markets": "BTC-LBC"}
def get_rate_from_response(self, json_response): def get_rate_from_response(self, json_response):
if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \ if "error" in json_response or len(json_response) != 1 or 'trade_price' not in json_response[0]:
'success' not in json_response or json_response['success'] is not True:
raise InvalidExchangeRateResponseError(self.name, 'result not found') raise InvalidExchangeRateResponseError(self.name, 'result not found')
return float(json_response['ticker']['price']) return 1.0 / float(json_response[0]['trade_price'])
class CryptonatorBTCFeed(CryptonatorFeed):
market = "USDBTC"
url = "https://api.cryptonator.com/api/ticker/usd-btc"
FEEDS: Iterable[Type[MarketFeed]] = ( FEEDS: Iterable[Type[MarketFeed]] = (
LBRYFeed, BittrexBTCFeed,
LBRYBTCFeed, BittrexUSDFeed,
BittrexFeed, CoinExBTCFeed,
# CryptonatorFeed, CoinExUSDFeed,
# CryptonatorBTCFeed, # HotbitBTCFeed,
# HotbitUSDFeed,
# UPbitBTCFeed,
) )
@ -191,20 +220,23 @@ class ExchangeRateManager:
source.stop() source.stop()
def convert_currency(self, from_currency, to_currency, amount): def convert_currency(self, from_currency, to_currency, amount):
rates = [market.rate for market in self.market_feeds] log.debug(
log.debug("Converting %f %s to %s, rates: %s", amount, from_currency, to_currency, rates) "Converting %f %s to %s, rates: %s",
amount, from_currency, to_currency,
[market.rate for market in self.market_feeds]
)
if from_currency == to_currency: if from_currency == to_currency:
return round(amount, 8) return round(amount, 8)
rates = []
for market in self.market_feeds: for market in self.market_feeds:
if (market.has_rate and market.is_online and if (market.has_rate and market.is_online and
market.rate.currency_pair == (from_currency, to_currency)): market.rate.currency_pair == (from_currency, to_currency)):
return round(amount * Decimal(market.rate.spot), 8) rates.append(market.rate.spot)
for market in self.market_feeds:
if (market.has_rate and market.is_online and if rates:
market.rate.currency_pair[0] == from_currency): return round(amount * Decimal(median(rates)), 8)
return round(self.convert_currency(
market.rate.currency_pair[1], to_currency, amount * Decimal(market.rate.spot)), 8)
raise CurrencyConversionError( raise CurrencyConversionError(
f'Unable to convert {amount} from {from_currency} to {to_currency}') f'Unable to convert {amount} from {from_currency} to {to_currency}')

View file

@ -7,8 +7,10 @@ from json import JSONEncoder
from google.protobuf.message import DecodeError from google.protobuf.message import DecodeError
from lbry.schema.claim import Claim from lbry.schema.claim import Claim
from lbry.schema.support import Support
from lbry.torrent.torrent_manager import TorrentSource
from lbry.wallet import Wallet, Ledger, Account, Transaction, Output from lbry.wallet import Wallet, Ledger, Account, Transaction, Output
from lbry.wallet.bip32 import PubKey from lbry.wallet.bip32 import PublicKey
from lbry.wallet.dewies import dewies_to_lbc from lbry.wallet.dewies import dewies_to_lbc
from lbry.stream.managed_stream import ManagedStream from lbry.stream.managed_stream import ManagedStream
@ -25,6 +27,8 @@ def encode_txo_doc():
'address': "address of who can spend the txo", 'address': "address of who can spend the txo",
'confirmations': "number of confirmed blocks", 'confirmations': "number of confirmed blocks",
'is_change': "payment to change address, only available when it can be determined", 'is_change': "payment to change address, only available when it can be determined",
'is_received': "true if txo was sent from external account to this account",
'is_spent': "true if txo is spent",
'is_mine': "payment to one of your accounts, only available when it can be determined", 'is_mine': "payment to one of your accounts, only available when it can be determined",
'type': "one of 'claim', 'support' or 'purchase'", 'type': "one of 'claim', 'support' or 'purchase'",
'name': "when type is 'claim' or 'support', this is the claim name", 'name': "when type is 'claim' or 'support', this is the claim name",
@ -106,7 +110,9 @@ def encode_file_doc():
'metadata': '(dict) None if claim is not found else the claim metadata', 'metadata': '(dict) None if claim is not found else the claim metadata',
'channel_claim_id': '(str) None if claim is not found or not signed', 'channel_claim_id': '(str) None if claim is not found or not signed',
'channel_name': '(str) None if claim is not found or not signed', 'channel_name': '(str) None if claim is not found or not signed',
'claim_name': '(str) None if claim is not found else the claim name' 'claim_name': '(str) None if claim is not found else the claim name',
'reflector_progress': '(int) reflector upload progress, 0 to 100',
'uploading_to_reflector': '(bool) set to True when currently uploading to reflector'
} }
@ -117,12 +123,12 @@ class JSONResponseEncoder(JSONEncoder):
self.ledger = ledger self.ledger = ledger
self.include_protobuf = include_protobuf self.include_protobuf = include_protobuf
def default(self, obj): # pylint: disable=method-hidden,arguments-differ,too-many-return-statements def default(self, obj): # pylint: disable=method-hidden,arguments-renamed,too-many-return-statements
if isinstance(obj, Account): if isinstance(obj, Account):
return self.encode_account(obj) return self.encode_account(obj)
if isinstance(obj, Wallet): if isinstance(obj, Wallet):
return self.encode_wallet(obj) return self.encode_wallet(obj)
if isinstance(obj, ManagedStream): if isinstance(obj, (ManagedStream, TorrentSource)):
return self.encode_file(obj) return self.encode_file(obj)
if isinstance(obj, Transaction): if isinstance(obj, Transaction):
return self.encode_transaction(obj) return self.encode_transaction(obj)
@ -130,7 +136,9 @@ class JSONResponseEncoder(JSONEncoder):
return self.encode_output(obj) return self.encode_output(obj)
if isinstance(obj, Claim): if isinstance(obj, Claim):
return self.encode_claim(obj) return self.encode_claim(obj)
if isinstance(obj, PubKey): if isinstance(obj, Support):
return obj.to_dict()
if isinstance(obj, PublicKey):
return obj.extended_key_string() return obj.extended_key_string()
if isinstance(obj, datetime): if isinstance(obj, datetime):
return obj.strftime("%Y%m%dT%H:%M:%S") return obj.strftime("%Y%m%dT%H:%M:%S")
@ -164,12 +172,22 @@ class JSONResponseEncoder(JSONEncoder):
'amount': dewies_to_lbc(txo.amount), 'amount': dewies_to_lbc(txo.amount),
'address': txo.get_address(self.ledger) if txo.has_address else None, 'address': txo.get_address(self.ledger) if txo.has_address else None,
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height, 'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None 'timestamp': self.ledger.headers.estimated_timestamp(tx_height)
} }
if txo.is_change is not None: if txo.is_spent is not None:
output['is_change'] = txo.is_change output['is_spent'] = txo.is_spent
if txo.is_my_account is not None: if txo.is_my_output is not None:
output['is_mine'] = txo.is_my_account output['is_my_output'] = txo.is_my_output
if txo.is_my_input is not None:
output['is_my_input'] = txo.is_my_input
if txo.sent_supports is not None:
output['sent_supports'] = dewies_to_lbc(txo.sent_supports)
if txo.sent_tips is not None:
output['sent_tips'] = dewies_to_lbc(txo.sent_tips)
if txo.received_tips is not None:
output['received_tips'] = dewies_to_lbc(txo.received_tips)
if txo.is_internal_transfer is not None:
output['is_internal_transfer'] = txo.is_internal_transfer
if txo.script.is_claim_name: if txo.script.is_claim_name:
output['type'] = 'claim' output['type'] = 'claim'
@ -205,22 +223,23 @@ class JSONResponseEncoder(JSONEncoder):
output['claims'] = [self.encode_output(o) for o in txo.claims] output['claims'] = [self.encode_output(o) for o in txo.claims]
if txo.reposted_claim is not None: if txo.reposted_claim is not None:
output['reposted_claim'] = self.encode_output(txo.reposted_claim) output['reposted_claim'] = self.encode_output(txo.reposted_claim)
if txo.script.is_claim_name or txo.script.is_update_claim: if txo.script.is_claim_name or txo.script.is_update_claim or txo.script.is_support_claim_data:
try: try:
output['value'] = txo.claim output['value'] = txo.signable
output['value_type'] = txo.claim.claim_type
if self.include_protobuf: if self.include_protobuf:
output['protobuf'] = hexlify(txo.claim.to_bytes()) output['protobuf'] = hexlify(txo.signable.to_bytes())
if txo.purchase_receipt is not None: if txo.purchase_receipt is not None:
output['purchase_receipt'] = self.encode_output(txo.purchase_receipt) output['purchase_receipt'] = self.encode_output(txo.purchase_receipt)
if txo.script.is_claim_name or txo.script.is_update_claim:
output['value_type'] = txo.claim.claim_type
if txo.claim.is_channel: if txo.claim.is_channel:
output['has_signing_key'] = txo.has_private_key output['has_signing_key'] = txo.has_private_key
if check_signature and txo.claim.is_signed: if check_signature and txo.signable.is_signed:
if txo.channel is not None: if txo.channel is not None:
output['signing_channel'] = self.encode_output(txo.channel) output['signing_channel'] = self.encode_output(txo.channel)
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger) output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
else: else:
output['signing_channel'] = {'channel_id': txo.claim.signing_channel_id} output['signing_channel'] = {'channel_id': txo.signable.signing_channel_id}
output['is_channel_signature_valid'] = False output['is_channel_signature_valid'] = False
except DecodeError: except DecodeError:
pass pass
@ -232,7 +251,7 @@ class JSONResponseEncoder(JSONEncoder):
if isinstance(value, int): if isinstance(value, int):
meta[key] = dewies_to_lbc(value) meta[key] = dewies_to_lbc(value)
if 0 < meta.get('creation_height', 0) <= self.ledger.headers.height: if 0 < meta.get('creation_height', 0) <= self.ledger.headers.height:
meta['creation_timestamp'] = self.ledger.headers[meta['creation_height']]['timestamp'] meta['creation_timestamp'] = self.ledger.headers.estimated_timestamp(meta['creation_height'])
return meta return meta
def encode_input(self, txi): def encode_input(self, txi):
@ -259,26 +278,32 @@ class JSONResponseEncoder(JSONEncoder):
output_exists = managed_stream.output_file_exists output_exists = managed_stream.output_file_exists
tx_height = managed_stream.stream_claim_info.height tx_height = managed_stream.stream_claim_info.height
best_height = self.ledger.headers.height best_height = self.ledger.headers.height
return { is_stream = hasattr(managed_stream, 'stream_hash')
'streaming_url': managed_stream.stream_url, if is_stream:
total_bytes_lower_bound = managed_stream.descriptor.lower_bound_decrypted_length()
total_bytes = managed_stream.descriptor.upper_bound_decrypted_length()
else:
total_bytes_lower_bound = total_bytes = managed_stream.torrent_length
result = {
'streaming_url': None,
'completed': managed_stream.completed, 'completed': managed_stream.completed,
'file_name': managed_stream.file_name if output_exists else None, 'file_name': None,
'download_directory': managed_stream.download_directory if output_exists else None, 'download_directory': None,
'download_path': managed_stream.full_path if output_exists else None, 'download_path': None,
'points_paid': 0.0, 'points_paid': 0.0,
'stopped': not managed_stream.running, 'stopped': not managed_stream.running,
'stream_hash': managed_stream.stream_hash, 'stream_hash': None,
'stream_name': managed_stream.descriptor.stream_name, 'stream_name': None,
'suggested_file_name': managed_stream.descriptor.suggested_file_name, 'suggested_file_name': None,
'sd_hash': managed_stream.descriptor.sd_hash, 'sd_hash': None,
'mime_type': managed_stream.mime_type, 'mime_type': None,
'key': managed_stream.descriptor.key, 'key': None,
'total_bytes_lower_bound': managed_stream.descriptor.lower_bound_decrypted_length(), 'total_bytes_lower_bound': total_bytes_lower_bound,
'total_bytes': managed_stream.descriptor.upper_bound_decrypted_length(), 'total_bytes': total_bytes,
'written_bytes': managed_stream.written_bytes, 'written_bytes': managed_stream.written_bytes,
'blobs_completed': managed_stream.blobs_completed, 'blobs_completed': None,
'blobs_in_stream': managed_stream.blobs_in_stream, 'blobs_in_stream': None,
'blobs_remaining': managed_stream.blobs_remaining, 'blobs_remaining': None,
'status': managed_stream.status, 'status': managed_stream.status,
'claim_id': managed_stream.claim_id, 'claim_id': managed_stream.claim_id,
'txid': managed_stream.txid, 'txid': managed_stream.txid,
@ -294,8 +319,38 @@ class JSONResponseEncoder(JSONEncoder):
'added_on': managed_stream.added_on, 'added_on': managed_stream.added_on,
'height': tx_height, 'height': tx_height,
'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height, 'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height,
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None 'timestamp': self.ledger.headers.estimated_timestamp(tx_height),
'is_fully_reflected': False,
'reflector_progress': False,
'uploading_to_reflector': False
} }
if is_stream:
result.update({
'streaming_url': managed_stream.stream_url,
'stream_hash': managed_stream.stream_hash,
'stream_name': managed_stream.stream_name,
'suggested_file_name': managed_stream.suggested_file_name,
'sd_hash': managed_stream.descriptor.sd_hash,
'mime_type': managed_stream.mime_type,
'key': managed_stream.descriptor.key,
'blobs_completed': managed_stream.blobs_completed,
'blobs_in_stream': managed_stream.blobs_in_stream,
'blobs_remaining': managed_stream.blobs_remaining,
'is_fully_reflected': managed_stream.is_fully_reflected,
'reflector_progress': managed_stream.reflector_progress,
'uploading_to_reflector': managed_stream.uploading_to_reflector
})
else:
result.update({
'streaming_url': f'file://{managed_stream.full_path}',
})
if output_exists:
result.update({
'file_name': managed_stream.file_name,
'download_directory': managed_stream.download_directory,
'download_path': managed_stream.full_path,
})
return result
def encode_claim(self, claim): def encode_claim(self, claim):
encoded = getattr(claim, claim.claim_type).to_dict() encoded = getattr(claim, claim.claim_type).to_dict()

View file

@ -1,89 +0,0 @@
import asyncio
import json
import logging.handlers
import traceback
from aiohttp.client_exceptions import ClientError
import aiohttp
from lbry import utils, __version__
LOGGLY_TOKEN = 'BQEzZmMzLJHgAGxkBF00LGD0YGuyATVgAmqxAQEuAQZ2BQH4'
class JsonFormatter(logging.Formatter):
"""Format log records using json serialization"""
def __init__(self, **kwargs):
super().__init__()
self.attributes = kwargs
def format(self, record):
data = {
'loggerName': record.name,
'asciTime': self.formatTime(record),
'fileName': record.filename,
'functionName': record.funcName,
'levelNo': record.levelno,
'lineNo': record.lineno,
'levelName': record.levelname,
'message': record.getMessage(),
}
data.update(self.attributes)
if record.exc_info:
data['exc_info'] = self.formatException(record.exc_info)
return json.dumps(data)
class HTTPSLogglyHandler(logging.Handler):
def __init__(self, loggly_token: str, fqdn=False, localname=None, facility=None, cookies=None):
super().__init__()
self.fqdn = fqdn
self.localname = localname
self.facility = facility
self.cookies = cookies or {}
self.url = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}".format(
token=utils.deobfuscate(loggly_token), tag='lbrynet-' + __version__
)
self._loop = asyncio.get_event_loop()
self._session = aiohttp.ClientSession()
@staticmethod
def get_full_message(record):
if record.exc_info:
return '\n'.join(traceback.format_exception(*record.exc_info))
else:
return record.getMessage()
async def _emit(self, record, retry=True):
data = self.format(record).encode()
try:
async with self._session.post(self.url, data=data,
cookies=self.cookies) as response:
self.cookies.update(response.cookies)
except ClientError:
if self._loop.is_running() and retry:
await self._session.close()
self._session = aiohttp.ClientSession()
return await self._emit(record, retry=False)
def emit(self, record):
try:
asyncio.ensure_future(self._emit(record), loop=self._loop)
except RuntimeError: # TODO: use a second loop
print(f"\nfailed to send traceback to loggly, please file an issue with the following traceback:\n"
f"{self.format(record)}")
def close(self):
super().close()
try:
loop = asyncio.get_event_loop()
loop.run_until_complete(self._session.close())
except RuntimeError:
pass
def get_loggly_handler():
handler = HTTPSLogglyHandler(LOGGLY_TOKEN)
handler.setFormatter(JsonFormatter())
return handler

View file

@ -35,6 +35,10 @@ def migrate_db(conf, start, end):
from .migrate12to13 import do_migration from .migrate12to13 import do_migration
elif current == 13: elif current == 13:
from .migrate13to14 import do_migration from .migrate13to14 import do_migration
elif current == 14:
from .migrate14to15 import do_migration
elif current == 15:
from .migrate15to16 import do_migration
else: else:
raise Exception(f"DB migration of version {current} to {current+1} is not available") raise Exception(f"DB migration of version {current} to {current+1} is not available")
try: try:

View file

@ -0,0 +1,16 @@
import os
import sqlite3
def do_migration(conf):
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
connection = sqlite3.connect(db_path)
cursor = connection.cursor()
cursor.executescript("""
alter table blob add column added_on integer not null default 0;
alter table blob add column is_mine integer not null default 1;
""")
connection.commit()
connection.close()

View file

@ -0,0 +1,17 @@
import os
import sqlite3
def do_migration(conf):
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
connection = sqlite3.connect(db_path)
cursor = connection.cursor()
cursor.executescript("""
update blob set should_announce=0
where should_announce=1 and
blob.blob_hash in (select stream_blob.blob_hash from stream_blob where position=0);
""")
connection.commit()
connection.close()

View file

@ -20,7 +20,7 @@ def do_migration(conf):
"left outer join blob b ON b.blob_hash=s.blob_hash order by s.position").fetchall() "left outer join blob b ON b.blob_hash=s.blob_hash order by s.position").fetchall()
blobs_by_stream = {} blobs_by_stream = {}
for stream_hash, position, iv, blob_hash, blob_length in blobs: for stream_hash, position, iv, blob_hash, blob_length in blobs:
blobs_by_stream.setdefault(stream_hash, []).append(BlobInfo(position, blob_length or 0, iv, blob_hash)) blobs_by_stream.setdefault(stream_hash, []).append(BlobInfo(position, blob_length or 0, iv, 0, blob_hash))
for stream_name, stream_key, suggested_filename, sd_hash, stream_hash in streams: for stream_name, stream_key, suggested_filename, sd_hash, stream_hash in streams:
sd = StreamDescriptor(None, blob_dir, stream_name, stream_key, suggested_filename, sd = StreamDescriptor(None, blob_dir, stream_name, stream_key, suggested_filename,

View file

@ -0,0 +1,31 @@
import logging
from aiohttp import web
log = logging.getLogger(__name__)
def ensure_request_allowed(request, conf):
if is_request_allowed(request, conf):
return
if conf.allowed_origin:
log.warning(
"API requests with Origin '%s' are not allowed, "
"configuration 'allowed_origin' limits requests to: '%s'",
request.headers.get('Origin'), conf.allowed_origin
)
else:
log.warning(
"API requests with Origin '%s' are not allowed, "
"update configuration 'allowed_origin' to enable this origin.",
request.headers.get('Origin')
)
raise web.HTTPForbidden()
def is_request_allowed(request, conf) -> bool:
origin = request.headers.get('Origin')
return (
origin is None or
origin == conf.allowed_origin or
conf.allowed_origin == '*'
)

View file

@ -9,7 +9,7 @@ from typing import Optional
from lbry.wallet import SQLiteMixin from lbry.wallet import SQLiteMixin
from lbry.conf import Config from lbry.conf import Config
from lbry.wallet.dewies import dewies_to_lbc, lbc_to_dewies from lbry.wallet.dewies import dewies_to_lbc, lbc_to_dewies
from lbry.wallet.transaction import Transaction from lbry.wallet.transaction import Transaction, Output
from lbry.schema.claim import Claim from lbry.schema.claim import Claim
from lbry.dht.constants import DATA_EXPIRATION from lbry.dht.constants import DATA_EXPIRATION
from lbry.blob.blob_info import BlobInfo from lbry.blob.blob_info import BlobInfo
@ -112,7 +112,7 @@ def _batched_select(transaction, query, parameters, batch_size=900):
def _get_lbry_file_stream_dict(rowid, added_on, stream_hash, file_name, download_dir, data_rate, status, def _get_lbry_file_stream_dict(rowid, added_on, stream_hash, file_name, download_dir, data_rate, status,
sd_hash, stream_key, stream_name, suggested_file_name, claim, saved_file, sd_hash, stream_key, stream_name, suggested_file_name, claim, saved_file,
raw_content_fee): raw_content_fee, fully_reflected):
return { return {
"rowid": rowid, "rowid": rowid,
"added_on": added_on, "added_on": added_on,
@ -129,7 +129,8 @@ def _get_lbry_file_stream_dict(rowid, added_on, stream_hash, file_name, download
"saved_file": bool(saved_file), "saved_file": bool(saved_file),
"content_fee": None if not raw_content_fee else Transaction( "content_fee": None if not raw_content_fee else Transaction(
binascii.unhexlify(raw_content_fee) binascii.unhexlify(raw_content_fee)
) ),
"fully_reflected": fully_reflected
} }
@ -138,11 +139,14 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
signed_claims = {} signed_claims = {}
for (rowid, stream_hash, _, file_name, download_dir, data_rate, status, saved_file, raw_content_fee, for (rowid, stream_hash, _, file_name, download_dir, data_rate, status, saved_file, raw_content_fee,
added_on, _, sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in transaction.execute( added_on, _, sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in transaction.execute(
"select file.rowid, file.*, stream.*, c.* " "select file.rowid, file.*, stream.*, c.*, "
" case when (SELECT 1 FROM reflected_stream r WHERE r.sd_hash=stream.sd_hash) "
" is null then 0 else 1 end as fully_reflected "
"from file inner join stream on file.stream_hash=stream.stream_hash " "from file inner join stream on file.stream_hash=stream.stream_hash "
"inner join content_claim cc on file.stream_hash=cc.stream_hash " "inner join content_claim cc on file.stream_hash=cc.stream_hash "
"inner join claim c on cc.claim_outpoint=c.claim_outpoint " "inner join claim c on cc.claim_outpoint=c.claim_outpoint "
"order by c.rowid desc").fetchall(): "order by c.rowid desc").fetchall():
claim_args, fully_reflected = tuple(claim_args[:-1]), claim_args[-1]
claim = StoredContentClaim(*claim_args) claim = StoredContentClaim(*claim_args)
if claim.channel_claim_id: if claim.channel_claim_id:
if claim.channel_claim_id not in signed_claims: if claim.channel_claim_id not in signed_claims:
@ -152,7 +156,7 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
_get_lbry_file_stream_dict( _get_lbry_file_stream_dict(
rowid, added_on, stream_hash, file_name, download_dir, data_rate, status, rowid, added_on, stream_hash, file_name, download_dir, data_rate, status,
sd_hash, stream_key, stream_name, suggested_file_name, claim, saved_file, sd_hash, stream_key, stream_name, suggested_file_name, claim, saved_file,
raw_content_fee raw_content_fee, fully_reflected
) )
) )
for claim_name, claim_id in _batched_select( for claim_name, claim_id in _batched_select(
@ -166,8 +170,8 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'): def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'):
# add all blobs, except the last one, which is empty # add all blobs, except the last one, which is empty
transaction.executemany( transaction.executemany(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)", "insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
((blob.blob_hash, blob.length, 0, 0, "pending", 0, 0) ((blob.blob_hash, blob.length, 0, 0, "pending", 0, 0, blob.added_on, blob.is_mine)
for blob in (descriptor.blobs[:-1] if len(descriptor.blobs) > 1 else descriptor.blobs) + [sd_blob]) for blob in (descriptor.blobs[:-1] if len(descriptor.blobs) > 1 else descriptor.blobs) + [sd_blob])
).fetchall() ).fetchall()
# associate the blobs to the stream # associate the blobs to the stream
@ -183,8 +187,8 @@ def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descripto
).fetchall() ).fetchall()
# ensure should_announce is set regardless if insert was ignored # ensure should_announce is set regardless if insert was ignored
transaction.execute( transaction.execute(
"update blob set should_announce=1 where blob_hash in (?, ?)", "update blob set should_announce=1 where blob_hash in (?)",
(sd_blob.blob_hash, descriptor.blobs[0].blob_hash,) (sd_blob.blob_hash,)
).fetchall() ).fetchall()
@ -238,7 +242,9 @@ class SQLiteStorage(SQLiteMixin):
should_announce integer not null default 0, should_announce integer not null default 0,
status text not null, status text not null,
last_announced_time integer, last_announced_time integer,
single_announce integer single_announce integer,
added_on integer not null,
is_mine integer not null default 0
); );
create table if not exists stream ( create table if not exists stream (
@ -331,6 +337,7 @@ class SQLiteStorage(SQLiteMixin):
tcp_port integer, tcp_port integer,
unique (address, udp_port) unique (address, udp_port)
); );
create index if not exists blob_data on blob(blob_hash, blob_length, is_mine);
""" """
def __init__(self, conf: Config, path, loop=None, time_getter: typing.Optional[typing.Callable[[], float]] = None): def __init__(self, conf: Config, path, loop=None, time_getter: typing.Optional[typing.Callable[[], float]] = None):
@ -352,19 +359,19 @@ class SQLiteStorage(SQLiteMixin):
# # # # # # # # # blob functions # # # # # # # # # # # # # # # # # # blob functions # # # # # # # # #
async def add_blobs(self, *blob_hashes_and_lengths: typing.Tuple[str, int], finished=False): async def add_blobs(self, *blob_hashes_and_lengths: typing.Tuple[str, int, int, int], finished=False):
def _add_blobs(transaction: sqlite3.Connection): def _add_blobs(transaction: sqlite3.Connection):
transaction.executemany( transaction.executemany(
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)", "insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
( (
(blob_hash, length, 0, 0, "pending" if not finished else "finished", 0, 0) (blob_hash, length, 0, 0, "pending" if not finished else "finished", 0, 0, added_on, is_mine)
for blob_hash, length in blob_hashes_and_lengths for blob_hash, length, added_on, is_mine in blob_hashes_and_lengths
) )
).fetchall() ).fetchall()
if finished: if finished:
transaction.executemany( transaction.executemany(
"update blob set status='finished' where blob.blob_hash=?", ( "update blob set status='finished' where blob.blob_hash=?", (
(blob_hash, ) for blob_hash, _ in blob_hashes_and_lengths (blob_hash, ) for blob_hash, _, _, _ in blob_hashes_and_lengths
) )
).fetchall() ).fetchall()
return await self.db.run(_add_blobs) return await self.db.run(_add_blobs)
@ -374,6 +381,11 @@ class SQLiteStorage(SQLiteMixin):
"select status from blob where blob_hash=?", blob_hash "select status from blob where blob_hash=?", blob_hash
) )
def set_announce(self, *blob_hashes):
return self.db.execute_fetchall(
"update blob set should_announce=1 where blob_hash in (?, ?)", blob_hashes
)
def update_last_announced_blobs(self, blob_hashes: typing.List[str]): def update_last_announced_blobs(self, blob_hashes: typing.List[str]):
def _update_last_announced_blobs(transaction: sqlite3.Connection): def _update_last_announced_blobs(transaction: sqlite3.Connection):
last_announced = self.time_getter() last_announced = self.time_getter()
@ -431,6 +443,62 @@ class SQLiteStorage(SQLiteMixin):
def get_all_blob_hashes(self): def get_all_blob_hashes(self):
return self.run_and_return_list("select blob_hash from blob") return self.run_and_return_list("select blob_hash from blob")
async def get_stored_blobs(self, is_mine: bool, is_network_blob=False):
is_mine = 1 if is_mine else 0
if is_network_blob:
return await self.db.execute_fetchall(
"select blob.blob_hash, blob.blob_length, blob.added_on "
"from blob left join stream_blob using (blob_hash) "
"where stream_blob.stream_hash is null and blob.is_mine=? and blob.status='finished'"
"order by blob.blob_length desc, blob.added_on asc",
(is_mine,)
)
sd_blobs = await self.db.execute_fetchall(
"select blob.blob_hash, blob.blob_length, blob.added_on "
"from blob join stream on blob.blob_hash=stream.sd_hash join file using (stream_hash) "
"where blob.is_mine=? order by blob.added_on asc",
(is_mine,)
)
content_blobs = await self.db.execute_fetchall(
"select blob.blob_hash, blob.blob_length, blob.added_on "
"from blob join stream_blob using (blob_hash) cross join stream using (stream_hash)"
"cross join file using (stream_hash)"
"where blob.is_mine=? and blob.status='finished' order by blob.added_on asc, blob.blob_length asc",
(is_mine,)
)
return content_blobs + sd_blobs
async def get_stored_blob_disk_usage(self):
total, network_size, content_size, private_size = await self.db.execute_fetchone("""
select coalesce(sum(blob_length), 0) as total,
coalesce(sum(case when
stream_blob.stream_hash is null
then blob_length else 0 end), 0) as network_storage,
coalesce(sum(case when
stream_blob.blob_hash is not null and is_mine=0
then blob_length else 0 end), 0) as content_storage,
coalesce(sum(case when
is_mine=1
then blob_length else 0 end), 0) as private_storage
from blob left join stream_blob using (blob_hash)
where blob_hash not in (select sd_hash from stream) and blob.status="finished"
""")
return {
'network_storage': network_size,
'content_storage': content_size,
'private_storage': private_size,
'total': total
}
async def update_blob_ownership(self, sd_hash, is_mine: bool):
is_mine = 1 if is_mine else 0
await self.db.execute_fetchall(
"update blob set is_mine = ? where blob_hash in ("
" select blob_hash from blob natural join stream_blob natural join stream where sd_hash = ?"
") OR blob_hash = ?", (is_mine, sd_hash, sd_hash)
)
def sync_missing_blobs(self, blob_files: typing.Set[str]) -> typing.Awaitable[typing.Set[str]]: def sync_missing_blobs(self, blob_files: typing.Set[str]) -> typing.Awaitable[typing.Set[str]]:
def _sync_blobs(transaction: sqlite3.Connection) -> typing.Set[str]: def _sync_blobs(transaction: sqlite3.Connection) -> typing.Set[str]:
finished_blob_hashes = tuple( finished_blob_hashes = tuple(
@ -466,7 +534,8 @@ class SQLiteStorage(SQLiteMixin):
def _get_blobs_for_stream(transaction): def _get_blobs_for_stream(transaction):
crypt_blob_infos = [] crypt_blob_infos = []
stream_blobs = transaction.execute( stream_blobs = transaction.execute(
"select blob_hash, position, iv from stream_blob where stream_hash=? " "select s.blob_hash, s.position, s.iv, b.added_on "
"from stream_blob s left outer join blob b on b.blob_hash=s.blob_hash where stream_hash=? "
"order by position asc", (stream_hash, ) "order by position asc", (stream_hash, )
).fetchall() ).fetchall()
if only_completed: if only_completed:
@ -486,9 +555,10 @@ class SQLiteStorage(SQLiteMixin):
for blob_hash, length in lengths: for blob_hash, length in lengths:
blob_length_dict[blob_hash] = length blob_length_dict[blob_hash] = length
for blob_hash, position, iv in stream_blobs: current_time = time.time()
for blob_hash, position, iv, added_on in stream_blobs:
blob_length = blob_length_dict.get(blob_hash, 0) blob_length = blob_length_dict.get(blob_hash, 0)
crypt_blob_infos.append(BlobInfo(position, blob_length, iv, blob_hash)) crypt_blob_infos.append(BlobInfo(position, blob_length, iv, added_on or current_time, blob_hash))
if not blob_hash: if not blob_hash:
break break
return crypt_blob_infos return crypt_blob_infos
@ -566,6 +636,10 @@ class SQLiteStorage(SQLiteMixin):
log.debug("update file status %s -> %s", stream_hash, new_status) log.debug("update file status %s -> %s", stream_hash, new_status)
return self.db.execute_fetchall("update file set status=? where stream_hash=?", (new_status, stream_hash)) return self.db.execute_fetchall("update file set status=? where stream_hash=?", (new_status, stream_hash))
def stop_all_files(self):
log.debug("stopping all files")
return self.db.execute_fetchall("update file set status=?", ("stopped",))
async def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: typing.Optional[str], async def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: typing.Optional[str],
file_name: typing.Optional[str]): file_name: typing.Optional[str]):
if not file_name or not download_dir: if not file_name or not download_dir:
@ -613,7 +687,7 @@ class SQLiteStorage(SQLiteMixin):
).fetchall() ).fetchall()
download_dir = binascii.hexlify(self.conf.download_dir.encode()).decode() download_dir = binascii.hexlify(self.conf.download_dir.encode()).decode()
transaction.executemany( transaction.executemany(
f"update file set download_directory=? where stream_hash=?", "update file set download_directory=? where stream_hash=?",
((download_dir, stream_hash) for stream_hash in stream_hashes) ((download_dir, stream_hash) for stream_hash in stream_hashes)
).fetchall() ).fetchall()
await self.db.run_with_foreign_keys_disabled(_recover) await self.db.run_with_foreign_keys_disabled(_recover)
@ -719,10 +793,23 @@ class SQLiteStorage(SQLiteMixin):
await self.db.run(_save_claims) await self.db.run(_save_claims)
if update_file_callbacks: if update_file_callbacks:
await asyncio.wait(update_file_callbacks) await asyncio.wait(map(asyncio.create_task, update_file_callbacks))
if claim_id_to_supports: if claim_id_to_supports:
await self.save_supports(claim_id_to_supports) await self.save_supports(claim_id_to_supports)
def save_claim_from_output(self, ledger, *outputs: Output):
return self.save_claims([{
"claim_id": output.claim_id,
"name": output.claim_name,
"amount": dewies_to_lbc(output.amount),
"address": output.get_address(ledger),
"txid": output.tx_ref.id,
"nout": output.position,
"value": output.claim,
"height": output.tx_ref.height,
"claim_sequence": -1,
} for output in outputs])
def save_claims_for_resolve(self, claim_infos): def save_claims_for_resolve(self, claim_infos):
to_save = {} to_save = {}
for info in claim_infos: for info in claim_infos:
@ -736,7 +823,8 @@ class SQLiteStorage(SQLiteMixin):
return self.save_claims(to_save.values()) return self.save_claims(to_save.values())
@staticmethod @staticmethod
def _save_content_claim(transaction, claim_outpoint, stream_hash): def _save_content_claim(transaction, claim_outpoint, stream_hash=None, bt_infohash=None):
assert stream_hash or bt_infohash
# get the claim id and serialized metadata # get the claim id and serialized metadata
claim_info = transaction.execute( claim_info = transaction.execute(
"select claim_id, serialized_metadata from claim where claim_outpoint=?", (claim_outpoint,) "select claim_id, serialized_metadata from claim where claim_outpoint=?", (claim_outpoint,)
@ -784,6 +872,19 @@ class SQLiteStorage(SQLiteMixin):
if stream_hash in self.content_claim_callbacks: if stream_hash in self.content_claim_callbacks:
await self.content_claim_callbacks[stream_hash]() await self.content_claim_callbacks[stream_hash]()
async def save_torrent_content_claim(self, bt_infohash, claim_outpoint, length, name):
def _save_torrent(transaction):
transaction.execute(
"insert or replace into torrent values (?, NULL, ?, ?)", (bt_infohash, length, name)
).fetchall()
transaction.execute(
"insert or replace into content_claim values (NULL, ?, ?)", (bt_infohash, claim_outpoint)
).fetchall()
await self.db.run(_save_torrent)
# update corresponding ManagedEncryptedFileDownloader object
if bt_infohash in self.content_claim_callbacks:
await self.content_claim_callbacks[bt_infohash]()
async def get_content_claim(self, stream_hash: str, include_supports: typing.Optional[bool] = True) -> typing.Dict: async def get_content_claim(self, stream_hash: str, include_supports: typing.Optional[bool] = True) -> typing.Dict:
claims = await self.db.run(get_claims_from_stream_hashes, [stream_hash]) claims = await self.db.run(get_claims_from_stream_hashes, [stream_hash])
claim = None claim = None
@ -795,6 +896,10 @@ class SQLiteStorage(SQLiteMixin):
claim['effective_amount'] = calculate_effective_amount(claim['amount'], supports) claim['effective_amount'] = calculate_effective_amount(claim['amount'], supports)
return claim return claim
async def get_content_claim_for_torrent(self, bt_infohash):
claims = await self.db.run(get_claims_from_torrent_info_hashes, [bt_infohash])
return claims[bt_infohash].as_dict() if claims else None
# # # # # # # # # reflector functions # # # # # # # # # # # # # # # # # # reflector functions # # # # # # # # #
def update_reflected_stream(self, sd_hash, reflector_address, success=True): def update_reflected_stream(self, sd_hash, reflector_address, success=True):
@ -826,6 +931,6 @@ class SQLiteStorage(SQLiteMixin):
transaction.execute('delete from peer').fetchall() transaction.execute('delete from peer').fetchall()
transaction.executemany( transaction.executemany(
'insert into peer(node_id, address, udp_port, tcp_port) values (?, ?, ?, ?)', 'insert into peer(node_id, address, udp_port, tcp_port) values (?, ?, ?, ?)',
tuple([(binascii.hexlify(p.node_id), p.address, p.udp_port, p.tcp_port) for p in peers]) ((binascii.hexlify(p.node_id), p.address, p.udp_port, p.tcp_port) for p in peers)
).fetchall() ).fetchall()
return await self.db.run(_save_kademlia_peers) return await self.db.run(_save_kademlia_peers)

307
lbry/file/file_manager.py Normal file
View file

@ -0,0 +1,307 @@
import asyncio
import logging
import typing
from typing import Optional
from aiohttp.web import Request
from lbry.error import ResolveError, DownloadSDTimeoutError, InsufficientFundsError
from lbry.error import ResolveTimeoutError, DownloadDataTimeoutError, KeyFeeAboveMaxAllowedError
from lbry.error import InvalidStreamURLError
from lbry.stream.managed_stream import ManagedStream
from lbry.torrent.torrent_manager import TorrentSource
from lbry.utils import cache_concurrent
from lbry.schema.url import URL
from lbry.wallet.dewies import dewies_to_lbc
from lbry.file.source_manager import SourceManager
from lbry.file.source import ManagedDownloadSource
from lbry.extras.daemon.storage import StoredContentClaim
if typing.TYPE_CHECKING:
from lbry.conf import Config
from lbry.extras.daemon.analytics import AnalyticsManager
from lbry.extras.daemon.storage import SQLiteStorage
from lbry.wallet import WalletManager
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
log = logging.getLogger(__name__)
class FileManager:
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', wallet_manager: 'WalletManager',
storage: 'SQLiteStorage', analytics_manager: Optional['AnalyticsManager'] = None):
self.loop = loop
self.config = config
self.wallet_manager = wallet_manager
self.storage = storage
self.analytics_manager = analytics_manager
self.source_managers: typing.Dict[str, SourceManager] = {}
self.started = asyncio.Event()
@property
def streams(self):
return self.source_managers['stream']._sources
async def create_stream(self, file_path: str, key: Optional[bytes] = None, **kwargs) -> ManagedDownloadSource:
if 'stream' in self.source_managers:
return await self.source_managers['stream'].create(file_path, key, **kwargs)
raise NotImplementedError
async def start(self):
await asyncio.gather(*(source_manager.start() for source_manager in self.source_managers.values()))
for manager in self.source_managers.values():
await manager.started.wait()
self.started.set()
async def stop(self):
for manager in self.source_managers.values():
# fixme: pop or not?
await manager.stop()
self.started.clear()
@cache_concurrent
async def download_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
timeout: Optional[float] = None, file_name: Optional[str] = None,
download_directory: Optional[str] = None,
save_file: Optional[bool] = None, resolve_timeout: float = 3.0,
wallet: Optional['Wallet'] = None) -> ManagedDownloadSource:
wallet = wallet or self.wallet_manager.default_wallet
timeout = timeout or self.config.download_timeout
start_time = self.loop.time()
resolved_time = None
stream = None
claim = None
error = None
outpoint = None
if save_file is None:
save_file = self.config.save_files
if file_name and not save_file:
save_file = True
if save_file:
download_directory = download_directory or self.config.download_dir
else:
download_directory = None
payment = None
try:
# resolve the claim
try:
if not URL.parse(uri).has_stream:
raise InvalidStreamURLError(uri)
except ValueError:
raise InvalidStreamURLError(uri)
try:
resolved_result = await asyncio.wait_for(
self.wallet_manager.ledger.resolve(
wallet.accounts, [uri],
include_purchase_receipt=True,
include_is_my_output=True
), resolve_timeout
)
except asyncio.TimeoutError:
raise ResolveTimeoutError(uri)
except Exception as err:
log.exception("Unexpected error resolving stream:")
raise ResolveError(f"Unexpected error resolving stream: {str(err)}")
if 'error' in resolved_result:
raise ResolveError(f"Unexpected error resolving uri for download: {resolved_result['error']}")
if not resolved_result or uri not in resolved_result:
raise ResolveError(f"Failed to resolve stream at '{uri}'")
txo = resolved_result[uri]
if isinstance(txo, dict):
raise ResolveError(f"Failed to resolve stream at '{uri}': {txo}")
claim = txo.claim
outpoint = f"{txo.tx_ref.id}:{txo.position}"
resolved_time = self.loop.time() - start_time
await self.storage.save_claim_from_output(self.wallet_manager.ledger, txo)
####################
# update or replace
####################
if claim.stream.source.bt_infohash:
source_manager = self.source_managers['torrent']
existing = source_manager.get_filtered(bt_infohash=claim.stream.source.bt_infohash)
elif claim.stream.source.sd_hash:
source_manager = self.source_managers['stream']
existing = source_manager.get_filtered(sd_hash=claim.stream.source.sd_hash)
else:
raise ResolveError(f"There is nothing to download at {uri} - Source is unknown or unset")
# resume or update an existing stream, if the stream changed: download it and delete the old one after
to_replace, updated_stream = None, None
if existing and existing[0].claim_id != txo.claim_id:
raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {txo.claim_id}")
if existing:
log.info("claim contains a metadata only update to a stream we have")
if claim.stream.source.bt_infohash:
await self.storage.save_torrent_content_claim(
existing[0].identifier, outpoint, existing[0].torrent_length, existing[0].torrent_name
)
claim_info = await self.storage.get_content_claim_for_torrent(existing[0].identifier)
existing[0].set_claim(claim_info, claim)
else:
await self.storage.save_content_claim(
existing[0].stream_hash, outpoint
)
await source_manager._update_content_claim(existing[0])
updated_stream = existing[0]
else:
existing_for_claim_id = self.get_filtered(claim_id=txo.claim_id)
if existing_for_claim_id:
log.info("claim contains an update to a stream we have, downloading it")
if save_file and existing_for_claim_id[0].output_file_exists:
save_file = False
if not claim.stream.source.bt_infohash:
existing_for_claim_id[0].downloader.node = source_manager.node
await existing_for_claim_id[0].start(timeout=timeout, save_now=save_file)
if not existing_for_claim_id[0].output_file_exists and (
save_file or file_name or download_directory):
await existing_for_claim_id[0].save_file(
file_name=file_name, download_directory=download_directory
)
to_replace = existing_for_claim_id[0]
# resume or update an existing stream, if the stream changed: download it and delete the old one after
if updated_stream:
log.info("already have stream for %s", uri)
if save_file and updated_stream.output_file_exists:
save_file = False
if not claim.stream.source.bt_infohash:
updated_stream.downloader.node = source_manager.node
await updated_stream.start(timeout=timeout, save_now=save_file)
if not updated_stream.output_file_exists and (save_file or file_name or download_directory):
await updated_stream.save_file(
file_name=file_name, download_directory=download_directory
)
return updated_stream
####################
# pay fee
####################
needs_purchasing = (
not to_replace and
not txo.is_my_output and
txo.has_price and
not txo.purchase_receipt
)
if needs_purchasing:
payment = await self.wallet_manager.create_purchase_transaction(
wallet.accounts, txo, exchange_rate_manager
)
####################
# make downloader and wait for start
####################
# temporary with fields we know so downloader can start. Missing fields are populated later.
stored_claim = StoredContentClaim(outpoint=outpoint, claim_id=txo.claim_id, name=txo.claim_name,
amount=txo.amount, height=txo.tx_ref.height,
serialized=claim.to_bytes().hex())
if not claim.stream.source.bt_infohash:
# fixme: this shouldnt be here
stream = ManagedStream(
self.loop, self.config, source_manager.blob_manager, claim.stream.source.sd_hash,
download_directory, file_name, ManagedStream.STATUS_RUNNING, content_fee=payment,
analytics_manager=self.analytics_manager, claim=stored_claim
)
stream.downloader.node = source_manager.node
else:
stream = TorrentSource(
self.loop, self.config, self.storage, identifier=claim.stream.source.bt_infohash,
file_name=file_name, download_directory=download_directory or self.config.download_dir,
status=ManagedStream.STATUS_RUNNING, claim=stored_claim, analytics_manager=self.analytics_manager,
torrent_session=source_manager.torrent_session
)
log.info("starting download for %s", uri)
before_download = self.loop.time()
await stream.start(timeout, save_file)
####################
# success case: delete to_replace if applicable, broadcast fee payment
####################
if to_replace: # delete old stream now that the replacement has started downloading
await source_manager.delete(to_replace)
if payment is not None:
await self.wallet_manager.broadcast_or_release(payment)
payment = None # to avoid releasing in `finally` later
log.info("paid fee of %s for %s", dewies_to_lbc(stream.content_fee.outputs[0].amount), uri)
await self.storage.save_content_fee(stream.stream_hash, stream.content_fee)
source_manager.add(stream)
if not claim.stream.source.bt_infohash:
await self.storage.save_content_claim(stream.stream_hash, outpoint)
else:
await self.storage.save_torrent_content_claim(
stream.identifier, outpoint, stream.torrent_length, stream.torrent_name
)
claim_info = await self.storage.get_content_claim_for_torrent(stream.identifier)
stream.set_claim(claim_info, claim)
if save_file:
await asyncio.wait_for(stream.save_file(), timeout - (self.loop.time() - before_download))
return stream
except asyncio.TimeoutError:
error = DownloadDataTimeoutError(stream.sd_hash)
raise error
except (Exception, asyncio.CancelledError) as err: # forgive data timeout, don't delete stream
expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError,
KeyFeeAboveMaxAllowedError, ResolveError, InvalidStreamURLError)
if isinstance(err, expected):
log.warning("Failed to download %s: %s", uri, str(err))
elif isinstance(err, asyncio.CancelledError):
pass
else:
log.exception("Unexpected error downloading stream:")
error = err
raise
finally:
if payment is not None:
# payment is set to None after broadcasting, if we're here an exception probably happened
await self.wallet_manager.ledger.release_tx(payment)
if self.analytics_manager and claim and claim.stream.source.bt_infohash:
# TODO: analytics for torrents
pass
elif self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or
stream.downloader.time_to_first_bytes))):
server = self.wallet_manager.ledger.network.client.server
self.loop.create_task(
self.analytics_manager.send_time_to_first_bytes(
resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id,
uri, outpoint,
None if not stream else len(stream.downloader.blob_downloader.active_connections),
None if not stream else len(stream.downloader.blob_downloader.scores),
None if not stream else len(stream.downloader.blob_downloader.connection_failures),
False if not stream else stream.downloader.added_fixed_peers,
self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay,
None if not stream else stream.sd_hash,
None if not stream else stream.downloader.time_to_descriptor,
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
None if not stream else stream.downloader.time_to_first_bytes,
None if not error else error.__class__.__name__,
None if not error else str(error),
None if not server else f"{server[0]}:{server[1]}"
)
)
async def stream_partial_content(self, request: Request, sd_hash: str):
return await self.source_managers['stream'].stream_partial_content(request, sd_hash)
def get_filtered(self, *args, **kwargs) -> typing.List[ManagedDownloadSource]:
"""
Get a list of filtered and sorted ManagedStream objects
:param sort_by: field to sort by
:param reverse: reverse sorting
:param comparison: comparison operator used for filtering
:param search_by: fields and values to filter by
"""
return sum((manager.get_filtered(*args, **kwargs) for manager in self.source_managers.values()), [])
async def delete(self, source: ManagedDownloadSource, delete_file=False):
for manager in self.source_managers.values():
await manager.delete(source, delete_file)

162
lbry/file/source.py Normal file
View file

@ -0,0 +1,162 @@
import os
import asyncio
import typing
import logging
import binascii
from typing import Optional
from lbry.utils import generate_id
from lbry.extras.daemon.storage import StoredContentClaim
if typing.TYPE_CHECKING:
from lbry.conf import Config
from lbry.extras.daemon.analytics import AnalyticsManager
from lbry.wallet.transaction import Transaction
from lbry.extras.daemon.storage import SQLiteStorage
log = logging.getLogger(__name__)
class ManagedDownloadSource:
STATUS_RUNNING = "running"
STATUS_STOPPED = "stopped"
STATUS_FINISHED = "finished"
SAVING_ID = 1
STREAMING_ID = 2
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage', identifier: str,
file_name: Optional[str] = None, download_directory: Optional[str] = None,
status: Optional[str] = STATUS_STOPPED, claim: Optional[StoredContentClaim] = None,
download_id: Optional[str] = None, rowid: Optional[int] = None,
content_fee: Optional['Transaction'] = None,
analytics_manager: Optional['AnalyticsManager'] = None,
added_on: Optional[int] = None):
self.loop = loop
self.storage = storage
self.config = config
self.identifier = identifier
self.download_directory = download_directory
self._file_name = file_name
self._status = status
self.stream_claim_info = claim
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
self.rowid = rowid
self.content_fee = content_fee
self.purchase_receipt = None
self._added_on = added_on
self.analytics_manager = analytics_manager
self.downloader = None
self.saving = asyncio.Event()
self.finished_writing = asyncio.Event()
self.started_writing = asyncio.Event()
self.finished_write_attempt = asyncio.Event()
# @classmethod
# async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', file_path: str,
# key: Optional[bytes] = None,
# iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedDownloadSource':
# raise NotImplementedError()
async def start(self, timeout: Optional[float] = None, save_now: Optional[bool] = False):
raise NotImplementedError()
async def stop(self, finished: bool = False):
raise NotImplementedError()
async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
raise NotImplementedError()
async def stop_tasks(self):
raise NotImplementedError()
def set_claim(self, claim_info: typing.Dict, claim: 'Claim'):
self.stream_claim_info = StoredContentClaim(
f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
claim_info['name'], claim_info['amount'], claim_info['height'],
binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'],
claim_info['claim_sequence'], claim_info.get('channel_name')
)
# async def update_content_claim(self, claim_info: Optional[typing.Dict] = None):
# if not claim_info:
# claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash)
# self.set_claim(claim_info, claim_info['value'])
@property
def file_name(self) -> Optional[str]:
return self._file_name
@property
def added_on(self) -> Optional[int]:
return self._added_on
@property
def status(self) -> str:
return self._status
@property
def completed(self):
raise NotImplementedError()
# @property
# def stream_url(self):
# return f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash}
@property
def finished(self) -> bool:
return self.status == self.STATUS_FINISHED
@property
def running(self) -> bool:
return self.status == self.STATUS_RUNNING
@property
def claim_id(self) -> Optional[str]:
return None if not self.stream_claim_info else self.stream_claim_info.claim_id
@property
def txid(self) -> Optional[str]:
return None if not self.stream_claim_info else self.stream_claim_info.txid
@property
def nout(self) -> Optional[int]:
return None if not self.stream_claim_info else self.stream_claim_info.nout
@property
def outpoint(self) -> Optional[str]:
return None if not self.stream_claim_info else self.stream_claim_info.outpoint
@property
def claim_height(self) -> Optional[int]:
return None if not self.stream_claim_info else self.stream_claim_info.height
@property
def channel_claim_id(self) -> Optional[str]:
return None if not self.stream_claim_info else self.stream_claim_info.channel_claim_id
@property
def channel_name(self) -> Optional[str]:
return None if not self.stream_claim_info else self.stream_claim_info.channel_name
@property
def claim_name(self) -> Optional[str]:
return None if not self.stream_claim_info else self.stream_claim_info.claim_name
@property
def metadata(self) -> Optional[typing.Dict]:
return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict()
@property
def metadata_protobuf(self) -> bytes:
if self.stream_claim_info:
return binascii.hexlify(self.stream_claim_info.claim.to_bytes())
@property
def full_path(self) -> Optional[str]:
return os.path.join(self.download_directory, os.path.basename(self.file_name)) \
if self.file_name and self.download_directory else None
@property
def output_file_exists(self):
return os.path.isfile(self.full_path) if self.full_path else False

138
lbry/file/source_manager.py Normal file
View file

@ -0,0 +1,138 @@
import os
import asyncio
import logging
import typing
from typing import Optional
from lbry.file.source import ManagedDownloadSource
if typing.TYPE_CHECKING:
from lbry.conf import Config
from lbry.extras.daemon.analytics import AnalyticsManager
from lbry.extras.daemon.storage import SQLiteStorage
log = logging.getLogger(__name__)
COMPARISON_OPERATORS = {
'eq': lambda a, b: a == b,
'ne': lambda a, b: a != b,
'g': lambda a, b: a > b,
'l': lambda a, b: a < b,
'ge': lambda a, b: a >= b,
'le': lambda a, b: a <= b,
}
class SourceManager:
filter_fields = {
'rowid',
'status',
'file_name',
'added_on',
'download_path',
'claim_name',
'claim_height',
'claim_id',
'outpoint',
'txid',
'nout',
'channel_claim_id',
'channel_name',
'completed'
}
set_filter_fields = {
"claim_ids": "claim_id",
"channel_claim_ids": "channel_claim_id",
"outpoints": "outpoint"
}
source_class = ManagedDownloadSource
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage',
analytics_manager: Optional['AnalyticsManager'] = None):
self.loop = loop
self.config = config
self.storage = storage
self.analytics_manager = analytics_manager
self._sources: typing.Dict[str, ManagedDownloadSource] = {}
self.started = asyncio.Event()
def add(self, source: ManagedDownloadSource):
self._sources[source.identifier] = source
async def remove(self, source: ManagedDownloadSource):
if source.identifier not in self._sources:
return
self._sources.pop(source.identifier)
await source.stop_tasks()
async def initialize_from_database(self):
raise NotImplementedError()
async def start(self):
await self.initialize_from_database()
self.started.set()
async def stop(self):
while self._sources:
_, source = self._sources.popitem()
await source.stop_tasks()
self.started.clear()
async def create(self, file_path: str, key: Optional[bytes] = None,
iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedDownloadSource:
raise NotImplementedError()
async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
await self.remove(source)
if delete_file and source.output_file_exists:
os.remove(source.full_path)
def get_filtered(self, sort_by: Optional[str] = None, reverse: Optional[bool] = False,
comparison: Optional[str] = None, **search_by) -> typing.List[ManagedDownloadSource]:
"""
Get a list of filtered and sorted ManagedStream objects
:param sort_by: field to sort by
:param reverse: reverse sorting
:param comparison: comparison operator used for filtering
:param search_by: fields and values to filter by
"""
if sort_by and sort_by not in self.filter_fields:
raise ValueError(f"'{sort_by}' is not a valid field to sort by")
if comparison and comparison not in COMPARISON_OPERATORS:
raise ValueError(f"'{comparison}' is not a valid comparison")
if 'full_status' in search_by:
del search_by['full_status']
for search in search_by:
if search not in self.filter_fields:
raise ValueError(f"'{search}' is not a valid search operation")
compare_sets = {}
if isinstance(search_by.get('claim_id'), list):
compare_sets['claim_ids'] = search_by.pop('claim_id')
if isinstance(search_by.get('outpoint'), list):
compare_sets['outpoints'] = search_by.pop('outpoint')
if isinstance(search_by.get('channel_claim_id'), list):
compare_sets['channel_claim_ids'] = search_by.pop('channel_claim_id')
if search_by or compare_sets:
comparison = comparison or 'eq'
streams = []
for stream in self._sources.values():
if compare_sets and not all(
getattr(stream, self.set_filter_fields[set_search]) in val
for set_search, val in compare_sets.items()):
continue
if search_by and not all(
COMPARISON_OPERATORS[comparison](getattr(stream, search), val)
for search, val in search_by.items()):
continue
streams.append(stream)
else:
streams = list(self._sources.values())
if sort_by:
streams.sort(key=lambda s: getattr(s, sort_by) or "")
if reverse:
streams.reverse()
return streams

View file

@ -3,10 +3,14 @@ import json
import logging import logging
import os import os
import pathlib import pathlib
import platform
import re import re
import shlex import shlex
import shutil import shutil
import subprocess
from math import ceil
import lbry.utils
from lbry.conf import TranscodeConfig from lbry.conf import TranscodeConfig
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -14,55 +18,104 @@ log = logging.getLogger(__name__)
class VideoFileAnalyzer: class VideoFileAnalyzer:
def _replace_or_pop_env(self, variable):
if variable + '_ORIG' in self._env_copy:
self._env_copy[variable] = self._env_copy[variable + '_ORIG']
else:
self._env_copy.pop(variable, None)
def __init__(self, conf: TranscodeConfig): def __init__(self, conf: TranscodeConfig):
self._conf = conf self._conf = conf
self._available_encoders = "" self._available_encoders = ""
self._ffmpeg_installed = False self._ffmpeg_installed = None
self._which = None self._which_ffmpeg = None
self._which_ffprobe = None
self._env_copy = dict(os.environ)
self._checked_ffmpeg = False
if lbry.utils.is_running_from_bundle():
# handle the situation where PyInstaller overrides our runtime environment:
self._replace_or_pop_env('LD_LIBRARY_PATH')
async def _execute(self, command, arguments): @staticmethod
args = shlex.split(arguments) def _execute(command, environment):
process = await asyncio.create_subprocess_exec(self._conf.ffmpeg_folder + command, *args, # log.debug("Executing: %s", command)
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
stdout, stderr = await process.communicate() # returns when the streams are closed
return stdout.decode() + stderr.decode(), process.returncode
async def _verify_executable(self, name):
try: try:
version, code = await self._execute(name, "-version") with subprocess.Popen(
shlex.split(command) if platform.system() != 'Windows' else command,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=environment
) as process:
(stdout, stderr) = process.communicate() # blocks until the process exits
return stdout.decode(errors='replace') + stderr.decode(errors='replace'), process.returncode
except subprocess.SubprocessError as e:
return str(e), -1
# This create_subprocess_exec call is broken in Windows Python 3.7, but it's prettier than what's here.
# The recommended fix is switching to ProactorEventLoop, but that breaks UDP in Linux Python 3.7.
# We work around that issue here by using run_in_executor. Check it again in Python 3.8.
async def _execute_ffmpeg(self, arguments):
arguments = self._which_ffmpeg + " " + arguments
return await asyncio.get_event_loop().run_in_executor(None, self._execute, arguments, self._env_copy)
async def _execute_ffprobe(self, arguments):
arguments = self._which_ffprobe + " " + arguments
return await asyncio.get_event_loop().run_in_executor(None, self._execute, arguments, self._env_copy)
async def _verify_executables(self):
try:
await self._execute_ffprobe("-version")
version, code = await self._execute_ffmpeg("-version")
except Exception as e: except Exception as e:
log.warning("Unable to run %s, but it was requested. Message: %s", name, str(e))
code = -1 code = -1
version = "" version = str(e)
if code != 0 or not version.startswith(name): if code != 0 or not version.startswith("ffmpeg"):
raise FileNotFoundError(f"Unable to locate or run {name}. Please install FFmpeg " log.warning("Unable to run ffmpeg, but it was requested. Code: %d; Message: %s", code, version)
f"and ensure that it is callable via PATH or conf.ffmpeg_folder") raise FileNotFoundError("Unable to locate or run ffmpeg or ffprobe. Please install FFmpeg "
"and ensure that it is callable via PATH or conf.ffmpeg_path")
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which_ffmpeg)
return version return version
@staticmethod
def _which_ffmpeg_and_ffmprobe(path):
return shutil.which("ffmpeg", path=path), shutil.which("ffprobe", path=path)
async def _verify_ffmpeg_installed(self): async def _verify_ffmpeg_installed(self):
if self._ffmpeg_installed: if self._ffmpeg_installed:
return return
await self._verify_executable("ffprobe") self._ffmpeg_installed = False
version = await self._verify_executable("ffmpeg") path = self._conf.ffmpeg_path
self._which = shutil.which(f"{self._conf.ffmpeg_folder}ffmpeg") if hasattr(self._conf, "data_dir"):
self._ffmpeg_installed = True path += os.path.pathsep + os.path.join(getattr(self._conf, "data_dir"), "ffmpeg", "bin")
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which) path += os.path.pathsep + self._env_copy.get("PATH", "")
self._which_ffmpeg, self._which_ffprobe = await asyncio.get_running_loop().run_in_executor(
None, self._which_ffmpeg_and_ffmprobe, path
)
if not self._which_ffmpeg:
log.warning("Unable to locate ffmpeg executable. Path: %s", path)
raise FileNotFoundError(f"Unable to locate ffmpeg executable. Path: {path}")
if not self._which_ffprobe:
log.warning("Unable to locate ffprobe executable. Path: %s", path)
raise FileNotFoundError(f"Unable to locate ffprobe executable. Path: {path}")
if os.path.dirname(self._which_ffmpeg) != os.path.dirname(self._which_ffprobe):
log.warning("ffmpeg and ffprobe are in different folders!")
async def status(self, reset=False): await self._verify_executables()
self._ffmpeg_installed = True
async def status(self, reset=False, recheck=False):
if reset: if reset:
self._available_encoders = "" self._available_encoders = ""
self._ffmpeg_installed = False self._ffmpeg_installed = None
self._which = None if self._checked_ffmpeg and not recheck:
pass
installed = True elif self._ffmpeg_installed is None:
try: try:
await self._verify_ffmpeg_installed() await self._verify_ffmpeg_installed()
except FileNotFoundError: except FileNotFoundError:
installed = False pass
self._checked_ffmpeg = True
return { return {
"available": installed, "available": self._ffmpeg_installed,
"which": self._which, "which": self._which_ffmpeg,
"analyze_audio_volume": int(self._conf.volume_analysis_time) > 0 "analyze_audio_volume": int(self._conf.volume_analysis_time) > 0
} }
@ -70,9 +123,24 @@ class VideoFileAnalyzer:
def _verify_container(scan_data: json): def _verify_container(scan_data: json):
container = scan_data["format"]["format_name"] container = scan_data["format"]["format_name"]
log.debug(" Detected container is %s", container) log.debug(" Detected container is %s", container)
if not {"webm", "mp4", "3gp", "ogg"}.intersection(container.split(",")): splits = container.split(",")
if not {"webm", "mp4", "3gp", "ogg"}.intersection(splits):
return "Container format is not in the approved list of WebM, MP4. " \ return "Container format is not in the approved list of WebM, MP4. " \
f"Actual: {container} [{scan_data['format']['format_long_name']}]" f"Actual: {container} [{scan_data['format']['format_long_name']}]"
if "matroska" in splits:
for stream in scan_data["streams"]:
if stream["codec_type"] == "video":
codec = stream["codec_name"]
if not {"vp8", "vp9", "av1"}.intersection(codec.split(",")):
return "WebM format requires VP8/9 or AV1 video. " \
f"Actual: {codec} [{stream['codec_long_name']}]"
elif stream["codec_type"] == "audio":
codec = stream["codec_name"]
if not {"vorbis", "opus"}.intersection(codec.split(",")):
return "WebM format requires Vorbis or Opus audio. " \
f"Actual: {codec} [{stream['codec_long_name']}]"
return "" return ""
@staticmethod @staticmethod
@ -92,24 +160,21 @@ class VideoFileAnalyzer:
return "" return ""
@staticmethod def _verify_bitrate(self, scan_data: json, file_path):
def _verify_bitrate(scan_data: json): bit_rate_max = float(self._conf.video_bitrate_maximum)
if "bit_rate" not in scan_data["format"]: if bit_rate_max <= 0:
return "" return ""
if "bit_rate" in scan_data["format"]:
bit_rate = float(scan_data["format"]["bit_rate"]) bit_rate = float(scan_data["format"]["bit_rate"])
log.debug(" Detected bitrate is %s Mbps", str(bit_rate / 1000000.0)) else:
pixels = -1.0 bit_rate = os.stat(file_path).st_size / float(scan_data["format"]["duration"])
for stream in scan_data["streams"]: log.debug(" Detected bitrate is %s Mbps. Allowed max: %s Mbps",
if stream["codec_type"] == "video": str(bit_rate / 1000000.0), str(bit_rate_max / 1000000.0))
pieces = stream["r_frame_rate"].split('/', 1)
frame_rate = float(pieces[0]) if len(pieces) == 1 \
else float(pieces[0]) / float(pieces[1])
pixels = max(pixels, float(stream["height"]) * float(stream["width"]) * frame_rate)
if pixels > 0.0 and pixels / bit_rate < 3.0: if bit_rate > bit_rate_max:
return "Bits per second is excessive for this data; this may impact web streaming performance. " \ return "The bit rate is above the configured maximum. Actual: " \
f"Actual: {str(bit_rate / 1000000.0)} Mbps" f"{bit_rate / 1000000.0} Mbps; Allowed max: {bit_rate_max / 1000000.0} Mbps"
return "" return ""
@ -118,10 +183,9 @@ class VideoFileAnalyzer:
if {"webm", "ogg"}.intersection(container.split(",")): if {"webm", "ogg"}.intersection(container.split(",")):
return "" return ""
result, _ = await self._execute("ffprobe", f'-v debug "{video_file}"') result, _ = await self._execute_ffprobe(f'-v debug "{video_file}"')
iterator = re.finditer(r"\s+seeks:(\d+)\s+", result) match = re.search(r"Before avformat_find_stream_info.+?\s+seeks:(\d+)\s+", result)
for match in iterator: if match and int(match.group(1)) != 0:
if int(match.group(1)) != 0:
return "Video stream descriptors are not at the start of the file (the faststart flag was not used)." return "Video stream descriptors are not at the start of the file (the faststart flag was not used)."
return "" return ""
@ -135,6 +199,8 @@ class VideoFileAnalyzer:
if not {"aac", "mp3", "flac", "vorbis", "opus"}.intersection(codec.split(",")): if not {"aac", "mp3", "flac", "vorbis", "opus"}.intersection(codec.split(",")):
return "Audio codec is not in the approved list of AAC, FLAC, MP3, Vorbis, and Opus. " \ return "Audio codec is not in the approved list of AAC, FLAC, MP3, Vorbis, and Opus. " \
f"Actual: {codec} [{stream['codec_long_name']}]" f"Actual: {codec} [{stream['codec_long_name']}]"
if int(stream['sample_rate']) > 48000:
return "Sample rate out of range"
return "" return ""
@ -147,7 +213,7 @@ class VideoFileAnalyzer:
if not validate_volume: if not validate_volume:
return "" return ""
result, _ = await self._execute("ffmpeg", f'-i "{video_file}" -t {seconds} ' result, _ = await self._execute_ffmpeg(f'-i "{video_file}" -t {seconds} '
f'-af volumedetect -vn -sn -dn -f null "{os.devnull}"') f'-af volumedetect -vn -sn -dn -f null "{os.devnull}"')
try: try:
mean_volume = float(re.search(r"mean_volume:\s+([-+]?\d*\.\d+|\d+)", result).group(1)) mean_volume = float(re.search(r"mean_volume:\s+([-+]?\d*\.\d+|\d+)", result).group(1))
@ -174,13 +240,16 @@ class VideoFileAnalyzer:
# https://developers.google.com/media/vp9/settings/vod/ # https://developers.google.com/media/vp9/settings/vod/
return int(-0.011 * height + 40) return int(-0.011 * height + 40)
def _get_video_scaler(self):
return self._conf.video_scaler
async def _get_video_encoder(self, scan_data): async def _get_video_encoder(self, scan_data):
# use what the user said if it's there: # use what the user said if it's there:
# if it's not there, use h264 if we can because it's way faster than the others # if it's not there, use h264 if we can because it's way faster than the others
# if we don't have h264 use vp9; it's fairly compatible even though it's slow # if we don't have h264 use vp9; it's fairly compatible even though it's slow
if not self._available_encoders: if not self._available_encoders:
self._available_encoders, _ = await self._execute("ffmpeg", "-encoders -v quiet") self._available_encoders, _ = await self._execute_ffmpeg("-encoders -v quiet")
encoder = self._conf.video_encoder.split(" ", 1)[0] encoder = self._conf.video_encoder.split(" ", 1)[0]
if re.search(fr"^\s*V..... {encoder} ", self._available_encoders, re.MULTILINE): if re.search(fr"^\s*V..... {encoder} ", self._available_encoders, re.MULTILINE):
@ -214,7 +283,7 @@ class VideoFileAnalyzer:
wants_opus = extension != "mp4" wants_opus = extension != "mp4"
if not self._available_encoders: if not self._available_encoders:
self._available_encoders, _ = await self._execute("ffmpeg", "-encoders -v quiet") self._available_encoders, _ = await self._execute_ffmpeg("-encoders -v quiet")
encoder = self._conf.audio_encoder.split(" ", 1)[0] encoder = self._conf.audio_encoder.split(" ", 1)[0]
if wants_opus and 'opus' in encoder: if wants_opus and 'opus' in encoder:
@ -237,9 +306,6 @@ class VideoFileAnalyzer:
raise Exception(f"The audio encoder is not available. Requested: {encoder or 'aac'}") raise Exception(f"The audio encoder is not available. Requested: {encoder or 'aac'}")
async def _get_volume_filter(self):
return self._conf.volume_filter if self._conf.volume_filter else "-af loudnorm"
@staticmethod @staticmethod
def _get_best_container_extension(scan_data, video_encoder): def _get_best_container_extension(scan_data, video_encoder):
# the container is chosen by the video format # the container is chosen by the video format
@ -247,61 +313,93 @@ class VideoFileAnalyzer:
# if we are vp8/vp9/av1 we want webm # if we are vp8/vp9/av1 we want webm
# use mp4 for anything else # use mp4 for anything else
if not video_encoder: # not re-encoding video if video_encoder: # not re-encoding video
if "theora" in video_encoder:
return "ogv"
if re.search(r"vp[89x]|av1", video_encoder.split(" ", 1)[0]):
return "webm"
return "mp4"
for stream in scan_data["streams"]: for stream in scan_data["streams"]:
if stream["codec_type"] != "video": if stream["codec_type"] != "video":
continue continue
codec = stream["codec_name"].split(",") codec = stream["codec_name"].split(",")
if "theora" in codec: if "theora" in codec:
return "ogg" return "ogv"
if {"vp8", "vp9", "av1"}.intersection(codec): if {"vp8", "vp9", "av1"}.intersection(codec):
return "webm" return "webm"
if "theora" in video_encoder:
return "ogg"
elif re.search(r"vp[89x]|av1", video_encoder.split(" ", 1)[0]):
return "webm"
return "mp4" return "mp4"
async def _get_scan_data(self, validate, file_path): async def _get_scan_data(self, validate, file_path):
result, _ = await self._execute("ffprobe", arguments = f'-v quiet -print_format json -show_format -show_streams "{file_path}"'
f'-v quiet -print_format json -show_format -show_streams "{file_path}"') result, _ = await self._execute_ffprobe(arguments)
try: try:
scan_data = json.loads(result) scan_data = json.loads(result)
except Exception as e: except Exception as e:
log.debug("Failure in JSON parsing ffprobe results. Message: %s", str(e)) log.debug("Failure in JSON parsing ffprobe results. Message: %s", str(e))
if validate: raise ValueError(f'Absent or unreadable video file: {file_path}')
raise Exception(f'Invalid video file: {file_path}')
log.info("Unable to optimize %s . FFmpeg output was unreadable.", file_path)
return
if "format" not in scan_data: if "format" not in scan_data or "duration" not in scan_data["format"]:
if validate: log.debug("Format data is missing from ffprobe results for: %s", file_path)
raise FileNotFoundError(f'Unexpected or absent video file contents at: {file_path}') raise ValueError(f'Media file does not appear to contain video content: {file_path}')
log.info("Unable to optimize %s . FFmpeg output is missing the format section.", file_path)
return if float(scan_data["format"]["duration"]) < 0.1:
log.debug("Media file appears to be an image: %s", file_path)
raise ValueError(f'Assuming image file at: {file_path}')
return scan_data return scan_data
async def verify_or_repair(self, validate, repair, file_path): @staticmethod
def _build_spec(scan_data):
assert scan_data
duration = ceil(float(scan_data["format"]["duration"])) # existence verified when scan_data made
width = -1
height = -1
for stream in scan_data["streams"]:
if stream["codec_type"] != "video":
continue
width = max(width, int(stream["width"]))
height = max(height, int(stream["height"]))
log.debug(" Detected duration: %d sec. with resolution: %d x %d", duration, width, height)
spec = {"duration": duration}
if height >= 0:
spec["height"] = height
if width >= 0:
spec["width"] = width
return spec
async def verify_or_repair(self, validate, repair, file_path, ignore_non_video=False):
if not validate and not repair: if not validate and not repair:
return file_path return file_path, {}
if ignore_non_video and not file_path:
return file_path, {}
await self._verify_ffmpeg_installed() await self._verify_ffmpeg_installed()
try:
scan_data = await self._get_scan_data(validate, file_path) scan_data = await self._get_scan_data(validate, file_path)
except ValueError:
if ignore_non_video:
return file_path, {}
raise
fast_start_msg = await self._verify_fast_start(scan_data, file_path) fast_start_msg = await self._verify_fast_start(scan_data, file_path)
log.debug("Analyzing %s:", file_path) log.debug("Analyzing %s:", file_path)
spec = self._build_spec(scan_data)
log.debug(" Detected faststart is %s", "false" if fast_start_msg else "true") log.debug(" Detected faststart is %s", "false" if fast_start_msg else "true")
container_msg = self._verify_container(scan_data) container_msg = self._verify_container(scan_data)
bitrate_msg = self._verify_bitrate(scan_data) bitrate_msg = self._verify_bitrate(scan_data, file_path)
video_msg = self._verify_video_encoding(scan_data) video_msg = self._verify_video_encoding(scan_data)
audio_msg = self._verify_audio_encoding(scan_data) audio_msg = self._verify_audio_encoding(scan_data)
volume_msg = await self._verify_audio_volume(self._conf.volume_analysis_time, file_path) volume_msg = await self._verify_audio_volume(self._conf.volume_analysis_time, file_path)
messages = [container_msg, bitrate_msg, fast_start_msg, video_msg, audio_msg, volume_msg] messages = [container_msg, bitrate_msg, fast_start_msg, video_msg, audio_msg, volume_msg]
if not any(messages): if not any(messages):
return file_path return file_path, spec
if not repair: if not repair:
errors = ["Streamability verification failed:"] errors = ["Streamability verification failed:"]
@ -311,7 +409,7 @@ class VideoFileAnalyzer:
# the plan for transcoding: # the plan for transcoding:
# we have to re-encode the video if it is in a nonstandard format # we have to re-encode the video if it is in a nonstandard format
# we also re-encode if we are h264 but not yuv420p (both errors caught in video_msg) # we also re-encode if we are h264 but not yuv420p (both errors caught in video_msg)
# we also re-encode if our bitrate is too high # we also re-encode if our bitrate or sample rate is too high
try: try:
transcode_command = [f'-i "{file_path}" -y -c:s copy -c:d copy -c:v'] transcode_command = [f'-i "{file_path}" -y -c:s copy -c:d copy -c:v']
@ -320,29 +418,32 @@ class VideoFileAnalyzer:
if video_msg or bitrate_msg: if video_msg or bitrate_msg:
video_encoder = await self._get_video_encoder(scan_data) video_encoder = await self._get_video_encoder(scan_data)
transcode_command.append(video_encoder) transcode_command.append(video_encoder)
# could do the scaling only if bitrate_msg, but if we're going to the effort to re-encode anyway...
transcode_command.append(self._get_video_scaler())
else: else:
transcode_command.append("copy") transcode_command.append("copy")
transcode_command.append("-movflags +faststart -c:a") transcode_command.append("-movflags +faststart -c:a")
path = pathlib.Path(file_path)
extension = self._get_best_container_extension(scan_data, video_encoder) extension = self._get_best_container_extension(scan_data, video_encoder)
if audio_msg or volume_msg: if audio_msg or volume_msg:
audio_encoder = await self._get_audio_encoder(extension) audio_encoder = await self._get_audio_encoder(extension)
transcode_command.append(audio_encoder) transcode_command.append(audio_encoder)
if volume_msg: if volume_msg and self._conf.volume_filter:
volume_filter = await self._get_volume_filter() transcode_command.append(self._conf.volume_filter)
transcode_command.append(volume_filter) if audio_msg == "Sample rate out of range":
transcode_command.append(" -ar 48000 ")
else: else:
transcode_command.append("copy") transcode_command.append("copy")
# TODO: put it in a temp folder and delete it after we upload? # TODO: put it in a temp folder and delete it after we upload?
path = pathlib.Path(file_path)
output = path.parent / f"{path.stem}_fixed.{extension}" output = path.parent / f"{path.stem}_fixed.{extension}"
transcode_command.append(f'"{output}"') transcode_command.append(f'"{output}"')
ffmpeg_command = " ".join(transcode_command) ffmpeg_command = " ".join(transcode_command)
log.info("Proceeding on transcode via: ffmpeg %s", ffmpeg_command) log.info("Proceeding on transcode via: ffmpeg %s", ffmpeg_command)
result, code = await self._execute("ffmpeg", ffmpeg_command) result, code = await self._execute_ffmpeg(ffmpeg_command)
if code != 0: if code != 0:
raise Exception(f"Failure to complete the transcode command. Output: {result}") raise Exception(f"Failure to complete the transcode command. Output: {result}")
except Exception as e: except Exception as e:
@ -350,6 +451,6 @@ class VideoFileAnalyzer:
raise raise
log.info("Unable to transcode %s . Message: %s", file_path, str(e)) log.info("Unable to transcode %s . Message: %s", file_path, str(e))
# TODO: delete partial output file here if it exists? # TODO: delete partial output file here if it exists?
return file_path return file_path, spec
return str(output) return str(output), spec

68
lbry/prometheus.py Normal file
View file

@ -0,0 +1,68 @@
import time
import logging
import asyncio
import asyncio.tasks
from aiohttp import web
from prometheus_client import generate_latest as prom_generate_latest
from prometheus_client import Counter, Histogram, Gauge
PROBES_IN_FLIGHT = Counter("probes_in_flight", "Number of loop probes in flight", namespace='asyncio')
PROBES_FINISHED = Counter("probes_finished", "Number of finished loop probes", namespace='asyncio')
PROBE_TIMES = Histogram("probe_times", "Loop probe times", namespace='asyncio')
TASK_COUNT = Gauge("running_tasks", "Number of running tasks", namespace='asyncio')
def get_loop_metrics(delay=1):
loop = asyncio.get_event_loop()
def callback(started):
PROBE_TIMES.observe(time.perf_counter() - started - delay)
PROBES_FINISHED.inc()
async def monitor_loop_responsiveness():
while True:
now = time.perf_counter()
loop.call_later(delay, callback, now)
PROBES_IN_FLIGHT.inc()
TASK_COUNT.set(len(asyncio.tasks._all_tasks))
await asyncio.sleep(delay)
return loop.create_task(monitor_loop_responsiveness())
class PrometheusServer:
def __init__(self, logger=None):
self.runner = None
self.logger = logger or logging.getLogger(__name__)
self._monitor_loop_task = None
async def start(self, interface: str, port: int):
self.logger.info("start prometheus metrics")
prom_app = web.Application()
prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
self.runner = web.AppRunner(prom_app)
await self.runner.setup()
metrics_site = web.TCPSite(self.runner, interface, port, shutdown_timeout=.5)
await metrics_site.start()
self.logger.info(
'prometheus metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2]
)
self._monitor_loop_task = get_loop_metrics()
async def handle_metrics_get_request(self, request: web.Request):
try:
return web.Response(
text=prom_generate_latest().decode(),
content_type='text/plain; version=0.0.4'
)
except Exception:
self.logger.exception('could not generate prometheus data')
raise
async def stop(self):
if self._monitor_loop_task and not self._monitor_loop_task.done():
self._monitor_loop_task.cancel()
self._monitor_loop_task = None
await self.runner.cleanup()

View file

@ -2,4 +2,5 @@ build:
rm types/v2/* -rf rm types/v2/* -rf
touch types/v2/__init__.py touch types/v2/__init__.py
cd types/v2/ && protoc --python_out=. -I ../../../../../types/v2/proto/ ../../../../../types/v2/proto/*.proto cd types/v2/ && protoc --python_out=. -I ../../../../../types/v2/proto/ ../../../../../types/v2/proto/*.proto
cd types/v2/ && cp ../../../../../types/jsonschema/* ./
sed -e 's/^import\ \(.*\)_pb2\ /from . import\ \1_pb2\ /g' -i types/v2/*.py sed -e 's/^import\ \(.*\)_pb2\ /from . import\ \1_pb2\ /g' -i types/v2/*.py

24
lbry/schema/README.md Normal file
View file

@ -0,0 +1,24 @@
Schema
=====
Those files are generated from the [types repo](https://github.com/lbryio/types). If you are modifying/adding a new type, make sure it is cloned in the same root folder as the SDK repo, like:
```
repos/
- lbry-sdk/
- types/
```
Then, [download protoc 3.2.0](https://github.com/protocolbuffers/protobuf/releases/tag/v3.2.0), add it to your PATH. On linux it is:
```bash
cd ~/.local/bin
wget https://github.com/protocolbuffers/protobuf/releases/download/v3.2.0/protoc-3.2.0-linux-x86_64.zip
unzip protoc-3.2.0-linux-x86_64.zip bin/protoc -d..
```
Finally, `make` should update everything in place.
### Why protoc 3.2.0?
Different/newer versions will generate larger diffs and we need to make sure they are good. In theory, we can just update to latest and it will all work, but it is a good practice to check blockchain data and retro compatibility before bumping versions (if you do, please update this section!).

View file

@ -10,6 +10,7 @@ from google.protobuf.json_format import MessageToDict
from lbry.crypto.base58 import Base58 from lbry.crypto.base58 import Base58
from lbry.constants import COIN from lbry.constants import COIN
from lbry.error import MissingPublishedFileError, EmptyPublishedFileError
from lbry.schema.mime_types import guess_media_type from lbry.schema.mime_types import guess_media_type
from lbry.schema.base import Metadata, BaseMessageList from lbry.schema.base import Metadata, BaseMessageList
@ -32,6 +33,17 @@ def calculate_sha384_file_hash(file_path):
return sha384.digest() return sha384.digest()
def country_int_to_str(country: int) -> str:
r = LocationMessage.Country.Name(country)
return r[1:] if r.startswith('R') else r
def country_str_to_int(country: str) -> int:
if len(country) == 3:
country = 'R' + country
return LocationMessage.Country.Value(country)
class Dimmensional(Metadata): class Dimmensional(Metadata):
__slots__ = () __slots__ = ()
@ -128,10 +140,10 @@ class Source(Metadata):
self.name = os.path.basename(file_path) self.name = os.path.basename(file_path)
self.media_type, stream_type = guess_media_type(file_path) self.media_type, stream_type = guess_media_type(file_path)
if not os.path.isfile(file_path): if not os.path.isfile(file_path):
raise Exception(f"File does not exist: {file_path}") raise MissingPublishedFileError(file_path)
self.size = os.path.getsize(file_path) self.size = os.path.getsize(file_path)
if self.size == 0: if self.size == 0:
raise Exception(f"Cannot publish empty file: {file_path}") raise EmptyPublishedFileError(file_path)
self.file_hash_bytes = calculate_sha384_file_hash(file_path) self.file_hash_bytes = calculate_sha384_file_hash(file_path)
return stream_type return stream_type
@ -423,14 +435,11 @@ class Language(Metadata):
@property @property
def region(self) -> str: def region(self) -> str:
if self.message.region: if self.message.region:
r = LocationMessage.Country.Name(self.message.region) return country_int_to_str(self.message.region)
return r[1:] if r.startswith('R') else r
@region.setter @region.setter
def region(self, region: str): def region(self, region: str):
if len(region) == 3: self.message.region = country_str_to_int(region)
region = 'R'+region
self.message.region = LocationMessage.Country.Value(region)
class LanguageList(BaseMessageList[Language]): class LanguageList(BaseMessageList[Language]):

View file

@ -2,6 +2,9 @@ import logging
from typing import List from typing import List
from binascii import hexlify, unhexlify from binascii import hexlify, unhexlify
from asn1crypto.keys import PublicKeyInfo
from coincurve import PublicKey as cPublicKey
from google.protobuf.json_format import MessageToDict from google.protobuf.json_format import MessageToDict
from google.protobuf.message import DecodeError from google.protobuf.message import DecodeError
from hachoir.core.log import log as hachoir_log from hachoir.core.log import log as hachoir_log
@ -30,14 +33,10 @@ class Claim(Signable):
COLLECTION = 'collection' COLLECTION = 'collection'
REPOST = 'repost' REPOST = 'repost'
__slots__ = 'version', __slots__ = ()
message_class = ClaimMessage message_class = ClaimMessage
def __init__(self, message=None):
super().__init__(message)
self.version = 2
@property @property
def claim_type(self) -> str: def claim_type(self) -> str:
return self.message.WhichOneof('type') return self.message.WhichOneof('type')
@ -253,7 +252,7 @@ class Stream(BaseClaim):
if stream_type in ('image', 'video', 'audio'): if stream_type in ('image', 'video', 'audio'):
media = getattr(self, stream_type) media = getattr(self, stream_type)
media_args = {'file_metadata': None} media_args = {'file_metadata': None}
if file_path is not None: if file_path is not None and not all((duration, width, height)):
try: try:
media_args['file_metadata'] = binary_file_metadata(binary_file_parser(file_path)) media_args['file_metadata'] = binary_file_metadata(binary_file_parser(file_path))
except: except:
@ -307,6 +306,10 @@ class Stream(BaseClaim):
def has_fee(self) -> bool: def has_fee(self) -> bool:
return self.message.HasField('fee') return self.message.HasField('fee')
@property
def has_source(self) -> bool:
return self.message.HasField('source')
@property @property
def source(self) -> Source: def source(self) -> Source:
return Source(self.message.source) return Source(self.message.source)
@ -346,7 +349,7 @@ class Channel(BaseClaim):
@property @property
def public_key(self) -> str: def public_key(self) -> str:
return hexlify(self.message.public_key).decode() return hexlify(self.public_key_bytes).decode()
@public_key.setter @public_key.setter
def public_key(self, sd_public_key: str): def public_key(self, sd_public_key: str):
@ -354,7 +357,11 @@ class Channel(BaseClaim):
@property @property
def public_key_bytes(self) -> bytes: def public_key_bytes(self) -> bytes:
if len(self.message.public_key) == 33:
return self.message.public_key return self.message.public_key
public_key_info = PublicKeyInfo.load(self.message.public_key)
public_key = cPublicKey(public_key_info.native['public_key'])
return public_key.format(compressed=True)
@public_key_bytes.setter @public_key_bytes.setter
def public_key_bytes(self, public_key: bytes): def public_key_bytes(self, public_key: bytes):
@ -391,6 +398,12 @@ class Repost(BaseClaim):
claim_type = Claim.REPOST claim_type = Claim.REPOST
def to_dict(self):
claim = super().to_dict()
if claim.pop('claim_hash', None):
claim['claim_id'] = self.reference.claim_id
return claim
@property @property
def reference(self) -> ClaimReference: def reference(self) -> ClaimReference:
return ClaimReference(self.message) return ClaimReference(self.message)

View file

@ -1,4 +1,6 @@
import os import os
import filetype
import logging
types_map = { types_map = {
# http://www.iana.org/assignments/media-types # http://www.iana.org/assignments/media-types
@ -46,8 +48,8 @@ types_map = {
'.ksh': ('text/plain', 'document'), '.ksh': ('text/plain', 'document'),
'.latex': ('application/x-latex', 'binary'), '.latex': ('application/x-latex', 'binary'),
'.m1v': ('video/mpeg', 'video'), '.m1v': ('video/mpeg', 'video'),
'.m3u': ('application/vnd.apple.mpegurl', 'audio'), '.m3u': ('application/x-mpegurl', 'audio'),
'.m3u8': ('application/vnd.apple.mpegurl', 'audio'), '.m3u8': ('application/x-mpegurl', 'video'),
'.man': ('application/x-troff-man', 'document'), '.man': ('application/x-troff-man', 'document'),
'.markdown': ('text/markdown', 'document'), '.markdown': ('text/markdown', 'document'),
'.md': ('text/markdown', 'document'), '.md': ('text/markdown', 'document'),
@ -61,10 +63,12 @@ types_map = {
'.mp3': ('audio/mpeg', 'audio'), '.mp3': ('audio/mpeg', 'audio'),
'.mp4': ('video/mp4', 'video'), '.mp4': ('video/mp4', 'video'),
'.mpa': ('video/mpeg', 'video'), '.mpa': ('video/mpeg', 'video'),
'.mpd': ('application/dash+xml', 'video'),
'.mpe': ('video/mpeg', 'video'), '.mpe': ('video/mpeg', 'video'),
'.mpeg': ('video/mpeg', 'video'), '.mpeg': ('video/mpeg', 'video'),
'.mpg': ('video/mpeg', 'video'), '.mpg': ('video/mpeg', 'video'),
'.ms': ('application/x-troff-ms', 'binary'), '.ms': ('application/x-troff-ms', 'binary'),
'.m4s': ('video/iso.segment', 'binary'),
'.nc': ('application/x-netcdf', 'binary'), '.nc': ('application/x-netcdf', 'binary'),
'.nws': ('message/rfc822', 'document'), '.nws': ('message/rfc822', 'document'),
'.o': ('application/octet-stream', 'binary'), '.o': ('application/octet-stream', 'binary'),
@ -118,10 +122,12 @@ types_map = {
'.tif': ('image/tiff', 'image'), '.tif': ('image/tiff', 'image'),
'.tiff': ('image/tiff', 'image'), '.tiff': ('image/tiff', 'image'),
'.tr': ('application/x-troff', 'binary'), '.tr': ('application/x-troff', 'binary'),
'.ts': ('video/mp2t', 'video'),
'.tsv': ('text/tab-separated-values', 'document'), '.tsv': ('text/tab-separated-values', 'document'),
'.txt': ('text/plain', 'document'), '.txt': ('text/plain', 'document'),
'.ustar': ('application/x-ustar', 'binary'), '.ustar': ('application/x-ustar', 'binary'),
'.vcf': ('text/x-vcard', 'document'), '.vcf': ('text/x-vcard', 'document'),
'.vtt': ('text/vtt', 'document'),
'.wav': ('audio/x-wav', 'audio'), '.wav': ('audio/x-wav', 'audio'),
'.webm': ('video/webm', 'video'), '.webm': ('video/webm', 'video'),
'.wiz': ('application/msword', 'document'), '.wiz': ('application/msword', 'document'),
@ -141,6 +147,7 @@ types_map = {
'.cbz': ('application/vnd.comicbook+zip', 'document'), '.cbz': ('application/vnd.comicbook+zip', 'document'),
'.flac': ('audio/flac', 'audio'), '.flac': ('audio/flac', 'audio'),
'.lbry': ('application/x-ext-lbry', 'document'), '.lbry': ('application/x-ext-lbry', 'document'),
'.m4a': ('audio/mp4', 'audio'),
'.m4v': ('video/m4v', 'video'), '.m4v': ('video/m4v', 'video'),
'.mid': ('audio/midi', 'audio'), '.mid': ('audio/midi', 'audio'),
'.midi': ('audio/midi', 'audio'), '.midi': ('audio/midi', 'audio'),
@ -148,6 +155,7 @@ types_map = {
'.mobi': ('application/x-mobipocket-ebook', 'document'), '.mobi': ('application/x-mobipocket-ebook', 'document'),
'.oga': ('audio/ogg', 'audio'), '.oga': ('audio/ogg', 'audio'),
'.ogv': ('video/ogg', 'video'), '.ogv': ('video/ogg', 'video'),
'.ogg': ('video/ogg', 'video'),
'.pct': ('image/pict', 'image'), '.pct': ('image/pict', 'image'),
'.pic': ('image/pict', 'image'), '.pic': ('image/pict', 'image'),
'.pict': ('image/pict', 'image'), '.pict': ('image/pict', 'image'),
@ -160,10 +168,38 @@ types_map = {
'.wmv': ('video/x-ms-wmv', 'video') '.wmv': ('video/x-ms-wmv', 'video')
} }
# maps detected extensions to the possible analogs
# i.e. .cbz file is actually a .zip
synonyms_map = {
'.zip': ['.cbz'],
'.rar': ['.cbr'],
'.ar': ['.a']
}
log = logging.getLogger(__name__)
def guess_media_type(path): def guess_media_type(path):
_, ext = os.path.splitext(path) _, ext = os.path.splitext(path)
extension = ext.strip().lower() extension = ext.strip().lower()
try:
kind = filetype.guess(path)
if kind:
real_extension = f".{kind.extension}"
if extension != real_extension:
if extension:
log.warning(f"file extension does not match it's contents: {path}, identified as {real_extension}")
else:
log.debug(f"file {path} does not have extension, identified by it's contents as {real_extension}")
if extension not in synonyms_map.get(real_extension, []):
extension = real_extension
except OSError as error:
pass
if extension[1:]: if extension[1:]:
if extension in types_map: if extension in types_map:
return types_map[extension] return types_map[extension]

View file

@ -1,6 +1,5 @@
import base64 import base64
import struct from typing import List, Union, Optional, NamedTuple
from typing import List
from binascii import hexlify from binascii import hexlify
from itertools import chain from itertools import chain
@ -13,38 +12,73 @@ NOT_FOUND = ErrorMessage.Code.Name(ErrorMessage.NOT_FOUND)
BLOCKED = ErrorMessage.Code.Name(ErrorMessage.BLOCKED) BLOCKED = ErrorMessage.Code.Name(ErrorMessage.BLOCKED)
def set_reference(reference, claim_hash, rows):
if claim_hash:
for txo in rows:
if claim_hash == txo.claim_hash:
reference.tx_hash = txo.tx_hash
reference.nout = txo.position
reference.height = txo.height
return
class ResolveResult(NamedTuple):
name: str
normalized_name: str
claim_hash: bytes
tx_num: int
position: int
tx_hash: bytes
height: int
amount: int
short_url: str
is_controlling: bool
canonical_url: str
creation_height: int
activation_height: int
expiration_height: int
effective_amount: int
support_amount: int
reposted: int
last_takeover_height: Optional[int]
claims_in_channel: Optional[int]
channel_hash: Optional[bytes]
reposted_claim_hash: Optional[bytes]
signature_valid: Optional[bool]
class Censor: class Censor:
__slots__ = 'streams', 'channels', 'censored', 'total' NOT_CENSORED = 0
SEARCH = 1
RESOLVE = 2
def __init__(self, streams: dict = None, channels: dict = None): __slots__ = 'censor_type', 'censored'
self.streams = streams or {}
self.channels = channels or {} def __init__(self, censor_type):
self.censor_type = censor_type
self.censored = {} self.censored = {}
self.total = 0
def censor(self, row) -> bool: def is_censored(self, row):
was_censored = False return (row.get('censor_type') or self.NOT_CENSORED) >= self.censor_type
for claim_hash, lookup in (
(row['claim_hash'], self.streams),
(row['claim_hash'], self.channels),
(row['channel_hash'], self.channels)):
censoring_channel_hash = lookup.get(claim_hash)
if censoring_channel_hash:
was_censored = True
self.censored.setdefault(censoring_channel_hash, 0)
self.censored[censoring_channel_hash] += 1
break
if was_censored:
self.total += 1
return was_censored
def to_message(self, outputs: OutputsMessage): def apply(self, rows):
outputs.blocked_total = self.total return [row for row in rows if not self.censor(row)]
def censor(self, row) -> Optional[bytes]:
if self.is_censored(row):
censoring_channel_hash = bytes.fromhex(row['censoring_channel_id'])[::-1]
self.censored.setdefault(censoring_channel_hash, set())
self.censored[censoring_channel_hash].add(row['tx_hash'])
return censoring_channel_hash
return None
def to_message(self, outputs: OutputsMessage, extra_txo_rows: dict):
for censoring_channel_hash, count in self.censored.items(): for censoring_channel_hash, count in self.censored.items():
block = outputs.blocked.add() blocked = outputs.blocked.add()
block.count = count blocked.count = len(count)
block.channel_hash = censoring_channel_hash set_reference(blocked.channel, censoring_channel_hash, extra_txo_rows)
outputs.blocked_total += len(count)
class Outputs: class Outputs:
@ -66,26 +100,35 @@ class Outputs:
for txo_message in self.extra_txos: for txo_message in self.extra_txos:
self.message_to_txo(txo_message, tx_map) self.message_to_txo(txo_message, tx_map)
txos = [self.message_to_txo(txo_message, tx_map) for txo_message in self.txos] txos = [self.message_to_txo(txo_message, tx_map) for txo_message in self.txos]
return txos, self.inflate_blocked() return txos, self.inflate_blocked(tx_map)
def inflate_blocked(self): def inflate_blocked(self, tx_map):
return { return {
"total": self.blocked_total, "total": self.blocked_total,
"channels": { "channels": [{
hexlify(message.channel_hash[::-1]).decode(): message.count 'channel': self.message_to_txo(blocked.channel, tx_map),
for message in self.blocked 'blocked': blocked.count
} } for blocked in self.blocked]
} }
def message_to_txo(self, txo_message, tx_map): def message_to_txo(self, txo_message, tx_map):
if txo_message.WhichOneof('meta') == 'error': if txo_message.WhichOneof('meta') == 'error':
return { error = {
'error': { 'error': {
'name': txo_message.error.Code.Name(txo_message.error.code).lower(), 'name': txo_message.error.Code.Name(txo_message.error.code),
'text': txo_message.error.text, 'text': txo_message.error.text,
} }
} }
txo = tx_map[txo_message.tx_hash].outputs[txo_message.nout] if error['error']['name'] == BLOCKED:
error['error']['censor'] = self.message_to_txo(
txo_message.error.blocked.channel, tx_map
)
return error
tx = tx_map.get(txo_message.tx_hash)
if not tx:
return
txo = tx.outputs[txo_message.nout]
if txo_message.WhichOneof('meta') == 'claim': if txo_message.WhichOneof('meta') == 'claim':
claim = txo_message.claim claim = txo_message.claim
txo.meta = { txo.meta = {
@ -99,10 +142,10 @@ class Outputs:
'expiration_height': claim.expiration_height, 'expiration_height': claim.expiration_height,
'effective_amount': claim.effective_amount, 'effective_amount': claim.effective_amount,
'support_amount': claim.support_amount, 'support_amount': claim.support_amount,
'trending_group': claim.trending_group, # 'trending_group': claim.trending_group,
'trending_mixed': claim.trending_mixed, # 'trending_mixed': claim.trending_mixed,
'trending_local': claim.trending_local, # 'trending_local': claim.trending_local,
'trending_global': claim.trending_global, # 'trending_global': claim.trending_global,
} }
if claim.HasField('channel'): if claim.HasField('channel'):
txo.channel = tx_map[claim.channel.tx_hash].outputs[claim.channel.nout] txo.channel = tx_map[claim.channel.tx_hash].outputs[claim.channel.nout]
@ -145,59 +188,55 @@ class Outputs:
if total is not None: if total is not None:
page.total = total page.total = total
if blocked is not None: if blocked is not None:
blocked.to_message(page) blocked.to_message(page, extra_txo_rows)
for row in txo_rows:
cls.row_to_message(row, page.txos.add(), extra_txo_rows)
for row in extra_txo_rows: for row in extra_txo_rows:
cls.row_to_message(row, page.extra_txos.add(), extra_txo_rows) txo_message: 'OutputsMessage' = page.extra_txos.add()
if not isinstance(row, Exception):
if row.channel_hash:
set_reference(txo_message.claim.channel, row.channel_hash, extra_txo_rows)
if row.reposted_claim_hash:
set_reference(txo_message.claim.repost, row.reposted_claim_hash, extra_txo_rows)
cls.encode_txo(txo_message, row)
for row in txo_rows:
# cls.row_to_message(row, page.txos.add(), extra_txo_rows)
txo_message: 'OutputsMessage' = page.txos.add()
cls.encode_txo(txo_message, row)
if not isinstance(row, Exception):
if row.channel_hash:
set_reference(txo_message.claim.channel, row.channel_hash, extra_txo_rows)
if row.reposted_claim_hash:
set_reference(txo_message.claim.repost, row.reposted_claim_hash, extra_txo_rows)
elif isinstance(row, ResolveCensoredError):
set_reference(txo_message.error.blocked.channel, row.censor_id, extra_txo_rows)
return page.SerializeToString() return page.SerializeToString()
@classmethod @classmethod
def row_to_message(cls, txo, txo_message, extra_txo_rows): def encode_txo(cls, txo_message, resolve_result: Union['ResolveResult', Exception]):
if isinstance(txo, Exception): if isinstance(resolve_result, Exception):
txo_message.error.text = txo.args[0] txo_message.error.text = resolve_result.args[0]
if isinstance(txo, ValueError): if isinstance(resolve_result, ValueError):
txo_message.error.code = ErrorMessage.INVALID txo_message.error.code = ErrorMessage.INVALID
elif isinstance(txo, LookupError): elif isinstance(resolve_result, LookupError):
txo_message.error.code = ErrorMessage.NOT_FOUND txo_message.error.code = ErrorMessage.NOT_FOUND
elif isinstance(txo, ResolveCensoredError): elif isinstance(resolve_result, ResolveCensoredError):
txo_message.error.code = ErrorMessage.BLOCKED txo_message.error.code = ErrorMessage.BLOCKED
return return
txo_message.tx_hash = txo['txo_hash'][:32] txo_message.tx_hash = resolve_result.tx_hash
txo_message.nout, = struct.unpack('<I', txo['txo_hash'][32:]) txo_message.nout = resolve_result.position
txo_message.height = txo['height'] txo_message.height = resolve_result.height
txo_message.claim.short_url = txo['short_url'] txo_message.claim.short_url = resolve_result.short_url
txo_message.claim.reposted = txo['reposted'] txo_message.claim.reposted = resolve_result.reposted
if txo['canonical_url'] is not None: txo_message.claim.is_controlling = resolve_result.is_controlling
txo_message.claim.canonical_url = txo['canonical_url'] txo_message.claim.creation_height = resolve_result.creation_height
txo_message.claim.is_controlling = bool(txo['is_controlling']) txo_message.claim.activation_height = resolve_result.activation_height
if txo['last_take_over_height'] is not None: txo_message.claim.expiration_height = resolve_result.expiration_height
txo_message.claim.take_over_height = txo['last_take_over_height'] txo_message.claim.effective_amount = resolve_result.effective_amount
txo_message.claim.creation_height = txo['creation_height'] txo_message.claim.support_amount = resolve_result.support_amount
txo_message.claim.activation_height = txo['activation_height']
txo_message.claim.expiration_height = txo['expiration_height']
if txo['claims_in_channel'] is not None:
txo_message.claim.claims_in_channel = txo['claims_in_channel']
txo_message.claim.effective_amount = txo['effective_amount']
txo_message.claim.support_amount = txo['support_amount']
txo_message.claim.trending_group = txo['trending_group']
txo_message.claim.trending_mixed = txo['trending_mixed']
txo_message.claim.trending_local = txo['trending_local']
txo_message.claim.trending_global = txo['trending_global']
cls.set_reference(txo_message, 'channel', txo['channel_hash'], extra_txo_rows)
cls.set_reference(txo_message, 'repost', txo['reposted_claim_hash'], extra_txo_rows)
@staticmethod if resolve_result.canonical_url is not None:
def set_blocked(message, blocked): txo_message.claim.canonical_url = resolve_result.canonical_url
message.blocked_total = blocked.total if resolve_result.last_takeover_height is not None:
txo_message.claim.take_over_height = resolve_result.last_takeover_height
@staticmethod if resolve_result.claims_in_channel is not None:
def set_reference(message, attr, claim_hash, rows): txo_message.claim.claims_in_channel = resolve_result.claims_in_channel
if claim_hash:
for txo in rows:
if claim_hash == txo['claim_hash']:
reference = getattr(message.claim, attr)
reference.tx_hash = txo['txo_hash'][:32]
reference.nout = struct.unpack('<I', txo['txo_hash'][32:])[0]
reference.height = txo['height']
break

View file

@ -1,6 +1,23 @@
from lbry.schema.base import Signable from lbry.schema.base import Signable
from lbry.schema.types.v2.support_pb2 import Support as SupportMessage
class Support(Signable): class Support(Signable):
__slots__ = () __slots__ = ()
message_class = None # TODO: add support protobufs message_class = SupportMessage
@property
def emoji(self) -> str:
return self.message.emoji
@emoji.setter
def emoji(self, emoji: str):
self.message.emoji = emoji
@property
def comment(self) -> str:
return self.message.comment
@comment.setter
def comment(self, comment: str):
self.message.comment = comment

View file

@ -1,13 +1,11 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT! # Generated by the protocol buffer compiler. DO NOT EDIT!
# source: result.proto # source: result.proto
"""Generated protocol buffer code."""
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message from google.protobuf import message as _message
from google.protobuf import reflection as _reflection from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database from google.protobuf import symbol_database as _symbol_database
from google.protobuf import descriptor_pb2
# @@protoc_insertion_point(imports) # @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default() _sym_db = _symbol_database.Default()
@ -19,9 +17,10 @@ DESCRIPTOR = _descriptor.FileDescriptor(
name='result.proto', name='result.proto',
package='pb', package='pb',
syntax='proto3', syntax='proto3',
serialized_pb=_b('\n\x0cresult.proto\x12\x02pb\"\x97\x01\n\x07Outputs\x12\x18\n\x04txos\x18\x01 \x03(\x0b\x32\n.pb.Output\x12\x1e\n\nextra_txos\x18\x02 \x03(\x0b\x32\n.pb.Output\x12\r\n\x05total\x18\x03 \x01(\r\x12\x0e\n\x06offset\x18\x04 \x01(\r\x12\x1c\n\x07\x62locked\x18\x05 \x03(\x0b\x32\x0b.pb.Blocked\x12\x15\n\rblocked_total\x18\x06 \x01(\r\"{\n\x06Output\x12\x0f\n\x07tx_hash\x18\x01 \x01(\x0c\x12\x0c\n\x04nout\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\x1e\n\x05\x63laim\x18\x07 \x01(\x0b\x32\r.pb.ClaimMetaH\x00\x12\x1a\n\x05\x65rror\x18\x0f \x01(\x0b\x32\t.pb.ErrorH\x00\x42\x06\n\x04meta\"\xaf\x03\n\tClaimMeta\x12\x1b\n\x07\x63hannel\x18\x01 \x01(\x0b\x32\n.pb.Output\x12\x1a\n\x06repost\x18\x02 \x01(\x0b\x32\n.pb.Output\x12\x11\n\tshort_url\x18\x03 \x01(\t\x12\x15\n\rcanonical_url\x18\x04 \x01(\t\x12\x16\n\x0eis_controlling\x18\x05 \x01(\x08\x12\x18\n\x10take_over_height\x18\x06 \x01(\r\x12\x17\n\x0f\x63reation_height\x18\x07 \x01(\r\x12\x19\n\x11\x61\x63tivation_height\x18\x08 \x01(\r\x12\x19\n\x11\x65xpiration_height\x18\t \x01(\r\x12\x19\n\x11\x63laims_in_channel\x18\n \x01(\r\x12\x10\n\x08reposted\x18\x0b \x01(\r\x12\x18\n\x10\x65\x66\x66\x65\x63tive_amount\x18\x14 \x01(\x04\x12\x16\n\x0esupport_amount\x18\x15 \x01(\x04\x12\x16\n\x0etrending_group\x18\x16 \x01(\r\x12\x16\n\x0etrending_mixed\x18\x17 \x01(\x02\x12\x16\n\x0etrending_local\x18\x18 \x01(\x02\x12\x17\n\x0ftrending_global\x18\x19 \x01(\x02\"\x94\x01\n\x05\x45rror\x12\x1c\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x0e.pb.Error.Code\x12\x0c\n\x04text\x18\x02 \x01(\t\x12\x1c\n\x07\x62locked\x18\x03 \x01(\x0b\x32\x0b.pb.Blocked\"A\n\x04\x43ode\x12\x10\n\x0cUNKNOWN_CODE\x10\x00\x12\r\n\tNOT_FOUND\x10\x01\x12\x0b\n\x07INVALID\x10\x02\x12\x0b\n\x07\x42LOCKED\x10\x03\".\n\x07\x42locked\x12\r\n\x05\x63ount\x18\x01 \x01(\r\x12\x14\n\x0c\x63hannel_hash\x18\x02 \x01(\x0c\x62\x06proto3') serialized_options=b'Z$github.com/lbryio/hub/protobuf/go/pb',
create_key=_descriptor._internal_create_key,
serialized_pb=b'\n\x0cresult.proto\x12\x02pb\"\x97\x01\n\x07Outputs\x12\x18\n\x04txos\x18\x01 \x03(\x0b\x32\n.pb.Output\x12\x1e\n\nextra_txos\x18\x02 \x03(\x0b\x32\n.pb.Output\x12\r\n\x05total\x18\x03 \x01(\r\x12\x0e\n\x06offset\x18\x04 \x01(\r\x12\x1c\n\x07\x62locked\x18\x05 \x03(\x0b\x32\x0b.pb.Blocked\x12\x15\n\rblocked_total\x18\x06 \x01(\r\"{\n\x06Output\x12\x0f\n\x07tx_hash\x18\x01 \x01(\x0c\x12\x0c\n\x04nout\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\x1e\n\x05\x63laim\x18\x07 \x01(\x0b\x32\r.pb.ClaimMetaH\x00\x12\x1a\n\x05\x65rror\x18\x0f \x01(\x0b\x32\t.pb.ErrorH\x00\x42\x06\n\x04meta\"\xe6\x02\n\tClaimMeta\x12\x1b\n\x07\x63hannel\x18\x01 \x01(\x0b\x32\n.pb.Output\x12\x1a\n\x06repost\x18\x02 \x01(\x0b\x32\n.pb.Output\x12\x11\n\tshort_url\x18\x03 \x01(\t\x12\x15\n\rcanonical_url\x18\x04 \x01(\t\x12\x16\n\x0eis_controlling\x18\x05 \x01(\x08\x12\x18\n\x10take_over_height\x18\x06 \x01(\r\x12\x17\n\x0f\x63reation_height\x18\x07 \x01(\r\x12\x19\n\x11\x61\x63tivation_height\x18\x08 \x01(\r\x12\x19\n\x11\x65xpiration_height\x18\t \x01(\r\x12\x19\n\x11\x63laims_in_channel\x18\n \x01(\r\x12\x10\n\x08reposted\x18\x0b \x01(\r\x12\x18\n\x10\x65\x66\x66\x65\x63tive_amount\x18\x14 \x01(\x04\x12\x16\n\x0esupport_amount\x18\x15 \x01(\x04\x12\x16\n\x0etrending_score\x18\x16 \x01(\x01\"\x94\x01\n\x05\x45rror\x12\x1c\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x0e.pb.Error.Code\x12\x0c\n\x04text\x18\x02 \x01(\t\x12\x1c\n\x07\x62locked\x18\x03 \x01(\x0b\x32\x0b.pb.Blocked\"A\n\x04\x43ode\x12\x10\n\x0cUNKNOWN_CODE\x10\x00\x12\r\n\tNOT_FOUND\x10\x01\x12\x0b\n\x07INVALID\x10\x02\x12\x0b\n\x07\x42LOCKED\x10\x03\"5\n\x07\x42locked\x12\r\n\x05\x63ount\x18\x01 \x01(\r\x12\x1b\n\x07\x63hannel\x18\x02 \x01(\x0b\x32\n.pb.OutputB&Z$github.com/lbryio/hub/protobuf/go/pbb\x06proto3'
) )
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
@ -30,28 +29,33 @@ _ERROR_CODE = _descriptor.EnumDescriptor(
full_name='pb.Error.Code', full_name='pb.Error.Code',
filename=None, filename=None,
file=DESCRIPTOR, file=DESCRIPTOR,
create_key=_descriptor._internal_create_key,
values=[ values=[
_descriptor.EnumValueDescriptor( _descriptor.EnumValueDescriptor(
name='UNKNOWN_CODE', index=0, number=0, name='UNKNOWN_CODE', index=0, number=0,
options=None, serialized_options=None,
type=None), type=None,
create_key=_descriptor._internal_create_key),
_descriptor.EnumValueDescriptor( _descriptor.EnumValueDescriptor(
name='NOT_FOUND', index=1, number=1, name='NOT_FOUND', index=1, number=1,
options=None, serialized_options=None,
type=None), type=None,
create_key=_descriptor._internal_create_key),
_descriptor.EnumValueDescriptor( _descriptor.EnumValueDescriptor(
name='INVALID', index=2, number=2, name='INVALID', index=2, number=2,
options=None, serialized_options=None,
type=None), type=None,
create_key=_descriptor._internal_create_key),
_descriptor.EnumValueDescriptor( _descriptor.EnumValueDescriptor(
name='BLOCKED', index=3, number=3, name='BLOCKED', index=3, number=3,
options=None, serialized_options=None,
type=None), type=None,
create_key=_descriptor._internal_create_key),
], ],
containing_type=None, containing_type=None,
options=None, serialized_options=None,
serialized_start=817, serialized_start=744,
serialized_end=882, serialized_end=809,
) )
_sym_db.RegisterEnumDescriptor(_ERROR_CODE) _sym_db.RegisterEnumDescriptor(_ERROR_CODE)
@ -62,6 +66,7 @@ _OUTPUTS = _descriptor.Descriptor(
filename=None, filename=None,
file=DESCRIPTOR, file=DESCRIPTOR,
containing_type=None, containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[ fields=[
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='txos', full_name='pb.Outputs.txos', index=0, name='txos', full_name='pb.Outputs.txos', index=0,
@ -69,49 +74,49 @@ _OUTPUTS = _descriptor.Descriptor(
has_default_value=False, default_value=[], has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='extra_txos', full_name='pb.Outputs.extra_txos', index=1, name='extra_txos', full_name='pb.Outputs.extra_txos', index=1,
number=2, type=11, cpp_type=10, label=3, number=2, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[], has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='total', full_name='pb.Outputs.total', index=2, name='total', full_name='pb.Outputs.total', index=2,
number=3, type=13, cpp_type=3, label=1, number=3, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0, has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='offset', full_name='pb.Outputs.offset', index=3, name='offset', full_name='pb.Outputs.offset', index=3,
number=4, type=13, cpp_type=3, label=1, number=4, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0, has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='blocked', full_name='pb.Outputs.blocked', index=4, name='blocked', full_name='pb.Outputs.blocked', index=4,
number=5, type=11, cpp_type=10, label=3, number=5, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[], has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='blocked_total', full_name='pb.Outputs.blocked_total', index=5, name='blocked_total', full_name='pb.Outputs.blocked_total', index=5,
number=6, type=13, cpp_type=3, label=1, number=6, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0, has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
], ],
extensions=[ extensions=[
], ],
nested_types=[], nested_types=[],
enum_types=[ enum_types=[
], ],
options=None, serialized_options=None,
is_extendable=False, is_extendable=False,
syntax='proto3', syntax='proto3',
extension_ranges=[], extension_ranges=[],
@ -128,56 +133,59 @@ _OUTPUT = _descriptor.Descriptor(
filename=None, filename=None,
file=DESCRIPTOR, file=DESCRIPTOR,
containing_type=None, containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[ fields=[
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='tx_hash', full_name='pb.Output.tx_hash', index=0, name='tx_hash', full_name='pb.Output.tx_hash', index=0,
number=1, type=12, cpp_type=9, label=1, number=1, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""), has_default_value=False, default_value=b"",
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='nout', full_name='pb.Output.nout', index=1, name='nout', full_name='pb.Output.nout', index=1,
number=2, type=13, cpp_type=3, label=1, number=2, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0, has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='height', full_name='pb.Output.height', index=2, name='height', full_name='pb.Output.height', index=2,
number=3, type=13, cpp_type=3, label=1, number=3, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0, has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='claim', full_name='pb.Output.claim', index=3, name='claim', full_name='pb.Output.claim', index=3,
number=7, type=11, cpp_type=10, label=1, number=7, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None, has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='error', full_name='pb.Output.error', index=4, name='error', full_name='pb.Output.error', index=4,
number=15, type=11, cpp_type=10, label=1, number=15, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None, has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
], ],
extensions=[ extensions=[
], ],
nested_types=[], nested_types=[],
enum_types=[ enum_types=[
], ],
options=None, serialized_options=None,
is_extendable=False, is_extendable=False,
syntax='proto3', syntax='proto3',
extension_ranges=[], extension_ranges=[],
oneofs=[ oneofs=[
_descriptor.OneofDescriptor( _descriptor.OneofDescriptor(
name='meta', full_name='pb.Output.meta', name='meta', full_name='pb.Output.meta',
index=0, containing_type=None, fields=[]), index=0, containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[]),
], ],
serialized_start=174, serialized_start=174,
serialized_end=297, serialized_end=297,
@ -190,6 +198,7 @@ _CLAIMMETA = _descriptor.Descriptor(
filename=None, filename=None,
file=DESCRIPTOR, file=DESCRIPTOR,
containing_type=None, containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[ fields=[
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='channel', full_name='pb.ClaimMeta.channel', index=0, name='channel', full_name='pb.ClaimMeta.channel', index=0,
@ -197,133 +206,112 @@ _CLAIMMETA = _descriptor.Descriptor(
has_default_value=False, default_value=None, has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='repost', full_name='pb.ClaimMeta.repost', index=1, name='repost', full_name='pb.ClaimMeta.repost', index=1,
number=2, type=11, cpp_type=10, label=1, number=2, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None, has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='short_url', full_name='pb.ClaimMeta.short_url', index=2, name='short_url', full_name='pb.ClaimMeta.short_url', index=2,
number=3, type=9, cpp_type=9, label=1, number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'), has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='canonical_url', full_name='pb.ClaimMeta.canonical_url', index=3, name='canonical_url', full_name='pb.ClaimMeta.canonical_url', index=3,
number=4, type=9, cpp_type=9, label=1, number=4, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'), has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='is_controlling', full_name='pb.ClaimMeta.is_controlling', index=4, name='is_controlling', full_name='pb.ClaimMeta.is_controlling', index=4,
number=5, type=8, cpp_type=7, label=1, number=5, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False, has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='take_over_height', full_name='pb.ClaimMeta.take_over_height', index=5, name='take_over_height', full_name='pb.ClaimMeta.take_over_height', index=5,
number=6, type=13, cpp_type=3, label=1, number=6, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0, has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='creation_height', full_name='pb.ClaimMeta.creation_height', index=6, name='creation_height', full_name='pb.ClaimMeta.creation_height', index=6,
number=7, type=13, cpp_type=3, label=1, number=7, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0, has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='activation_height', full_name='pb.ClaimMeta.activation_height', index=7, name='activation_height', full_name='pb.ClaimMeta.activation_height', index=7,
number=8, type=13, cpp_type=3, label=1, number=8, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0, has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='expiration_height', full_name='pb.ClaimMeta.expiration_height', index=8, name='expiration_height', full_name='pb.ClaimMeta.expiration_height', index=8,
number=9, type=13, cpp_type=3, label=1, number=9, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0, has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='claims_in_channel', full_name='pb.ClaimMeta.claims_in_channel', index=9, name='claims_in_channel', full_name='pb.ClaimMeta.claims_in_channel', index=9,
number=10, type=13, cpp_type=3, label=1, number=10, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0, has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='reposted', full_name='pb.ClaimMeta.reposted', index=10, name='reposted', full_name='pb.ClaimMeta.reposted', index=10,
number=11, type=13, cpp_type=3, label=1, number=11, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0, has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='effective_amount', full_name='pb.ClaimMeta.effective_amount', index=11, name='effective_amount', full_name='pb.ClaimMeta.effective_amount', index=11,
number=20, type=4, cpp_type=4, label=1, number=20, type=4, cpp_type=4, label=1,
has_default_value=False, default_value=0, has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='support_amount', full_name='pb.ClaimMeta.support_amount', index=12, name='support_amount', full_name='pb.ClaimMeta.support_amount', index=12,
number=21, type=4, cpp_type=4, label=1, number=21, type=4, cpp_type=4, label=1,
has_default_value=False, default_value=0, has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='trending_group', full_name='pb.ClaimMeta.trending_group', index=13, name='trending_score', full_name='pb.ClaimMeta.trending_score', index=13,
number=22, type=13, cpp_type=3, label=1, number=22, type=1, cpp_type=5, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='trending_mixed', full_name='pb.ClaimMeta.trending_mixed', index=14,
number=23, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0), has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='trending_local', full_name='pb.ClaimMeta.trending_local', index=15,
number=24, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='trending_global', full_name='pb.ClaimMeta.trending_global', index=16,
number=25, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
], ],
extensions=[ extensions=[
], ],
nested_types=[], nested_types=[],
enum_types=[ enum_types=[
], ],
options=None, serialized_options=None,
is_extendable=False, is_extendable=False,
syntax='proto3', syntax='proto3',
extension_ranges=[], extension_ranges=[],
oneofs=[ oneofs=[
], ],
serialized_start=300, serialized_start=300,
serialized_end=731, serialized_end=658,
) )
@ -333,6 +321,7 @@ _ERROR = _descriptor.Descriptor(
filename=None, filename=None,
file=DESCRIPTOR, file=DESCRIPTOR,
containing_type=None, containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[ fields=[
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='code', full_name='pb.Error.code', index=0, name='code', full_name='pb.Error.code', index=0,
@ -340,21 +329,21 @@ _ERROR = _descriptor.Descriptor(
has_default_value=False, default_value=0, has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='text', full_name='pb.Error.text', index=1, name='text', full_name='pb.Error.text', index=1,
number=2, type=9, cpp_type=9, label=1, number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'), has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='blocked', full_name='pb.Error.blocked', index=2, name='blocked', full_name='pb.Error.blocked', index=2,
number=3, type=11, cpp_type=10, label=1, number=3, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None, has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
], ],
extensions=[ extensions=[
], ],
@ -362,14 +351,14 @@ _ERROR = _descriptor.Descriptor(
enum_types=[ enum_types=[
_ERROR_CODE, _ERROR_CODE,
], ],
options=None, serialized_options=None,
is_extendable=False, is_extendable=False,
syntax='proto3', syntax='proto3',
extension_ranges=[], extension_ranges=[],
oneofs=[ oneofs=[
], ],
serialized_start=734, serialized_start=661,
serialized_end=882, serialized_end=809,
) )
@ -379,6 +368,7 @@ _BLOCKED = _descriptor.Descriptor(
filename=None, filename=None,
file=DESCRIPTOR, file=DESCRIPTOR,
containing_type=None, containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[ fields=[
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='count', full_name='pb.Blocked.count', index=0, name='count', full_name='pb.Blocked.count', index=0,
@ -386,28 +376,28 @@ _BLOCKED = _descriptor.Descriptor(
has_default_value=False, default_value=0, has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor( _descriptor.FieldDescriptor(
name='channel_hash', full_name='pb.Blocked.channel_hash', index=1, name='channel', full_name='pb.Blocked.channel', index=1,
number=2, type=12, cpp_type=9, label=1, number=2, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=_b(""), has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None, message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None, is_extension=False, extension_scope=None,
options=None), serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
], ],
extensions=[ extensions=[
], ],
nested_types=[], nested_types=[],
enum_types=[ enum_types=[
], ],
options=None, serialized_options=None,
is_extendable=False, is_extendable=False,
syntax='proto3', syntax='proto3',
extension_ranges=[], extension_ranges=[],
oneofs=[ oneofs=[
], ],
serialized_start=884, serialized_start=811,
serialized_end=930, serialized_end=864,
) )
_OUTPUTS.fields_by_name['txos'].message_type = _OUTPUT _OUTPUTS.fields_by_name['txos'].message_type = _OUTPUT
@ -426,46 +416,49 @@ _CLAIMMETA.fields_by_name['repost'].message_type = _OUTPUT
_ERROR.fields_by_name['code'].enum_type = _ERROR_CODE _ERROR.fields_by_name['code'].enum_type = _ERROR_CODE
_ERROR.fields_by_name['blocked'].message_type = _BLOCKED _ERROR.fields_by_name['blocked'].message_type = _BLOCKED
_ERROR_CODE.containing_type = _ERROR _ERROR_CODE.containing_type = _ERROR
_BLOCKED.fields_by_name['channel'].message_type = _OUTPUT
DESCRIPTOR.message_types_by_name['Outputs'] = _OUTPUTS DESCRIPTOR.message_types_by_name['Outputs'] = _OUTPUTS
DESCRIPTOR.message_types_by_name['Output'] = _OUTPUT DESCRIPTOR.message_types_by_name['Output'] = _OUTPUT
DESCRIPTOR.message_types_by_name['ClaimMeta'] = _CLAIMMETA DESCRIPTOR.message_types_by_name['ClaimMeta'] = _CLAIMMETA
DESCRIPTOR.message_types_by_name['Error'] = _ERROR DESCRIPTOR.message_types_by_name['Error'] = _ERROR
DESCRIPTOR.message_types_by_name['Blocked'] = _BLOCKED DESCRIPTOR.message_types_by_name['Blocked'] = _BLOCKED
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
Outputs = _reflection.GeneratedProtocolMessageType('Outputs', (_message.Message,), dict( Outputs = _reflection.GeneratedProtocolMessageType('Outputs', (_message.Message,), {
DESCRIPTOR = _OUTPUTS, 'DESCRIPTOR' : _OUTPUTS,
__module__ = 'result_pb2' '__module__' : 'result_pb2'
# @@protoc_insertion_point(class_scope:pb.Outputs) # @@protoc_insertion_point(class_scope:pb.Outputs)
)) })
_sym_db.RegisterMessage(Outputs) _sym_db.RegisterMessage(Outputs)
Output = _reflection.GeneratedProtocolMessageType('Output', (_message.Message,), dict( Output = _reflection.GeneratedProtocolMessageType('Output', (_message.Message,), {
DESCRIPTOR = _OUTPUT, 'DESCRIPTOR' : _OUTPUT,
__module__ = 'result_pb2' '__module__' : 'result_pb2'
# @@protoc_insertion_point(class_scope:pb.Output) # @@protoc_insertion_point(class_scope:pb.Output)
)) })
_sym_db.RegisterMessage(Output) _sym_db.RegisterMessage(Output)
ClaimMeta = _reflection.GeneratedProtocolMessageType('ClaimMeta', (_message.Message,), dict( ClaimMeta = _reflection.GeneratedProtocolMessageType('ClaimMeta', (_message.Message,), {
DESCRIPTOR = _CLAIMMETA, 'DESCRIPTOR' : _CLAIMMETA,
__module__ = 'result_pb2' '__module__' : 'result_pb2'
# @@protoc_insertion_point(class_scope:pb.ClaimMeta) # @@protoc_insertion_point(class_scope:pb.ClaimMeta)
)) })
_sym_db.RegisterMessage(ClaimMeta) _sym_db.RegisterMessage(ClaimMeta)
Error = _reflection.GeneratedProtocolMessageType('Error', (_message.Message,), dict( Error = _reflection.GeneratedProtocolMessageType('Error', (_message.Message,), {
DESCRIPTOR = _ERROR, 'DESCRIPTOR' : _ERROR,
__module__ = 'result_pb2' '__module__' : 'result_pb2'
# @@protoc_insertion_point(class_scope:pb.Error) # @@protoc_insertion_point(class_scope:pb.Error)
)) })
_sym_db.RegisterMessage(Error) _sym_db.RegisterMessage(Error)
Blocked = _reflection.GeneratedProtocolMessageType('Blocked', (_message.Message,), dict( Blocked = _reflection.GeneratedProtocolMessageType('Blocked', (_message.Message,), {
DESCRIPTOR = _BLOCKED, 'DESCRIPTOR' : _BLOCKED,
__module__ = 'result_pb2' '__module__' : 'result_pb2'
# @@protoc_insertion_point(class_scope:pb.Blocked) # @@protoc_insertion_point(class_scope:pb.Blocked)
)) })
_sym_db.RegisterMessage(Blocked) _sym_db.RegisterMessage(Blocked)
DESCRIPTOR._options = None
# @@protoc_insertion_point(module_scope) # @@protoc_insertion_point(module_scope)

View file

@ -0,0 +1,76 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: support.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
from google.protobuf import descriptor_pb2
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='support.proto',
package='pb',
syntax='proto3',
serialized_pb=_b('\n\rsupport.proto\x12\x02pb\")\n\x07Support\x12\r\n\x05\x65moji\x18\x01 \x01(\t\x12\x0f\n\x07\x63omment\x18\x02 \x01(\tb\x06proto3')
)
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
_SUPPORT = _descriptor.Descriptor(
name='Support',
full_name='pb.Support',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='emoji', full_name='pb.Support.emoji', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='comment', full_name='pb.Support.comment', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=21,
serialized_end=62,
)
DESCRIPTOR.message_types_by_name['Support'] = _SUPPORT
Support = _reflection.GeneratedProtocolMessageType('Support', (_message.Message,), dict(
DESCRIPTOR = _SUPPORT,
__module__ = 'support_pb2'
# @@protoc_insertion_point(class_scope:pb.Support)
))
_sym_db.RegisterMessage(Support)
# @@protoc_insertion_point(module_scope)

View file

@ -0,0 +1,139 @@
{
"title": "Wallet",
"description": "An LBC wallet",
"type": "object",
"required": ["name", "version", "accounts", "preferences"],
"additionalProperties": false,
"properties": {
"name": {
"description": "Human readable name for this wallet",
"type": "string"
},
"version": {
"description": "Wallet spec version",
"type": "integer",
"$comment": "Should this be a string? We may need some sort of decimal type if we want exact decimal versions."
},
"accounts": {
"description": "Accounts associated with this wallet",
"type": "array",
"items": {
"type": "object",
"required": ["address_generator", "certificates", "encrypted", "ledger", "modified_on", "name", "private_key", "public_key", "seed"],
"additionalProperties": false,
"properties": {
"address_generator": {
"description": "Higher level manager of either singular or deterministically generated addresses",
"type": "object",
"oneOf": [
{
"required": ["name", "change", "receiving"],
"additionalProperties": false,
"properties": {
"name": {
"description": "type of address generator: a deterministic chain of addresses",
"enum": ["deterministic-chain"],
"type": "string"
},
"change": {
"$ref": "#/$defs/address_manager",
"description": "Manager for deterministically generated change address (not used for single address)"
},
"receiving": {
"$ref": "#/$defs/address_manager",
"description": "Manager for deterministically generated receiving address (not used for single address)"
}
}
}, {
"required": ["name"],
"additionalProperties": false,
"properties": {
"name": {
"description": "type of address generator: a single address",
"enum": ["single-address"],
"type": "string"
}
}
}
]
},
"certificates": {
"type": "object",
"description": "Channel keys. Mapping from public key address to pem-formatted private key.",
"additionalProperties": {"type": "string"}
},
"encrypted": {
"type": "boolean",
"description": "Whether private key and seed are encrypted with a password"
},
"ledger": {
"description": "Which network to use",
"type": "string",
"examples": [
"lbc_mainnet",
"lbc_testnet"
]
},
"modified_on": {
"description": "last modified time in Unix Time",
"type": "integer"
},
"name": {
"description": "Name for account, possibly human readable",
"type": "string"
},
"private_key": {
"description": "Private key for address if `address_generator` is a single address. Root of chain of private keys for addresses if `address_generator` is a deterministic chain of addresses. Encrypted if `encrypted` is true.",
"type": "string"
},
"public_key": {
"description": "Public key for address if `address_generator` is a single address. Root of chain of public keys for addresses if `address_generator` is a deterministic chain of addresses.",
"type": "string"
},
"seed": {
"description": "Human readable representation of `private_key`. encrypted if `encrypted` is set to `true`",
"type": "string"
}
}
}
},
"preferences": {
"description": "Timestamped application-level preferences. Values can be objects or of a primitive type.",
"$comment": "enable-sync is seen in example wallet. encrypt-on-disk is seen in example wallet. they both have a boolean `value` field. Do we want them explicitly defined here? local and shared seem to have at least a similar structure (type, value [yes, again], version), value being the free-form part. Should we define those here? Or can there be any key under preferences, and `value` be literally be anything in any form?",
"type": "object",
"additionalProperties": {
"type": "object",
"required": ["ts", "value"],
"additionalProperties": false,
"properties": {
"ts": {
"type": "number",
"description": "When the item was set, in Unix time format.",
"$comment": "Do we want a string (decimal)?"
},
"value": {
"$comment": "Sometimes this has been an object, sometimes just a boolean. I don't want to prescribe anything."
}
}
}
}
},
"$defs": {
"address_manager": {
"description": "Manager for deterministically generated addresses",
"type": "object",
"required": ["gap", "maximum_uses_per_address"],
"additionalProperties": false,
"properties": {
"gap": {
"description": "Maximum allowed consecutive generated addresses with no transactions",
"type": "integer"
},
"maximum_uses_per_address": {
"description": "Maximum number of uses for each generated address",
"type": "integer"
}
}
}
}
}

View file

@ -7,7 +7,7 @@ def _create_url_regex():
# see https://spec.lbry.com/ and test_url.py # see https://spec.lbry.com/ and test_url.py
invalid_names_regex = \ invalid_names_regex = \
r"[^=&#:$@%?;\"/\\<>%{}|^~`\[\]" \ r"[^=&#:$@%?;\"/\\<>%{}|^~`\[\]" \
r"\u0000-\u0008\u000b-\u000c\u000e-\u001F\uD800-\uDFFF\uFFFE-\uFFFF]+" r"\u0000-\u0020\uD800-\uDFFF\uFFFE-\uFFFF]+"
def _named(name, regex): def _named(name, regex):
return "(?P<" + name + ">" + regex + ")" return "(?P<" + name + ">" + regex + ")"
@ -22,8 +22,7 @@ def _create_url_regex():
return _group( return _group(
_named(name+"_name", prefix + invalid_names_regex) + _named(name+"_name", prefix + invalid_names_regex) +
_oneof( _oneof(
_group('#' + _named(name+"_claim_id", "[0-9a-f]{1,40}")), _group('[:#]' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
_group(':' + _named(name+"_sequence", '[1-9][0-9]*')),
_group(r'\$' + _named(name+"_amount_order", '[1-9][0-9]*')) _group(r'\$' + _named(name+"_amount_order", '[1-9][0-9]*'))
) + '?' ) + '?'
) )
@ -50,28 +49,31 @@ def normalize_name(name):
class PathSegment(NamedTuple): class PathSegment(NamedTuple):
name: str name: str
claim_id: str = None claim_id: str = None
sequence: int = None
amount_order: int = None amount_order: int = None
@property @property
def normalized(self): def normalized(self):
return normalize_name(self.name) return normalize_name(self.name)
@property
def is_shortid(self):
return self.claim_id is not None and len(self.claim_id) < 40
@property
def is_fullid(self):
return self.claim_id is not None and len(self.claim_id) == 40
def to_dict(self): def to_dict(self):
q = {'name': self.name} q = {'name': self.name}
if self.claim_id is not None: if self.claim_id is not None:
q['claim_id'] = self.claim_id q['claim_id'] = self.claim_id
if self.sequence is not None:
q['sequence'] = self.sequence
if self.amount_order is not None: if self.amount_order is not None:
q['amount_order'] = self.amount_order q['amount_order'] = self.amount_order
return q return q
def __str__(self): def __str__(self):
if self.claim_id is not None: if self.claim_id is not None:
return f"{self.name}#{self.claim_id}" return f"{self.name}:{self.claim_id}"
elif self.sequence is not None:
return f"{self.name}:{self.sequence}"
elif self.amount_order is not None: elif self.amount_order is not None:
return f"{self.name}${self.amount_order}" return f"{self.name}${self.amount_order}"
return self.name return self.name
@ -118,7 +120,6 @@ class URL(NamedTuple):
segments[segment] = PathSegment( segments[segment] = PathSegment(
parts[f'{segment}_name'], parts[f'{segment}_name'],
parts[f'{segment}_claim_id'], parts[f'{segment}_claim_id'],
parts[f'{segment}_sequence'],
parts[f'{segment}_amount_order'] parts[f'{segment}_amount_order']
) )

View file

@ -0,0 +1,31 @@
import asyncio
import logging
from lbry.stream.downloader import StreamDownloader
log = logging.getLogger(__name__)
class BackgroundDownloader:
def __init__(self, conf, storage, blob_manager, dht_node=None):
self.storage = storage
self.blob_manager = blob_manager
self.node = dht_node
self.conf = conf
async def download_blobs(self, sd_hash):
downloader = StreamDownloader(asyncio.get_running_loop(), self.conf, self.blob_manager, sd_hash)
try:
await downloader.start(self.node, save_stream=False)
for blob_info in downloader.descriptor.blobs[:-1]:
await downloader.download_stream_blob(blob_info)
except ValueError:
return
except asyncio.CancelledError:
log.debug("Cancelled background downloader")
raise
except Exception:
log.error("Unexpected download error on background downloader")
finally:
downloader.stop()

View file

@ -4,6 +4,7 @@ import binascii
import logging import logging
import typing import typing
import asyncio import asyncio
import time
import re import re
from collections import OrderedDict from collections import OrderedDict
from cryptography.hazmat.primitives.ciphers.algorithms import AES from cryptography.hazmat.primitives.ciphers.algorithms import AES
@ -44,16 +45,23 @@ def random_iv_generator() -> typing.Generator[bytes, None, None]:
yield os.urandom(AES.block_size // 8) yield os.urandom(AES.block_size // 8)
def file_reader(file_path: str): def read_bytes(file_path: str, offset: int, to_read: int):
with open(file_path, 'rb') as f:
f.seek(offset)
return f.read(to_read)
async def file_reader(file_path: str):
length = int(os.stat(file_path).st_size) length = int(os.stat(file_path).st_size)
offset = 0 offset = 0
with open(file_path, 'rb') as stream_file:
while offset < length: while offset < length:
bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1) bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1)
if not bytes_to_read: if not bytes_to_read:
break break
blob_bytes = stream_file.read(bytes_to_read) blob_bytes = await asyncio.get_event_loop().run_in_executor(
None, read_bytes, file_path, offset, bytes_to_read
)
yield blob_bytes yield blob_bytes
offset += bytes_to_read offset += bytes_to_read
@ -145,15 +153,19 @@ class StreamDescriptor:
h.update(self.old_sort_json()) h.update(self.old_sort_json())
return h.hexdigest() return h.hexdigest()
async def make_sd_blob(self, blob_file_obj: typing.Optional[AbstractBlob] = None, async def make_sd_blob(
old_sort: typing.Optional[bool] = False, self, blob_file_obj: typing.Optional[AbstractBlob] = None, old_sort: typing.Optional[bool] = False,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None): blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None,
added_on: float = None, is_mine: bool = False
):
sd_hash = self.calculate_sd_hash() if not old_sort else self.calculate_old_sort_sd_hash() sd_hash = self.calculate_sd_hash() if not old_sort else self.calculate_old_sort_sd_hash()
if not old_sort: if not old_sort:
sd_data = self.as_json() sd_data = self.as_json()
else: else:
sd_data = self.old_sort_json() sd_data = self.old_sort_json()
sd_blob = blob_file_obj or BlobFile(self.loop, sd_hash, len(sd_data), blob_completed_callback, self.blob_dir) sd_blob = blob_file_obj or BlobFile(
self.loop, sd_hash, len(sd_data), blob_completed_callback, self.blob_dir, added_on, is_mine
)
if blob_file_obj: if blob_file_obj:
blob_file_obj.set_length(len(sd_data)) blob_file_obj.set_length(len(sd_data))
if not sd_blob.get_is_verified(): if not sd_blob.get_is_verified():
@ -176,18 +188,19 @@ class StreamDescriptor:
raise InvalidStreamDescriptorError("Does not decode as valid JSON") raise InvalidStreamDescriptorError("Does not decode as valid JSON")
if decoded['blobs'][-1]['length'] != 0: if decoded['blobs'][-1]['length'] != 0:
raise InvalidStreamDescriptorError("Does not end with a zero-length blob.") raise InvalidStreamDescriptorError("Does not end with a zero-length blob.")
if any([blob_info['length'] == 0 for blob_info in decoded['blobs'][:-1]]): if any(blob_info['length'] == 0 for blob_info in decoded['blobs'][:-1]):
raise InvalidStreamDescriptorError("Contains zero-length data blob") raise InvalidStreamDescriptorError("Contains zero-length data blob")
if 'blob_hash' in decoded['blobs'][-1]: if 'blob_hash' in decoded['blobs'][-1]:
raise InvalidStreamDescriptorError("Stream terminator blob should not have a hash") raise InvalidStreamDescriptorError("Stream terminator blob should not have a hash")
if any([i != blob_info['blob_num'] for i, blob_info in enumerate(decoded['blobs'])]): if any(i != blob_info['blob_num'] for i, blob_info in enumerate(decoded['blobs'])):
raise InvalidStreamDescriptorError("Stream contains out of order or skipped blobs") raise InvalidStreamDescriptorError("Stream contains out of order or skipped blobs")
added_on = time.time()
descriptor = cls( descriptor = cls(
loop, blob_dir, loop, blob_dir,
binascii.unhexlify(decoded['stream_name']).decode(), binascii.unhexlify(decoded['stream_name']).decode(),
decoded['key'], decoded['key'],
binascii.unhexlify(decoded['suggested_file_name']).decode(), binascii.unhexlify(decoded['suggested_file_name']).decode(),
[BlobInfo(info['blob_num'], info['length'], info['iv'], info.get('blob_hash')) [BlobInfo(info['blob_num'], info['length'], info['iv'], added_on, info.get('blob_hash'))
for info in decoded['blobs']], for info in decoded['blobs']],
decoded['stream_hash'], decoded['stream_hash'],
blob.blob_hash blob.blob_hash
@ -245,20 +258,25 @@ class StreamDescriptor:
iv_generator = iv_generator or random_iv_generator() iv_generator = iv_generator or random_iv_generator()
key = key or os.urandom(AES.block_size // 8) key = key or os.urandom(AES.block_size // 8)
blob_num = -1 blob_num = -1
for blob_bytes in file_reader(file_path): added_on = time.time()
async for blob_bytes in file_reader(file_path):
blob_num += 1 blob_num += 1
blob_info = await BlobFile.create_from_unencrypted( blob_info = await BlobFile.create_from_unencrypted(
loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num, blob_completed_callback loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num, added_on, True, blob_completed_callback
) )
blobs.append(blob_info) blobs.append(blob_info)
blobs.append( blobs.append(
BlobInfo(len(blobs), 0, binascii.hexlify(next(iv_generator)).decode())) # add the stream terminator # add the stream terminator
BlobInfo(len(blobs), 0, binascii.hexlify(next(iv_generator)).decode(), added_on, None, True)
)
file_name = os.path.basename(file_path) file_name = os.path.basename(file_path)
suggested_file_name = sanitize_file_name(file_name) suggested_file_name = sanitize_file_name(file_name)
descriptor = cls( descriptor = cls(
loop, blob_dir, file_name, binascii.hexlify(key).decode(), suggested_file_name, blobs loop, blob_dir, file_name, binascii.hexlify(key).decode(), suggested_file_name, blobs
) )
sd_blob = await descriptor.make_sd_blob(old_sort=old_sort, blob_completed_callback=blob_completed_callback) sd_blob = await descriptor.make_sd_blob(
old_sort=old_sort, blob_completed_callback=blob_completed_callback, added_on=added_on, is_mine=True
)
descriptor.sd_hash = sd_blob.blob_hash descriptor.sd_hash = sd_blob.blob_hash
return descriptor return descriptor

View file

@ -3,11 +3,13 @@ import typing
import logging import logging
import binascii import binascii
from lbry.dht.peer import make_kademlia_peer from lbry.dht.node import get_kademlia_peers_from_hosts
from lbry.error import DownloadSDTimeoutError from lbry.error import DownloadSDTimeoutError
from lbry.utils import resolve_host, lru_cache_concurrent from lbry.utils import lru_cache_concurrent
from lbry.stream.descriptor import StreamDescriptor from lbry.stream.descriptor import StreamDescriptor
from lbry.blob_exchange.downloader import BlobDownloader from lbry.blob_exchange.downloader import BlobDownloader
from lbry.torrent.tracker import enqueue_tracker_search
if typing.TYPE_CHECKING: if typing.TYPE_CHECKING:
from lbry.conf import Config from lbry.conf import Config
from lbry.dht.node import Node from lbry.dht.node import Node
@ -25,8 +27,8 @@ class StreamDownloader:
self.config = config self.config = config
self.blob_manager = blob_manager self.blob_manager = blob_manager
self.sd_hash = sd_hash self.sd_hash = sd_hash
self.search_queue = asyncio.Queue(loop=loop) # blob hashes to feed into the iterative finder self.search_queue = asyncio.Queue() # blob hashes to feed into the iterative finder
self.peer_queue = asyncio.Queue(loop=loop) # new peers to try self.peer_queue = asyncio.Queue() # new peers to try
self.blob_downloader = BlobDownloader(self.loop, self.config, self.blob_manager, self.peer_queue) self.blob_downloader = BlobDownloader(self.loop, self.config, self.blob_manager, self.peer_queue)
self.descriptor: typing.Optional[StreamDescriptor] = descriptor self.descriptor: typing.Optional[StreamDescriptor] = descriptor
self.node: typing.Optional['Node'] = None self.node: typing.Optional['Node'] = None
@ -40,7 +42,7 @@ class StreamDownloader:
async def cached_read_blob(blob_info: 'BlobInfo') -> bytes: async def cached_read_blob(blob_info: 'BlobInfo') -> bytes:
return await self.read_blob(blob_info, 2) return await self.read_blob(blob_info, 2)
if self.blob_manager.decrypted_blob_lru_cache: if self.blob_manager.decrypted_blob_lru_cache is not None:
cached_read_blob = lru_cache_concurrent(override_lru_cache=self.blob_manager.decrypted_blob_lru_cache)( cached_read_blob = lru_cache_concurrent(override_lru_cache=self.blob_manager.decrypted_blob_lru_cache)(
cached_read_blob cached_read_blob
) )
@ -48,26 +50,19 @@ class StreamDownloader:
self.cached_read_blob = cached_read_blob self.cached_read_blob = cached_read_blob
async def add_fixed_peers(self): async def add_fixed_peers(self):
def _delayed_add_fixed_peers(): def _add_fixed_peers(fixed_peers):
self.peer_queue.put_nowait(fixed_peers)
self.added_fixed_peers = True self.added_fixed_peers = True
self.peer_queue.put_nowait([
make_kademlia_peer(None, address, None, tcp_port=port + 1, allow_localhost=True)
for address, port in addresses
])
if not self.config.reflector_servers: if not self.config.fixed_peers:
return return
addresses = [
(await resolve_host(url, port + 1, proto='tcp'), port)
for url, port in self.config.reflector_servers
]
if 'dht' in self.config.components_to_skip or not self.node or not \ if 'dht' in self.config.components_to_skip or not self.node or not \
len(self.node.protocol.routing_table.get_peers()) > 0: len(self.node.protocol.routing_table.get_peers()) > 0:
self.fixed_peers_delay = 0.0 self.fixed_peers_delay = 0.0
else: else:
self.fixed_peers_delay = self.config.fixed_peer_delay self.fixed_peers_delay = self.config.fixed_peer_delay
fixed_peers = await get_kademlia_peers_from_hosts(self.config.fixed_peers)
self.fixed_peers_handle = self.loop.call_later(self.fixed_peers_delay, _delayed_add_fixed_peers) self.fixed_peers_handle = self.loop.call_later(self.fixed_peers_delay, _add_fixed_peers, fixed_peers)
async def load_descriptor(self, connection_id: int = 0): async def load_descriptor(self, connection_id: int = 0):
# download or get the sd blob # download or get the sd blob
@ -77,7 +72,7 @@ class StreamDownloader:
now = self.loop.time() now = self.loop.time()
sd_blob = await asyncio.wait_for( sd_blob = await asyncio.wait_for(
self.blob_downloader.download_blob(self.sd_hash, connection_id), self.blob_downloader.download_blob(self.sd_hash, connection_id),
self.config.blob_download_timeout, loop=self.loop self.config.blob_download_timeout
) )
log.info("downloaded sd blob %s", self.sd_hash) log.info("downloaded sd blob %s", self.sd_hash)
self.time_to_descriptor = self.loop.time() - now self.time_to_descriptor = self.loop.time() - now
@ -90,14 +85,15 @@ class StreamDownloader:
) )
log.info("loaded stream manifest %s", self.sd_hash) log.info("loaded stream manifest %s", self.sd_hash)
async def start(self, node: typing.Optional['Node'] = None, connection_id: int = 0): async def start(self, node: typing.Optional['Node'] = None, connection_id: int = 0, save_stream=True):
# set up peer accumulation # set up peer accumulation
if node: self.node = node or self.node # fixme: this shouldnt be set here!
self.node = node if self.node:
if self.accumulate_task and not self.accumulate_task.done(): if self.accumulate_task and not self.accumulate_task.done():
self.accumulate_task.cancel() self.accumulate_task.cancel()
_, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue) _, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue)
await self.add_fixed_peers() await self.add_fixed_peers()
enqueue_tracker_search(bytes.fromhex(self.sd_hash), self.peer_queue)
# start searching for peers for the sd hash # start searching for peers for the sd hash
self.search_queue.put_nowait(self.sd_hash) self.search_queue.put_nowait(self.sd_hash)
log.info("searching for peers for stream %s", self.sd_hash) log.info("searching for peers for stream %s", self.sd_hash)
@ -105,11 +101,7 @@ class StreamDownloader:
if not self.descriptor: if not self.descriptor:
await self.load_descriptor(connection_id) await self.load_descriptor(connection_id)
# add the head blob to the peer search if not await self.blob_manager.storage.stream_exists(self.sd_hash) and save_stream:
self.search_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
log.info("added head blob to peer search for stream %s", self.sd_hash)
if not await self.blob_manager.storage.stream_exists(self.sd_hash):
await self.blob_manager.storage.store_stream( await self.blob_manager.storage.store_stream(
self.blob_manager.get_blob(self.sd_hash, length=self.descriptor.length), self.descriptor self.blob_manager.get_blob(self.sd_hash, length=self.descriptor.length), self.descriptor
) )
@ -119,7 +111,7 @@ class StreamDownloader:
raise ValueError(f"blob {blob_info.blob_hash} is not part of stream with sd hash {self.sd_hash}") raise ValueError(f"blob {blob_info.blob_hash} is not part of stream with sd hash {self.sd_hash}")
blob = await asyncio.wait_for( blob = await asyncio.wait_for(
self.blob_downloader.download_blob(blob_info.blob_hash, blob_info.length, connection_id), self.blob_downloader.download_blob(blob_info.blob_hash, blob_info.length, connection_id),
self.config.blob_download_timeout * 10, loop=self.loop self.config.blob_download_timeout * 10
) )
return blob return blob

View file

@ -3,9 +3,8 @@ import asyncio
import time import time
import typing import typing
import logging import logging
import binascii from typing import Optional
from aiohttp.web import Request, StreamResponse, HTTPRequestRangeNotSatisfiable from aiohttp.web import Request, StreamResponse, HTTPRequestRangeNotSatisfiable
from lbry.utils import generate_id
from lbry.error import DownloadSDTimeoutError from lbry.error import DownloadSDTimeoutError
from lbry.schema.mime_types import guess_media_type from lbry.schema.mime_types import guess_media_type
from lbry.stream.downloader import StreamDownloader from lbry.stream.downloader import StreamDownloader
@ -13,13 +12,12 @@ from lbry.stream.descriptor import StreamDescriptor, sanitize_file_name
from lbry.stream.reflector.client import StreamReflectorClient from lbry.stream.reflector.client import StreamReflectorClient
from lbry.extras.daemon.storage import StoredContentClaim from lbry.extras.daemon.storage import StoredContentClaim
from lbry.blob import MAX_BLOB_SIZE from lbry.blob import MAX_BLOB_SIZE
from lbry.file.source import ManagedDownloadSource
if typing.TYPE_CHECKING: if typing.TYPE_CHECKING:
from lbry.conf import Config from lbry.conf import Config
from lbry.schema.claim import Claim
from lbry.blob.blob_manager import BlobManager from lbry.blob.blob_manager import BlobManager
from lbry.blob.blob_info import BlobInfo from lbry.blob.blob_info import BlobInfo
from lbry.dht.node import Node
from lbry.extras.daemon.analytics import AnalyticsManager from lbry.extras.daemon.analytics import AnalyticsManager
from lbry.wallet.transaction import Transaction from lbry.wallet.transaction import Transaction
@ -40,76 +38,39 @@ async def get_next_available_file_name(loop: asyncio.AbstractEventLoop, download
return await loop.run_in_executor(None, _get_next_available_file_name, download_directory, file_name) return await loop.run_in_executor(None, _get_next_available_file_name, download_directory, file_name)
class ManagedStream: class ManagedStream(ManagedDownloadSource):
STATUS_RUNNING = "running"
STATUS_STOPPED = "stopped"
STATUS_FINISHED = "finished"
SAVING_ID = 1
STREAMING_ID = 2
__slots__ = [
'loop',
'config',
'blob_manager',
'sd_hash',
'download_directory',
'_file_name',
'_added_on',
'_status',
'stream_claim_info',
'download_id',
'rowid',
'content_fee',
'purchase_receipt',
'downloader',
'analytics_manager',
'fully_reflected',
'file_output_task',
'delayed_stop_task',
'streaming_responses',
'streaming',
'_running',
'saving',
'finished_writing',
'started_writing',
'finished_write_attempt'
]
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager', def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
sd_hash: str, download_directory: typing.Optional[str] = None, file_name: typing.Optional[str] = None, sd_hash: str, download_directory: Optional[str] = None, file_name: Optional[str] = None,
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional[StoredContentClaim] = None, status: Optional[str] = ManagedDownloadSource.STATUS_STOPPED,
download_id: typing.Optional[str] = None, rowid: typing.Optional[int] = None, claim: Optional[StoredContentClaim] = None,
descriptor: typing.Optional[StreamDescriptor] = None, download_id: Optional[str] = None, rowid: Optional[int] = None,
content_fee: typing.Optional['Transaction'] = None, descriptor: Optional[StreamDescriptor] = None,
analytics_manager: typing.Optional['AnalyticsManager'] = None, content_fee: Optional['Transaction'] = None,
added_on: typing.Optional[int] = None): analytics_manager: Optional['AnalyticsManager'] = None,
self.loop = loop added_on: Optional[int] = None):
self.config = config super().__init__(loop, config, blob_manager.storage, sd_hash, file_name, download_directory, status, claim,
download_id, rowid, content_fee, analytics_manager, added_on)
self.blob_manager = blob_manager self.blob_manager = blob_manager
self.sd_hash = sd_hash
self.download_directory = download_directory
self._file_name = file_name
self._status = status
self.stream_claim_info = claim
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
self.rowid = rowid
self.content_fee = content_fee
self.purchase_receipt = None self.purchase_receipt = None
self._added_on = added_on
self.downloader = StreamDownloader(self.loop, self.config, self.blob_manager, sd_hash, descriptor) self.downloader = StreamDownloader(self.loop, self.config, self.blob_manager, sd_hash, descriptor)
self.analytics_manager = analytics_manager self.analytics_manager = analytics_manager
self.fully_reflected = asyncio.Event(loop=self.loop) self.reflector_progress = 0
self.uploading_to_reflector = False
self.file_output_task: typing.Optional[asyncio.Task] = None self.file_output_task: typing.Optional[asyncio.Task] = None
self.delayed_stop_task: typing.Optional[asyncio.Task] = None self.delayed_stop_task: typing.Optional[asyncio.Task] = None
self.streaming_responses: typing.List[typing.Tuple[Request, StreamResponse]] = [] self.streaming_responses: typing.List[typing.Tuple[Request, StreamResponse]] = []
self.streaming = asyncio.Event(loop=self.loop) self.fully_reflected = asyncio.Event()
self._running = asyncio.Event(loop=self.loop) self.streaming = asyncio.Event()
self.saving = asyncio.Event(loop=self.loop) self._running = asyncio.Event()
self.finished_writing = asyncio.Event(loop=self.loop)
self.started_writing = asyncio.Event(loop=self.loop) @property
self.finished_write_attempt = asyncio.Event(loop=self.loop) def sd_hash(self) -> str:
return self.identifier
@property
def is_fully_reflected(self) -> bool:
return self.fully_reflected.is_set()
@property @property
def descriptor(self) -> StreamDescriptor: def descriptor(self) -> StreamDescriptor:
@ -120,16 +81,20 @@ class ManagedStream:
return self.descriptor.stream_hash return self.descriptor.stream_hash
@property @property
def file_name(self) -> typing.Optional[str]: def file_name(self) -> Optional[str]:
return self._file_name or (self.descriptor.suggested_file_name if self.descriptor else None) return self._file_name or self.suggested_file_name
@property @property
def added_on(self) -> typing.Optional[int]: def suggested_file_name(self) -> Optional[str]:
return self._added_on first_option = ((self.descriptor and self.descriptor.suggested_file_name) or '').strip()
return sanitize_file_name(first_option or (self.stream_claim_info and self.stream_claim_info.claim and
self.stream_claim_info.claim.stream.source.name))
@property @property
def status(self) -> str: def stream_name(self) -> Optional[str]:
return self._status first_option = ((self.descriptor and self.descriptor.stream_name) or '').strip()
return first_option or (self.stream_claim_info and self.stream_claim_info.claim and
self.stream_claim_info.claim.stream.source.name)
@property @property
def written_bytes(self) -> int: def written_bytes(self) -> int:
@ -148,55 +113,6 @@ class ManagedStream:
self._status = status self._status = status
await self.blob_manager.storage.change_file_status(self.stream_hash, status) await self.blob_manager.storage.change_file_status(self.stream_hash, status)
@property
def finished(self) -> bool:
return self.status == self.STATUS_FINISHED
@property
def running(self) -> bool:
return self.status == self.STATUS_RUNNING
@property
def claim_id(self) -> typing.Optional[str]:
return None if not self.stream_claim_info else self.stream_claim_info.claim_id
@property
def txid(self) -> typing.Optional[str]:
return None if not self.stream_claim_info else self.stream_claim_info.txid
@property
def nout(self) -> typing.Optional[int]:
return None if not self.stream_claim_info else self.stream_claim_info.nout
@property
def outpoint(self) -> typing.Optional[str]:
return None if not self.stream_claim_info else self.stream_claim_info.outpoint
@property
def claim_height(self) -> typing.Optional[int]:
return None if not self.stream_claim_info else self.stream_claim_info.height
@property
def channel_claim_id(self) -> typing.Optional[str]:
return None if not self.stream_claim_info else self.stream_claim_info.channel_claim_id
@property
def channel_name(self) -> typing.Optional[str]:
return None if not self.stream_claim_info else self.stream_claim_info.channel_name
@property
def claim_name(self) -> typing.Optional[str]:
return None if not self.stream_claim_info else self.stream_claim_info.claim_name
@property
def metadata(self) -> typing.Optional[typing.Dict]:
return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict()
@property
def metadata_protobuf(self) -> bytes:
if self.stream_claim_info:
return binascii.hexlify(self.stream_claim_info.claim.to_bytes())
@property @property
def blobs_completed(self) -> int: def blobs_completed(self) -> int:
return sum([1 if b.blob_hash in self.blob_manager.completed_blob_hashes else 0 return sum([1 if b.blob_hash in self.blob_manager.completed_blob_hashes else 0
@ -210,39 +126,34 @@ class ManagedStream:
def blobs_remaining(self) -> int: def blobs_remaining(self) -> int:
return self.blobs_in_stream - self.blobs_completed return self.blobs_in_stream - self.blobs_completed
@property
def full_path(self) -> typing.Optional[str]:
return os.path.join(self.download_directory, os.path.basename(self.file_name)) \
if self.file_name and self.download_directory else None
@property
def output_file_exists(self):
return os.path.isfile(self.full_path) if self.full_path else False
@property @property
def mime_type(self): def mime_type(self):
return guess_media_type(os.path.basename(self.descriptor.suggested_file_name))[0] return guess_media_type(os.path.basename(self.suggested_file_name))[0]
@classmethod @property
async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager', def download_path(self):
file_path: str, key: typing.Optional[bytes] = None, return f"{self.download_directory}/{self._file_name}" if self.download_directory and self._file_name else None
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedStream':
"""
Generate a stream from a file and save it to the db
"""
descriptor = await StreamDescriptor.create_stream(
loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
blob_completed_callback=blob_manager.blob_completed
)
await blob_manager.storage.store_stream(
blob_manager.get_blob(descriptor.sd_hash), descriptor
)
row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path),
os.path.dirname(file_path), 0)
return cls(loop, config, blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
os.path.basename(file_path), status=cls.STATUS_FINISHED, rowid=row_id, descriptor=descriptor)
async def start(self, node: typing.Optional['Node'] = None, timeout: typing.Optional[float] = None, # @classmethod
# async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config',
# file_path: str, key: Optional[bytes] = None,
# iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedDownloadSource':
# """
# Generate a stream from a file and save it to the db
# """
# descriptor = await StreamDescriptor.create_stream(
# loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
# blob_completed_callback=blob_manager.blob_completed
# )
# await blob_manager.storage.store_stream(
# blob_manager.get_blob(descriptor.sd_hash), descriptor
# )
# row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path),
# os.path.dirname(file_path), 0)
# return cls(loop, config, blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
# os.path.basename(file_path), status=cls.STATUS_FINISHED, rowid=row_id, descriptor=descriptor)
async def start(self, timeout: Optional[float] = None,
save_now: bool = False): save_now: bool = False):
timeout = timeout or self.config.download_timeout timeout = timeout or self.config.download_timeout
if self._running.is_set(): if self._running.is_set():
@ -250,7 +161,7 @@ class ManagedStream:
log.info("start downloader for stream (sd hash: %s)", self.sd_hash) log.info("start downloader for stream (sd hash: %s)", self.sd_hash)
self._running.set() self._running.set()
try: try:
await asyncio.wait_for(self.downloader.start(node), timeout, loop=self.loop) await asyncio.wait_for(self.downloader.start(), timeout)
except asyncio.TimeoutError: except asyncio.TimeoutError:
self._running.clear() self._running.clear()
raise DownloadSDTimeoutError(self.sd_hash) raise DownloadSDTimeoutError(self.sd_hash)
@ -260,6 +171,11 @@ class ManagedStream:
self.delayed_stop_task = self.loop.create_task(self._delayed_stop()) self.delayed_stop_task = self.loop.create_task(self._delayed_stop())
if not await self.blob_manager.storage.file_exists(self.sd_hash): if not await self.blob_manager.storage.file_exists(self.sd_hash):
if save_now: if save_now:
if not self._file_name:
self._file_name = await get_next_available_file_name(
self.loop, self.download_directory,
self._file_name or sanitize_file_name(self.suggested_file_name)
)
file_name, download_dir = self._file_name, self.download_directory file_name, download_dir = self._file_name, self.download_directory
else: else:
file_name, download_dir = None, None file_name, download_dir = None, None
@ -275,11 +191,11 @@ class ManagedStream:
Stop any running save/stream tasks as well as the downloader and update the status in the database Stop any running save/stream tasks as well as the downloader and update the status in the database
""" """
self.stop_tasks() await self.stop_tasks()
if (finished and self.status != self.STATUS_FINISHED) or self.status == self.STATUS_RUNNING: if (finished and self.status != self.STATUS_FINISHED) or self.status == self.STATUS_RUNNING:
await self.update_status(self.STATUS_FINISHED if finished else self.STATUS_STOPPED) await self.update_status(self.STATUS_FINISHED if finished else self.STATUS_STOPPED)
async def _aiter_read_stream(self, start_blob_num: typing.Optional[int] = 0, connection_id: int = 0)\ async def _aiter_read_stream(self, start_blob_num: Optional[int] = 0, connection_id: int = 0)\
-> typing.AsyncIterator[typing.Tuple['BlobInfo', bytes]]: -> typing.AsyncIterator[typing.Tuple['BlobInfo', bytes]]:
if start_blob_num >= len(self.descriptor.blobs[:-1]): if start_blob_num >= len(self.descriptor.blobs[:-1]):
raise IndexError(start_blob_num) raise IndexError(start_blob_num)
@ -291,13 +207,13 @@ class ManagedStream:
decrypted = await self.downloader.read_blob(blob_info, connection_id) decrypted = await self.downloader.read_blob(blob_info, connection_id)
yield (blob_info, decrypted) yield (blob_info, decrypted)
async def stream_file(self, request: Request, node: typing.Optional['Node'] = None) -> StreamResponse: async def stream_file(self, request: Request) -> StreamResponse:
log.info("stream file to browser for lbry://%s#%s (sd hash %s...)", self.claim_name, self.claim_id, log.info("stream file to browser for lbry://%s#%s (sd hash %s...)", self.claim_name, self.claim_id,
self.sd_hash[:6]) self.sd_hash[:6])
headers, size, skip_blobs, first_blob_start_offset = self._prepare_range_response_headers( headers, size, skip_blobs, first_blob_start_offset = self._prepare_range_response_headers(
request.headers.get('range', 'bytes=0-') request.headers.get('range', 'bytes=0-')
) )
await self.start(node) await self.start()
response = StreamResponse( response = StreamResponse(
status=206, status=206,
headers=headers headers=headers
@ -335,7 +251,8 @@ class ManagedStream:
self.streaming.clear() self.streaming.clear()
@staticmethod @staticmethod
def _write_decrypted_blob(handle: typing.IO, data: bytes): def _write_decrypted_blob(output_path: str, data: bytes):
with open(output_path, 'ab') as handle:
handle.write(data) handle.write(data)
handle.flush() handle.flush()
@ -347,10 +264,10 @@ class ManagedStream:
self.finished_writing.clear() self.finished_writing.clear()
self.started_writing.clear() self.started_writing.clear()
try: try:
with open(output_path, 'wb') as file_write_handle: open(output_path, 'wb').close() # pylint: disable=consider-using-with
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID): async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1) log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
await self.loop.run_in_executor(None, self._write_decrypted_blob, file_write_handle, decrypted) await self.loop.run_in_executor(None, self._write_decrypted_blob, output_path, decrypted)
if not self.started_writing.is_set(): if not self.started_writing.is_set():
self.started_writing.set() self.started_writing.set()
await self.update_status(ManagedStream.STATUS_FINISHED) await self.update_status(ManagedStream.STATUS_FINISHED)
@ -362,7 +279,7 @@ class ManagedStream:
log.info("finished saving file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id, log.info("finished saving file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id,
self.sd_hash[:6], self.full_path) self.sd_hash[:6], self.full_path)
await self.blob_manager.storage.set_saved_file(self.stream_hash) await self.blob_manager.storage.set_saved_file(self.stream_hash)
except Exception as err: except (Exception, asyncio.CancelledError) as err:
if os.path.isfile(output_path): if os.path.isfile(output_path):
log.warning("removing incomplete download %s for %s", output_path, self.sd_hash) log.warning("removing incomplete download %s for %s", output_path, self.sd_hash)
os.remove(output_path) os.remove(output_path)
@ -382,22 +299,21 @@ class ManagedStream:
self.saving.clear() self.saving.clear()
self.finished_write_attempt.set() self.finished_write_attempt.set()
async def save_file(self, file_name: typing.Optional[str] = None, download_directory: typing.Optional[str] = None, async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
node: typing.Optional['Node'] = None): await self.start()
await self.start(node)
if self.file_output_task and not self.file_output_task.done(): # cancel an already running save task if self.file_output_task and not self.file_output_task.done(): # cancel an already running save task
self.file_output_task.cancel() self.file_output_task.cancel()
self.download_directory = download_directory or self.download_directory or self.config.download_dir self.download_directory = download_directory or self.download_directory or self.config.download_dir
if not self.download_directory: if not self.download_directory:
raise ValueError("no directory to download to") raise ValueError("no directory to download to")
if not (file_name or self._file_name or self.descriptor.suggested_file_name): if not (file_name or self._file_name or self.suggested_file_name):
raise ValueError("no file name to download to") raise ValueError("no file name to download to")
if not os.path.isdir(self.download_directory): if not os.path.isdir(self.download_directory):
log.warning("download directory '%s' does not exist, attempting to make it", self.download_directory) log.warning("download directory '%s' does not exist, attempting to make it", self.download_directory)
os.mkdir(self.download_directory) os.mkdir(self.download_directory)
self._file_name = await get_next_available_file_name( self._file_name = await get_next_available_file_name(
self.loop, self.download_directory, self.loop, self.download_directory,
file_name or self._file_name or sanitize_file_name(self.descriptor.suggested_file_name) file_name or self._file_name or sanitize_file_name(self.suggested_file_name)
) )
await self.blob_manager.storage.change_file_download_dir_and_file_name( await self.blob_manager.storage.change_file_download_dir_and_file_name(
self.stream_hash, self.download_directory, self.file_name self.stream_hash, self.download_directory, self.file_name
@ -405,15 +321,16 @@ class ManagedStream:
await self.update_status(ManagedStream.STATUS_RUNNING) await self.update_status(ManagedStream.STATUS_RUNNING)
self.file_output_task = self.loop.create_task(self._save_file(self.full_path)) self.file_output_task = self.loop.create_task(self._save_file(self.full_path))
try: try:
await asyncio.wait_for(self.started_writing.wait(), self.config.download_timeout, loop=self.loop) await asyncio.wait_for(self.started_writing.wait(), self.config.download_timeout)
except asyncio.TimeoutError: except asyncio.TimeoutError:
log.warning("timeout starting to write data for lbry://%s#%s", self.claim_name, self.claim_id) log.warning("timeout starting to write data for lbry://%s#%s", self.claim_name, self.claim_id)
self.stop_tasks() await self.stop_tasks()
await self.update_status(ManagedStream.STATUS_STOPPED) await self.update_status(ManagedStream.STATUS_STOPPED)
def stop_tasks(self): async def stop_tasks(self):
if self.file_output_task and not self.file_output_task.done(): if self.file_output_task and not self.file_output_task.done():
self.file_output_task.cancel() self.file_output_task.cancel()
await asyncio.gather(self.file_output_task, return_exceptions=True)
self.file_output_task = None self.file_output_task = None
while self.streaming_responses: while self.streaming_responses:
req, response = self.streaming_responses.pop() req, response = self.streaming_responses.pop()
@ -426,12 +343,13 @@ class ManagedStream:
sent = [] sent = []
protocol = StreamReflectorClient(self.blob_manager, self.descriptor) protocol = StreamReflectorClient(self.blob_manager, self.descriptor)
try: try:
self.uploading_to_reflector = True
await self.loop.create_connection(lambda: protocol, host, port) await self.loop.create_connection(lambda: protocol, host, port)
await protocol.send_handshake() await protocol.send_handshake()
sent_sd, needed = await protocol.send_descriptor() sent_sd, needed = await protocol.send_descriptor()
if sent_sd: if sent_sd: # reflector needed the sd blob
sent.append(self.sd_hash) sent.append(self.sd_hash)
if not sent_sd and not needed: if not sent_sd and not needed: # reflector already has the stream
if not self.fully_reflected.is_set(): if not self.fully_reflected.is_set():
self.fully_reflected.set() self.fully_reflected.set()
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}") await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
@ -441,30 +359,33 @@ class ManagedStream:
] ]
log.info("we have %i/%i needed blobs needed by reflector for lbry://%s#%s", len(we_have), len(needed), log.info("we have %i/%i needed blobs needed by reflector for lbry://%s#%s", len(we_have), len(needed),
self.claim_name, self.claim_id) self.claim_name, self.claim_id)
for blob_hash in we_have: for i, blob_hash in enumerate(we_have):
await protocol.send_blob(blob_hash) await protocol.send_blob(blob_hash)
sent.append(blob_hash) sent.append(blob_hash)
self.reflector_progress = int((i + 1) / len(we_have) * 100)
except (asyncio.TimeoutError, ValueError): except (asyncio.TimeoutError, ValueError):
return sent return sent
except ConnectionRefusedError: except ConnectionError:
return sent
except (OSError, Exception, asyncio.CancelledError) as err:
if isinstance(err, asyncio.CancelledError):
log.warning("stopped uploading %s#%s to reflector", self.claim_name, self.claim_id)
elif isinstance(err, OSError):
log.warning(
"stopped uploading %s#%s to reflector because blobs were deleted or moved", self.claim_name,
self.claim_id
)
else:
log.exception("unexpected error reflecting %s#%s", self.claim_name, self.claim_id)
return sent return sent
finally: finally:
if protocol.transport: if protocol.transport:
protocol.transport.close() protocol.transport.close()
if not self.fully_reflected.is_set(): self.uploading_to_reflector = False
self.fully_reflected.set()
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
return sent return sent
def set_claim(self, claim_info: typing.Dict, claim: 'Claim'): async def update_content_claim(self, claim_info: Optional[typing.Dict] = None):
self.stream_claim_info = StoredContentClaim(
f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
claim_info['name'], claim_info['amount'], claim_info['height'],
binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'],
claim_info['claim_sequence'], claim_info.get('channel_name')
)
async def update_content_claim(self, claim_info: typing.Optional[typing.Dict] = None):
if not claim_info: if not claim_info:
claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash) claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash)
self.set_claim(claim_info, claim_info['value']) self.set_claim(claim_info, claim_info['value'])
@ -481,7 +402,7 @@ class ManagedStream:
self.sd_hash[:6]) self.sd_hash[:6])
await self.stop() await self.stop()
return return
await asyncio.sleep(1, loop=self.loop) await asyncio.sleep(1)
def _prepare_range_response_headers(self, get_range: str) -> typing.Tuple[typing.Dict[str, str], int, int, int]: def _prepare_range_response_headers(self, get_range: str) -> typing.Tuple[typing.Dict[str, str], int, int, int]:
if '=' in get_range: if '=' in get_range:

View file

@ -35,6 +35,8 @@ class StreamReflectorClient(asyncio.Protocol):
def connection_lost(self, exc: typing.Optional[Exception]): def connection_lost(self, exc: typing.Optional[Exception]):
self.transport = None self.transport = None
self.connected.clear() self.connected.clear()
if self.pending_request:
self.pending_request.cancel()
if self.reflected_blobs: if self.reflected_blobs:
log.info("Finished sending reflector %i blobs", len(self.reflected_blobs)) log.info("Finished sending reflector %i blobs", len(self.reflected_blobs))
@ -56,12 +58,18 @@ class StreamReflectorClient(asyncio.Protocol):
self.response_buff = b'' self.response_buff = b''
return return
async def send_request(self, request_dict: typing.Dict): async def send_request(self, request_dict: typing.Dict, timeout: int = 180):
msg = json.dumps(request_dict) msg = json.dumps(request_dict, sort_keys=True)
self.transport.write(msg.encode())
try: try:
self.pending_request = self.loop.create_task(self.response_queue.get()) self.transport.write(msg.encode())
self.pending_request = self.loop.create_task(asyncio.wait_for(self.response_queue.get(), timeout))
return await self.pending_request return await self.pending_request
except (AttributeError, asyncio.CancelledError) as err:
# attribute error happens when we transport.write after disconnect
# cancelled error happens when the pending_request task is cancelled by a disconnect
if self.transport:
self.transport.close()
raise err if isinstance(err, asyncio.CancelledError) else asyncio.CancelledError()
finally: finally:
self.pending_request = None self.pending_request = None
@ -86,8 +94,16 @@ class StreamReflectorClient(asyncio.Protocol):
needed = response.get('needed_blobs', []) needed = response.get('needed_blobs', [])
sent_sd = False sent_sd = False
if response['send_sd_blob']: if response['send_sd_blob']:
await sd_blob.sendfile(self) try:
received = await self.response_queue.get() sent = await sd_blob.sendfile(self)
if sent == -1:
log.warning("failed to send sd blob")
raise asyncio.CancelledError()
received = await asyncio.wait_for(self.response_queue.get(), 30)
except asyncio.CancelledError as err:
if self.transport:
self.transport.close()
raise err
if received.get('received_sd_blob'): if received.get('received_sd_blob'):
sent_sd = True sent_sd = True
if not needed: if not needed:
@ -110,8 +126,16 @@ class StreamReflectorClient(asyncio.Protocol):
if 'send_blob' not in response: if 'send_blob' not in response:
raise ValueError("I don't know whether to send the blob or not!") raise ValueError("I don't know whether to send the blob or not!")
if response['send_blob']: if response['send_blob']:
await blob.sendfile(self) try:
received = await self.response_queue.get() sent = await blob.sendfile(self)
if sent == -1:
log.warning("failed to send blob")
raise asyncio.CancelledError()
received = await asyncio.wait_for(self.response_queue.get(), 30)
except asyncio.CancelledError as err:
if self.transport:
self.transport.close()
raise err
if received.get('received_blob'): if received.get('received_blob'):
self.reflected_blobs.append(blob.blob_hash) self.reflected_blobs.append(blob.blob_hash)
log.info("Sent reflector blob %s", blob.blob_hash[:8]) log.info("Sent reflector blob %s", blob.blob_hash[:8])

View file

@ -15,11 +15,13 @@ log = logging.getLogger(__name__)
class ReflectorServerProtocol(asyncio.Protocol): class ReflectorServerProtocol(asyncio.Protocol):
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000): def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000,
stop_event: asyncio.Event = None, incoming_event: asyncio.Event = None,
not_incoming_event: asyncio.Event = None, partial_event: asyncio.Event = None):
self.loop = asyncio.get_event_loop() self.loop = asyncio.get_event_loop()
self.blob_manager = blob_manager self.blob_manager = blob_manager
self.server_task: asyncio.Task = None self.server_task: asyncio.Task = None
self.started_listening = asyncio.Event(loop=self.loop) self.started_listening = asyncio.Event()
self.buf = b'' self.buf = b''
self.transport: asyncio.StreamWriter = None self.transport: asyncio.StreamWriter = None
self.writer: typing.Optional['HashBlobWriter'] = None self.writer: typing.Optional['HashBlobWriter'] = None
@ -27,11 +29,26 @@ class ReflectorServerProtocol(asyncio.Protocol):
self.descriptor: typing.Optional['StreamDescriptor'] = None self.descriptor: typing.Optional['StreamDescriptor'] = None
self.sd_blob: typing.Optional['BlobFile'] = None self.sd_blob: typing.Optional['BlobFile'] = None
self.received = [] self.received = []
self.incoming = asyncio.Event(loop=self.loop) self.incoming = incoming_event or asyncio.Event()
self.not_incoming = not_incoming_event or asyncio.Event()
self.stop_event = stop_event or asyncio.Event()
self.chunk_size = response_chunk_size self.chunk_size = response_chunk_size
self.wait_for_stop_task: typing.Optional[asyncio.Task] = None
self.partial_event = partial_event
async def wait_for_stop(self):
await self.stop_event.wait()
if self.transport:
self.transport.close()
def connection_made(self, transport): def connection_made(self, transport):
self.transport = transport self.transport = transport
self.wait_for_stop_task = self.loop.create_task(self.wait_for_stop())
def connection_lost(self, exc):
if self.wait_for_stop_task:
self.wait_for_stop_task.cancel()
self.wait_for_stop_task = None
def data_received(self, data: bytes): def data_received(self, data: bytes):
if self.incoming.is_set(): if self.incoming.is_set():
@ -73,10 +90,11 @@ class ReflectorServerProtocol(asyncio.Protocol):
self.sd_blob = self.blob_manager.get_blob(request['sd_blob_hash'], request['sd_blob_size']) self.sd_blob = self.blob_manager.get_blob(request['sd_blob_hash'], request['sd_blob_size'])
if not self.sd_blob.get_is_verified(): if not self.sd_blob.get_is_verified():
self.writer = self.sd_blob.get_blob_writer(self.transport.get_extra_info('peername')) self.writer = self.sd_blob.get_blob_writer(self.transport.get_extra_info('peername'))
self.not_incoming.clear()
self.incoming.set() self.incoming.set()
self.send_response({"send_sd_blob": True}) self.send_response({"send_sd_blob": True})
try: try:
await asyncio.wait_for(self.sd_blob.verified.wait(), 30, loop=self.loop) await asyncio.wait_for(self.sd_blob.verified.wait(), 30)
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob( self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
self.loop, self.blob_manager.blob_dir, self.sd_blob self.loop, self.blob_manager.blob_dir, self.sd_blob
) )
@ -86,6 +104,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
self.transport.close() self.transport.close()
finally: finally:
self.incoming.clear() self.incoming.clear()
self.not_incoming.set()
self.writer.close_handle() self.writer.close_handle()
self.writer = None self.writer = None
else: else:
@ -93,13 +112,18 @@ class ReflectorServerProtocol(asyncio.Protocol):
self.loop, self.blob_manager.blob_dir, self.sd_blob self.loop, self.blob_manager.blob_dir, self.sd_blob
) )
self.incoming.clear() self.incoming.clear()
self.not_incoming.set()
if self.writer: if self.writer:
self.writer.close_handle() self.writer.close_handle()
self.writer = None self.writer = None
self.send_response({"send_sd_blob": False, 'needed': [
blob.blob_hash for blob in self.descriptor.blobs[:-1] needs = [blob.blob_hash
if not self.blob_manager.get_blob(blob.blob_hash).get_is_verified() for blob in self.descriptor.blobs[:-1]
]}) if not self.blob_manager.get_blob(blob.blob_hash).get_is_verified()]
if needs and not self.partial_event.is_set():
needs = needs[:3]
self.partial_event.set()
self.send_response({"send_sd_blob": False, 'needed_blobs': needs})
return return
return return
elif self.descriptor: elif self.descriptor:
@ -112,14 +136,16 @@ class ReflectorServerProtocol(asyncio.Protocol):
blob = self.blob_manager.get_blob(request['blob_hash'], request['blob_size']) blob = self.blob_manager.get_blob(request['blob_hash'], request['blob_size'])
if not blob.get_is_verified(): if not blob.get_is_verified():
self.writer = blob.get_blob_writer(self.transport.get_extra_info('peername')) self.writer = blob.get_blob_writer(self.transport.get_extra_info('peername'))
self.not_incoming.clear()
self.incoming.set() self.incoming.set()
self.send_response({"send_blob": True}) self.send_response({"send_blob": True})
try: try:
await asyncio.wait_for(blob.verified.wait(), 30, loop=self.loop) await asyncio.wait_for(blob.verified.wait(), 30)
self.send_response({"received_blob": True}) self.send_response({"received_blob": True})
except asyncio.TimeoutError: except asyncio.TimeoutError:
self.send_response({"received_blob": False}) self.send_response({"received_blob": False})
self.incoming.clear() self.incoming.clear()
self.not_incoming.set()
self.writer.close_handle() self.writer.close_handle()
self.writer = None self.writer = None
else: else:
@ -130,26 +156,39 @@ class ReflectorServerProtocol(asyncio.Protocol):
class ReflectorServer: class ReflectorServer:
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000): def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000,
stop_event: asyncio.Event = None, incoming_event: asyncio.Event = None,
not_incoming_event: asyncio.Event = None, partial_needs=False):
self.loop = asyncio.get_event_loop() self.loop = asyncio.get_event_loop()
self.blob_manager = blob_manager self.blob_manager = blob_manager
self.server_task: typing.Optional[asyncio.Task] = None self.server_task: typing.Optional[asyncio.Task] = None
self.started_listening = asyncio.Event(loop=self.loop) self.started_listening = asyncio.Event()
self.stopped_listening = asyncio.Event()
self.incoming_event = incoming_event or asyncio.Event()
self.not_incoming_event = not_incoming_event or asyncio.Event()
self.response_chunk_size = response_chunk_size self.response_chunk_size = response_chunk_size
self.stop_event = stop_event
self.partial_needs = partial_needs # for testing cases where it doesn't know what it wants
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'): def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
if self.server_task is not None: if self.server_task is not None:
raise Exception("already running") raise Exception("already running")
async def _start_server(): async def _start_server():
server = await self.loop.create_server( partial_event = asyncio.Event()
lambda: ReflectorServerProtocol(self.blob_manager, self.response_chunk_size), if not self.partial_needs:
interface, port partial_event.set()
) server = await self.loop.create_server(lambda: ReflectorServerProtocol(
self.blob_manager, self.response_chunk_size, self.stop_event, self.incoming_event,
self.not_incoming_event, partial_event), interface, port)
self.started_listening.set() self.started_listening.set()
self.stopped_listening.clear()
log.info("Reflector server listening on TCP %s:%i", interface, port) log.info("Reflector server listening on TCP %s:%i", interface, port)
try:
async with server: async with server:
await server.serve_forever() await server.serve_forever()
finally:
self.stopped_listening.set()
self.server_task = self.loop.create_task(_start_server()) self.server_task = self.loop.create_task(_start_server())

View file

@ -6,86 +6,71 @@ import random
import typing import typing
from typing import Optional from typing import Optional
from aiohttp.web import Request from aiohttp.web import Request
from lbry.error import ResolveError, InvalidStreamDescriptorError, DownloadSDTimeoutError, InsufficientFundsError from lbry.error import InvalidStreamDescriptorError
from lbry.error import ResolveTimeoutError, DownloadDataTimeoutError, KeyFeeAboveMaxAllowedError from lbry.file.source_manager import SourceManager
from lbry.utils import cache_concurrent
from lbry.stream.descriptor import StreamDescriptor from lbry.stream.descriptor import StreamDescriptor
from lbry.stream.managed_stream import ManagedStream from lbry.stream.managed_stream import ManagedStream
from lbry.schema.claim import Claim from lbry.file.source import ManagedDownloadSource
from lbry.schema.url import URL
from lbry.wallet.dewies import dewies_to_lbc
from lbry.wallet import Output
if typing.TYPE_CHECKING: if typing.TYPE_CHECKING:
from lbry.conf import Config from lbry.conf import Config
from lbry.blob.blob_manager import BlobManager from lbry.blob.blob_manager import BlobManager
from lbry.dht.node import Node from lbry.dht.node import Node
from lbry.wallet.wallet import WalletManager
from lbry.wallet.transaction import Transaction
from lbry.extras.daemon.analytics import AnalyticsManager from lbry.extras.daemon.analytics import AnalyticsManager
from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
FILTER_FIELDS = [
'rowid', def path_or_none(encoded_path) -> Optional[str]:
'status', if not encoded_path:
'file_name', return
'added_on', return binascii.unhexlify(encoded_path).decode()
class StreamManager(SourceManager):
_sources: typing.Dict[str, ManagedStream]
filter_fields = SourceManager.filter_fields
filter_fields.update({
'sd_hash', 'sd_hash',
'stream_hash', 'stream_hash',
'claim_name',
'claim_height',
'claim_id',
'outpoint',
'txid',
'nout',
'channel_claim_id',
'channel_name',
'full_status', # TODO: remove 'full_status', # TODO: remove
'blobs_remaining', 'blobs_remaining',
'blobs_in_stream' 'blobs_in_stream',
] 'uploading_to_reflector',
'is_fully_reflected'
})
COMPARISON_OPERATORS = {
'eq': lambda a, b: a == b,
'ne': lambda a, b: a != b,
'g': lambda a, b: a > b,
'l': lambda a, b: a < b,
'ge': lambda a, b: a >= b,
'le': lambda a, b: a <= b,
}
def path_or_none(path) -> Optional[str]:
if not path:
return
return binascii.unhexlify(path).decode()
class StreamManager:
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager', def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
wallet_manager: 'WalletManager', storage: 'SQLiteStorage', node: Optional['Node'], wallet_manager: 'WalletManager', storage: 'SQLiteStorage', node: Optional['Node'],
analytics_manager: Optional['AnalyticsManager'] = None): analytics_manager: Optional['AnalyticsManager'] = None):
self.loop = loop super().__init__(loop, config, storage, analytics_manager)
self.config = config
self.blob_manager = blob_manager self.blob_manager = blob_manager
self.wallet_manager = wallet_manager self.wallet_manager = wallet_manager
self.storage = storage
self.node = node self.node = node
self.analytics_manager = analytics_manager
self.streams: typing.Dict[str, ManagedStream] = {}
self.resume_saving_task: Optional[asyncio.Task] = None self.resume_saving_task: Optional[asyncio.Task] = None
self.re_reflect_task: Optional[asyncio.Task] = None self.re_reflect_task: Optional[asyncio.Task] = None
self.update_stream_finished_futs: typing.List[asyncio.Future] = [] self.update_stream_finished_futs: typing.List[asyncio.Future] = []
self.running_reflector_uploads: typing.Dict[str, asyncio.Task] = {} self.running_reflector_uploads: typing.Dict[str, asyncio.Task] = {}
self.started = asyncio.Event(loop=self.loop) self.started = asyncio.Event()
@property
def streams(self):
return self._sources
def add(self, source: ManagedStream):
super().add(source)
self.storage.content_claim_callbacks[source.stream_hash] = lambda: self._update_content_claim(source)
async def _update_content_claim(self, stream: ManagedStream): async def _update_content_claim(self, stream: ManagedStream):
claim_info = await self.storage.get_content_claim(stream.stream_hash) claim_info = await self.storage.get_content_claim(stream.stream_hash)
self.streams.setdefault(stream.sd_hash, stream).set_claim(claim_info, claim_info['value']) self._sources.setdefault(stream.sd_hash, stream).set_claim(claim_info, claim_info['value'])
async def recover_streams(self, file_infos: typing.List[typing.Dict]): async def recover_streams(self, file_infos: typing.List[typing.Dict]):
to_restore = [] to_restore = []
to_check = []
async def recover_stream(sd_hash: str, stream_hash: str, stream_name: str, async def recover_stream(sd_hash: str, stream_hash: str, stream_name: str,
suggested_file_name: str, key: str, suggested_file_name: str, key: str,
@ -98,6 +83,7 @@ class StreamManager:
if not descriptor: if not descriptor:
return return
to_restore.append((descriptor, sd_blob, content_fee)) to_restore.append((descriptor, sd_blob, content_fee))
to_check.extend([sd_blob.blob_hash] + [blob.blob_hash for blob in descriptor.blobs[:-1]])
await asyncio.gather(*[ await asyncio.gather(*[
recover_stream( recover_stream(
@ -109,14 +95,16 @@ class StreamManager:
if to_restore: if to_restore:
await self.storage.recover_streams(to_restore, self.config.download_dir) await self.storage.recover_streams(to_restore, self.config.download_dir)
if to_check:
await self.blob_manager.ensure_completed_blobs_status(to_check)
# if self.blob_manager._save_blobs: # if self.blob_manager._save_blobs:
# log.info("Recovered %i/%i attempted streams", len(to_restore), len(file_infos)) # log.info("Recovered %i/%i attempted streams", len(to_restore), len(file_infos))
async def add_stream(self, rowid: int, sd_hash: str, file_name: Optional[str], async def _load_stream(self, rowid: int, sd_hash: str, file_name: Optional[str],
download_directory: Optional[str], status: str, download_directory: Optional[str], status: str,
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'], claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
added_on: Optional[int]): added_on: Optional[int], fully_reflected: Optional[bool]):
try: try:
descriptor = await self.blob_manager.get_stream_descriptor(sd_hash) descriptor = await self.blob_manager.get_stream_descriptor(sd_hash)
except InvalidStreamDescriptorError as err: except InvalidStreamDescriptorError as err:
@ -127,10 +115,11 @@ class StreamManager:
claim, content_fee=content_fee, rowid=rowid, descriptor=descriptor, claim, content_fee=content_fee, rowid=rowid, descriptor=descriptor,
analytics_manager=self.analytics_manager, added_on=added_on analytics_manager=self.analytics_manager, added_on=added_on
) )
self.streams[sd_hash] = stream if fully_reflected:
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream) stream.fully_reflected.set()
self.add(stream)
async def load_and_resume_streams_from_database(self): async def initialize_from_database(self):
to_recover = [] to_recover = []
to_start = [] to_start = []
@ -144,7 +133,6 @@ class StreamManager:
to_recover.append(file_info) to_recover.append(file_info)
to_start.append(file_info) to_start.append(file_info)
if to_recover: if to_recover:
log.info("Recover %i files", len(to_recover))
await self.recover_streams(to_recover) await self.recover_streams(to_recover)
log.info("Initializing %i files", len(to_start)) log.info("Initializing %i files", len(to_start))
@ -155,33 +143,37 @@ class StreamManager:
download_directory = path_or_none(file_info['download_directory']) download_directory = path_or_none(file_info['download_directory'])
if file_name and download_directory and not file_info['saved_file'] and file_info['status'] == 'running': if file_name and download_directory and not file_info['saved_file'] and file_info['status'] == 'running':
to_resume_saving.append((file_name, download_directory, file_info['sd_hash'])) to_resume_saving.append((file_name, download_directory, file_info['sd_hash']))
add_stream_tasks.append(self.loop.create_task(self.add_stream( add_stream_tasks.append(self.loop.create_task(self._load_stream(
file_info['rowid'], file_info['sd_hash'], file_name, file_info['rowid'], file_info['sd_hash'], file_name,
download_directory, file_info['status'], download_directory, file_info['status'],
file_info['claim'], file_info['content_fee'], file_info['claim'], file_info['content_fee'],
file_info['added_on'] file_info['added_on'], file_info['fully_reflected']
))) )))
if add_stream_tasks: if add_stream_tasks:
await asyncio.gather(*add_stream_tasks, loop=self.loop) await asyncio.gather(*add_stream_tasks)
log.info("Started stream manager with %i files", len(self.streams)) log.info("Started stream manager with %i files", len(self._sources))
if not self.node: if not self.node:
log.info("no DHT node given, resuming downloads trusting that we can contact reflector") log.info("no DHT node given, resuming downloads trusting that we can contact reflector")
if to_resume_saving: if to_resume_saving:
self.resume_saving_task = self.loop.create_task(self.resume(to_resume_saving))
async def resume(self, to_resume_saving):
log.info("Resuming saving %i files", len(to_resume_saving)) log.info("Resuming saving %i files", len(to_resume_saving))
await asyncio.gather( self.resume_saving_task = asyncio.ensure_future(asyncio.gather(
*(self.streams[sd_hash].save_file(file_name, download_directory, node=self.node) *(self._sources[sd_hash].save_file(file_name, download_directory)
for (file_name, download_directory, sd_hash) in to_resume_saving), for (file_name, download_directory, sd_hash) in to_resume_saving),
loop=self.loop ))
)
async def reflect_streams(self): async def reflect_streams(self):
try:
return await self._reflect_streams()
except Exception:
log.exception("reflector task encountered an unexpected error!")
async def _reflect_streams(self):
# todo: those debug statements are temporary for #2987 - remove them if its closed
while True: while True:
if self.config.reflect_streams and self.config.reflector_servers: if self.config.reflect_streams and self.config.reflector_servers:
log.debug("collecting streams to reflect")
sd_hashes = await self.storage.get_streams_to_re_reflect() sd_hashes = await self.storage.get_streams_to_re_reflect()
sd_hashes = [sd for sd in sd_hashes if sd in self.streams] sd_hashes = [sd for sd in sd_hashes if sd in self._sources]
batch = [] batch = []
while sd_hashes: while sd_hashes:
stream = self.streams[sd_hashes.pop()] stream = self.streams[sd_hashes.pop()]
@ -190,25 +182,26 @@ class StreamManager:
stream.fully_reflected.is_set(): stream.fully_reflected.is_set():
batch.append(self.reflect_stream(stream)) batch.append(self.reflect_stream(stream))
if len(batch) >= self.config.concurrent_reflector_uploads: if len(batch) >= self.config.concurrent_reflector_uploads:
await asyncio.gather(*batch, loop=self.loop) log.debug("waiting for batch of %s reflecting streams", len(batch))
await asyncio.gather(*batch)
log.debug("done processing %s streams", len(batch))
batch = [] batch = []
if batch: if batch:
await asyncio.gather(*batch, loop=self.loop) log.debug("waiting for batch of %s reflecting streams", len(batch))
await asyncio.sleep(300, loop=self.loop) await asyncio.gather(*batch)
log.debug("done processing %s streams", len(batch))
await asyncio.sleep(300)
async def start(self): async def start(self):
await self.load_and_resume_streams_from_database() await super().start()
self.re_reflect_task = self.loop.create_task(self.reflect_streams()) self.re_reflect_task = self.loop.create_task(self.reflect_streams())
self.started.set()
def stop(self): async def stop(self):
await super().stop()
if self.resume_saving_task and not self.resume_saving_task.done(): if self.resume_saving_task and not self.resume_saving_task.done():
self.resume_saving_task.cancel() self.resume_saving_task.cancel()
if self.re_reflect_task and not self.re_reflect_task.done(): if self.re_reflect_task and not self.re_reflect_task.done():
self.re_reflect_task.cancel() self.re_reflect_task.cancel()
while self.streams:
_, stream = self.streams.popitem()
stream.stop_tasks()
while self.update_stream_finished_futs: while self.update_stream_finished_futs:
self.update_stream_finished_futs.pop().cancel() self.update_stream_finished_futs.pop().cancel()
while self.running_reflector_uploads: while self.running_reflector_uploads:
@ -223,7 +216,7 @@ class StreamManager:
server, port = random.choice(self.config.reflector_servers) server, port = random.choice(self.config.reflector_servers)
if stream.sd_hash in self.running_reflector_uploads: if stream.sd_hash in self.running_reflector_uploads:
return self.running_reflector_uploads[stream.sd_hash] return self.running_reflector_uploads[stream.sd_hash]
task = self.loop.create_task(stream.upload_to_reflector(server, port)) task = self.loop.create_task(self._retriable_reflect_stream(stream, server, port))
self.running_reflector_uploads[stream.sd_hash] = task self.running_reflector_uploads[stream.sd_hash] = task
task.add_done_callback( task.add_done_callback(
lambda _: None if stream.sd_hash not in self.running_reflector_uploads else lambda _: None if stream.sd_hash not in self.running_reflector_uploads else
@ -231,261 +224,53 @@ class StreamManager:
) )
return task return task
async def create_stream(self, file_path: str, key: Optional[bytes] = None, @staticmethod
async def _retriable_reflect_stream(stream, host, port):
sent = await stream.upload_to_reflector(host, port)
while not stream.is_fully_reflected and stream.reflector_progress > 0 and len(sent) > 0:
stream.reflector_progress = 0
sent = await stream.upload_to_reflector(host, port)
return sent
async def create(self, file_path: str, key: Optional[bytes] = None,
iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream: iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
stream = await ManagedStream.create(self.loop, self.config, self.blob_manager, file_path, key, iv_generator) descriptor = await StreamDescriptor.create_stream(
self.loop, self.blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
blob_completed_callback=self.blob_manager.blob_completed
)
await self.storage.store_stream(
self.blob_manager.get_blob(descriptor.sd_hash, is_mine=True), descriptor
)
row_id = await self.storage.save_published_file(
descriptor.stream_hash, os.path.basename(file_path), os.path.dirname(file_path), 0
)
stream = ManagedStream(
self.loop, self.config, self.blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
os.path.basename(file_path), status=ManagedDownloadSource.STATUS_FINISHED,
rowid=row_id, descriptor=descriptor
)
self.streams[stream.sd_hash] = stream self.streams[stream.sd_hash] = stream
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream) self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
if self.config.reflect_streams and self.config.reflector_servers: if self.config.reflect_streams and self.config.reflector_servers:
self.reflect_stream(stream) self.reflect_stream(stream)
return stream return stream
async def delete_stream(self, stream: ManagedStream, delete_file: Optional[bool] = False): async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
if stream.sd_hash in self.running_reflector_uploads: if not isinstance(source, ManagedStream):
self.running_reflector_uploads[stream.sd_hash].cancel() return
stream.stop_tasks() if source.identifier in self.running_reflector_uploads:
if stream.sd_hash in self.streams: self.running_reflector_uploads[source.identifier].cancel()
del self.streams[stream.sd_hash] await source.stop_tasks()
blob_hashes = [stream.sd_hash] + [b.blob_hash for b in stream.descriptor.blobs[:-1]] if source.identifier in self.streams:
del self.streams[source.identifier]
blob_hashes = [source.identifier] + [b.blob_hash for b in source.descriptor.blobs[:-1]]
await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False) await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False)
await self.storage.delete_stream(stream.descriptor) await self.storage.delete_stream(source.descriptor)
if delete_file and stream.output_file_exists: if delete_file and source.output_file_exists:
os.remove(stream.full_path) os.remove(source.full_path)
def get_stream_by_stream_hash(self, stream_hash: str) -> Optional[ManagedStream]:
streams = tuple(filter(lambda stream: stream.stream_hash == stream_hash, self.streams.values()))
if streams:
return streams[0]
def get_filtered_streams(self, sort_by: Optional[str] = None, reverse: Optional[bool] = False,
comparison: Optional[str] = None,
**search_by) -> typing.List[ManagedStream]:
"""
Get a list of filtered and sorted ManagedStream objects
:param sort_by: field to sort by
:param reverse: reverse sorting
:param comparison: comparison operator used for filtering
:param search_by: fields and values to filter by
"""
if sort_by and sort_by not in FILTER_FIELDS:
raise ValueError(f"'{sort_by}' is not a valid field to sort by")
if comparison and comparison not in COMPARISON_OPERATORS:
raise ValueError(f"'{comparison}' is not a valid comparison")
if 'full_status' in search_by:
del search_by['full_status']
for search in search_by:
if search not in FILTER_FIELDS:
raise ValueError(f"'{search}' is not a valid search operation")
if search_by:
comparison = comparison or 'eq'
streams = []
for stream in self.streams.values():
for search, val in search_by.items():
if COMPARISON_OPERATORS[comparison](getattr(stream, search), val):
streams.append(stream)
break
else:
streams = list(self.streams.values())
if sort_by:
streams.sort(key=lambda s: getattr(s, sort_by))
if reverse:
streams.reverse()
return streams
async def _check_update_or_replace(self, outpoint: str, claim_id: str, claim: Claim
) -> typing.Tuple[Optional[ManagedStream], Optional[ManagedStream]]:
existing = self.get_filtered_streams(outpoint=outpoint)
if existing:
return existing[0], None
existing = self.get_filtered_streams(sd_hash=claim.stream.source.sd_hash)
if existing and existing[0].claim_id != claim_id:
raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {claim_id}")
if existing:
log.info("claim contains a metadata only update to a stream we have")
await self.storage.save_content_claim(
existing[0].stream_hash, outpoint
)
await self._update_content_claim(existing[0])
return existing[0], None
else:
existing_for_claim_id = self.get_filtered_streams(claim_id=claim_id)
if existing_for_claim_id:
log.info("claim contains an update to a stream we have, downloading it")
return None, existing_for_claim_id[0]
return None, None
@staticmethod
def _convert_to_old_resolve_output(wallet_manager, resolves):
result = {}
for url, txo in resolves.items():
if isinstance(txo, Output):
tx_height = txo.tx_ref.height
best_height = wallet_manager.ledger.headers.height
result[url] = {
'name': txo.claim_name,
'value': txo.claim,
'protobuf': binascii.hexlify(txo.claim.to_bytes()),
'claim_id': txo.claim_id,
'txid': txo.tx_ref.id,
'nout': txo.position,
'amount': dewies_to_lbc(txo.amount),
'effective_amount': txo.meta.get('effective_amount', 0),
'height': tx_height,
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
'claim_sequence': -1,
'address': txo.get_address(wallet_manager.ledger),
'valid_at_height': txo.meta.get('activation_height', None),
'timestamp': wallet_manager.ledger.headers[tx_height]['timestamp'],
'supports': []
}
else:
result[url] = txo
return result
@cache_concurrent
async def download_stream_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
timeout: Optional[float] = None,
file_name: Optional[str] = None,
download_directory: Optional[str] = None,
save_file: Optional[bool] = None,
resolve_timeout: float = 3.0,
wallet: Optional['Wallet'] = None) -> ManagedStream:
manager = self.wallet_manager
wallet = wallet or manager.default_wallet
timeout = timeout or self.config.download_timeout
start_time = self.loop.time()
resolved_time = None
stream = None
txo: Optional[Output] = None
error = None
outpoint = None
if save_file is None:
save_file = self.config.save_files
if file_name and not save_file:
save_file = True
if save_file:
download_directory = download_directory or self.config.download_dir
else:
download_directory = None
payment = None
try:
# resolve the claim
if not URL.parse(uri).has_stream:
raise ResolveError("cannot download a channel claim, specify a /path")
try:
response = await asyncio.wait_for(
manager.ledger.resolve(wallet.accounts, [uri]),
resolve_timeout
)
resolved_result = self._convert_to_old_resolve_output(manager, response)
except asyncio.TimeoutError:
raise ResolveTimeoutError(uri)
except Exception as err:
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
raise
log.exception("Unexpected error resolving stream:")
raise ResolveError(f"Unexpected error resolving stream: {str(err)}")
await self.storage.save_claims_for_resolve([
value for value in resolved_result.values() if 'error' not in value
])
resolved = resolved_result.get(uri, {})
resolved = resolved if 'value' in resolved else resolved.get('claim')
if not resolved:
raise ResolveError(f"Failed to resolve stream at '{uri}'")
if 'error' in resolved:
raise ResolveError(f"error resolving stream: {resolved['error']}")
txo = response[uri]
claim = Claim.from_bytes(binascii.unhexlify(resolved['protobuf']))
outpoint = f"{resolved['txid']}:{resolved['nout']}"
resolved_time = self.loop.time() - start_time
# resume or update an existing stream, if the stream changed: download it and delete the old one after
updated_stream, to_replace = await self._check_update_or_replace(outpoint, resolved['claim_id'], claim)
if updated_stream:
log.info("already have stream for %s", uri)
if save_file and updated_stream.output_file_exists:
save_file = False
await updated_stream.start(node=self.node, timeout=timeout, save_now=save_file)
if not updated_stream.output_file_exists and (save_file or file_name or download_directory):
await updated_stream.save_file(
file_name=file_name, download_directory=download_directory, node=self.node
)
return updated_stream
if not to_replace and txo.has_price and not txo.purchase_receipt:
payment = await manager.create_purchase_transaction(
wallet.accounts, txo, exchange_rate_manager
)
stream = ManagedStream(
self.loop, self.config, self.blob_manager, claim.stream.source.sd_hash, download_directory,
file_name, ManagedStream.STATUS_RUNNING, content_fee=payment,
analytics_manager=self.analytics_manager
)
log.info("starting download for %s", uri)
before_download = self.loop.time()
await stream.start(self.node, timeout)
stream.set_claim(resolved, claim)
if to_replace: # delete old stream now that the replacement has started downloading
await self.delete_stream(to_replace)
if payment is not None:
await manager.broadcast_or_release(payment)
payment = None # to avoid releasing in `finally` later
log.info("paid fee of %s for %s", dewies_to_lbc(stream.content_fee.outputs[0].amount), uri)
await self.storage.save_content_fee(stream.stream_hash, stream.content_fee)
self.streams[stream.sd_hash] = stream
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
await self.storage.save_content_claim(stream.stream_hash, outpoint)
if save_file:
await asyncio.wait_for(stream.save_file(node=self.node), timeout - (self.loop.time() - before_download),
loop=self.loop)
return stream
except asyncio.TimeoutError:
error = DownloadDataTimeoutError(stream.sd_hash)
raise error
except Exception as err: # forgive data timeout, don't delete stream
expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError,
KeyFeeAboveMaxAllowedError)
if isinstance(err, expected):
log.warning("Failed to download %s: %s", uri, str(err))
elif isinstance(err, asyncio.CancelledError):
pass
else:
log.exception("Unexpected error downloading stream:")
error = err
raise
finally:
if payment is not None:
# payment is set to None after broadcasting, if we're here an exception probably happened
await manager.ledger.release_tx(payment)
if self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or
stream.downloader.time_to_first_bytes))):
server = self.wallet_manager.ledger.network.client.server
self.loop.create_task(
self.analytics_manager.send_time_to_first_bytes(
resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id,
uri, outpoint,
None if not stream else len(stream.downloader.blob_downloader.active_connections),
None if not stream else len(stream.downloader.blob_downloader.scores),
None if not stream else len(stream.downloader.blob_downloader.connection_failures),
False if not stream else stream.downloader.added_fixed_peers,
self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay,
None if not stream else stream.sd_hash,
None if not stream else stream.downloader.time_to_descriptor,
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
None if not stream else stream.downloader.time_to_first_bytes,
None if not error else error.__class__.__name__,
None if not error else str(error),
None if not server else f"{server[0]}:{server[1]}"
)
)
async def stream_partial_content(self, request: Request, sd_hash: str): async def stream_partial_content(self, request: Request, sd_hash: str):
return await self.streams[sd_hash].stream_file(request, self.node) stream = self._sources[sd_hash]
if not stream.downloader.node:
stream.downloader.node = self.node
return await stream.stream_file(request)

View file

@ -17,18 +17,21 @@ from functools import partial
from lbry.wallet import WalletManager, Wallet, Ledger, Account, Transaction from lbry.wallet import WalletManager, Wallet, Ledger, Account, Transaction
from lbry.conf import Config from lbry.conf import Config
from lbry.wallet.util import satoshis_to_coins from lbry.wallet.util import satoshis_to_coins
from lbry.wallet.dewies import lbc_to_dewies
from lbry.wallet.orchstr8 import Conductor from lbry.wallet.orchstr8 import Conductor
from lbry.wallet.orchstr8.node import BlockchainNode, WalletNode from lbry.wallet.orchstr8.node import LBCWalletNode, WalletNode
from lbry.schema.claim import Claim
from lbry.extras.daemon.daemon import Daemon, jsonrpc_dumps_pretty from lbry.extras.daemon.daemon import Daemon, jsonrpc_dumps_pretty
from lbry.extras.daemon.components import Component, WalletComponent from lbry.extras.daemon.components import Component, WalletComponent
from lbry.extras.daemon.components import ( from lbry.extras.daemon.components import (
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT, DHT_COMPONENT,
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, LIBTORRENT_COMPONENT
) )
from lbry.extras.daemon.componentmanager import ComponentManager from lbry.extras.daemon.componentmanager import ComponentManager
from lbry.extras.daemon.exchange_rate_manager import ( from lbry.extras.daemon.exchange_rate_manager import (
ExchangeRateManager, ExchangeRate, LBRYFeed, LBRYBTCFeed ExchangeRateManager, ExchangeRate, BittrexBTCFeed, BittrexUSDFeed
) )
from lbry.extras.daemon.storage import SQLiteStorage from lbry.extras.daemon.storage import SQLiteStorage
from lbry.blob.blob_manager import BlobManager from lbry.blob.blob_manager import BlobManager
@ -83,6 +86,7 @@ class AsyncioTestCase(unittest.TestCase):
# https://bugs.python.org/issue32972 # https://bugs.python.org/issue32972
LOOP_SLOW_CALLBACK_DURATION = 0.2 LOOP_SLOW_CALLBACK_DURATION = 0.2
TIMEOUT = 120.0
maxDiff = None maxDiff = None
@ -130,15 +134,18 @@ class AsyncioTestCase(unittest.TestCase):
with outcome.testPartExecutor(self): with outcome.testPartExecutor(self):
self.setUp() self.setUp()
self.add_timeout()
self.loop.run_until_complete(self.asyncSetUp()) self.loop.run_until_complete(self.asyncSetUp())
if outcome.success: if outcome.success:
outcome.expecting_failure = expecting_failure outcome.expecting_failure = expecting_failure
with outcome.testPartExecutor(self, isTest=True): with outcome.testPartExecutor(self, isTest=True):
maybe_coroutine = testMethod() maybe_coroutine = testMethod()
if asyncio.iscoroutine(maybe_coroutine): if asyncio.iscoroutine(maybe_coroutine):
self.add_timeout()
self.loop.run_until_complete(maybe_coroutine) self.loop.run_until_complete(maybe_coroutine)
outcome.expecting_failure = False outcome.expecting_failure = False
with outcome.testPartExecutor(self): with outcome.testPartExecutor(self):
self.add_timeout()
self.loop.run_until_complete(self.asyncTearDown()) self.loop.run_until_complete(self.asyncTearDown())
self.tearDown() self.tearDown()
@ -186,8 +193,25 @@ class AsyncioTestCase(unittest.TestCase):
with outcome.testPartExecutor(self): with outcome.testPartExecutor(self):
maybe_coroutine = function(*args, **kwargs) maybe_coroutine = function(*args, **kwargs)
if asyncio.iscoroutine(maybe_coroutine): if asyncio.iscoroutine(maybe_coroutine):
self.add_timeout()
self.loop.run_until_complete(maybe_coroutine) self.loop.run_until_complete(maybe_coroutine)
def cancel(self):
for task in asyncio.all_tasks(self.loop):
if not task.done():
task.print_stack()
task.cancel()
def add_timeout(self):
if self.TIMEOUT:
self.loop.call_later(self.TIMEOUT, self.check_timeout, time())
def check_timeout(self, started):
if time() - started >= self.TIMEOUT:
self.cancel()
else:
self.loop.call_later(self.TIMEOUT, self.check_timeout, started)
class AdvanceTimeTestCase(AsyncioTestCase): class AdvanceTimeTestCase(AsyncioTestCase):
@ -212,7 +236,7 @@ class IntegrationTestCase(AsyncioTestCase):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
self.conductor: Optional[Conductor] = None self.conductor: Optional[Conductor] = None
self.blockchain: Optional[BlockchainNode] = None self.blockchain: Optional[LBCWalletNode] = None
self.wallet_node: Optional[WalletNode] = None self.wallet_node: Optional[WalletNode] = None
self.manager: Optional[WalletManager] = None self.manager: Optional[WalletManager] = None
self.ledger: Optional[Ledger] = None self.ledger: Optional[Ledger] = None
@ -221,13 +245,15 @@ class IntegrationTestCase(AsyncioTestCase):
async def asyncSetUp(self): async def asyncSetUp(self):
self.conductor = Conductor(seed=self.SEED) self.conductor = Conductor(seed=self.SEED)
await self.conductor.start_blockchain() await self.conductor.start_lbcd()
self.addCleanup(self.conductor.stop_blockchain) self.addCleanup(self.conductor.stop_lbcd)
await self.conductor.start_lbcwallet()
self.addCleanup(self.conductor.stop_lbcwallet)
await self.conductor.start_spv() await self.conductor.start_spv()
self.addCleanup(self.conductor.stop_spv) self.addCleanup(self.conductor.stop_spv)
await self.conductor.start_wallet() await self.conductor.start_wallet()
self.addCleanup(self.conductor.stop_wallet) self.addCleanup(self.conductor.stop_wallet)
self.blockchain = self.conductor.blockchain_node self.blockchain = self.conductor.lbcwallet_node
self.wallet_node = self.conductor.wallet_node self.wallet_node = self.conductor.wallet_node
self.manager = self.wallet_node.manager self.manager = self.wallet_node.manager
self.ledger = self.wallet_node.ledger self.ledger = self.wallet_node.ledger
@ -241,6 +267,13 @@ class IntegrationTestCase(AsyncioTestCase):
def broadcast(self, tx): def broadcast(self, tx):
return self.ledger.broadcast(tx) return self.ledger.broadcast(tx)
async def broadcast_and_confirm(self, tx, ledger=None):
ledger = ledger or self.ledger
notifications = asyncio.create_task(ledger.wait(tx))
await ledger.broadcast(tx)
await notifications
await self.generate_and_wait(1, [tx.id], ledger)
async def on_header(self, height): async def on_header(self, height):
if self.ledger.headers.height < height: if self.ledger.headers.height < height:
await self.ledger.on_header.where( await self.ledger.on_header.where(
@ -248,9 +281,32 @@ class IntegrationTestCase(AsyncioTestCase):
) )
return True return True
def on_transaction_id(self, txid, ledger=None): async def send_to_address_and_wait(self, address, amount, blocks_to_generate=0, ledger=None):
return (ledger or self.ledger).on_transaction.where( tx_watch = []
lambda e: e.tx.id == txid txid = None
done = False
watcher = (ledger or self.ledger).on_transaction.where(
lambda e: e.tx.id == txid or done or tx_watch.append(e.tx.id)
)
txid = await self.blockchain.send_to_address(address, amount)
done = txid in tx_watch
await watcher
await self.generate_and_wait(blocks_to_generate, [txid], ledger)
return txid
async def generate_and_wait(self, blocks_to_generate, txids, ledger=None):
if blocks_to_generate > 0:
watcher = (ledger or self.ledger).on_transaction.where(
lambda e: ((e.tx.id in txids and txids.remove(e.tx.id)), len(txids) <= 0)[-1] # multi-statement lambda
)
await self.generate(blocks_to_generate)
await watcher
def on_address_update(self, address):
return self.ledger.on_transaction.where(
lambda e: e.address == address
) )
def on_transaction_address(self, tx, address): def on_transaction_address(self, tx, address):
@ -258,6 +314,22 @@ class IntegrationTestCase(AsyncioTestCase):
lambda e: e.tx.id == tx.id and e.address == address lambda e: e.tx.id == tx.id and e.address == address
) )
async def generate(self, blocks):
""" Ask lbrycrd to generate some blocks and wait until ledger has them. """
prepare = self.ledger.on_header.where(self.blockchain.is_expected_block)
self.conductor.spv_node.server.synchronized.clear()
await self.blockchain.generate(blocks)
height = self.blockchain.block_expected
await prepare # no guarantee that it didn't happen already, so start waiting from before calling generate
while True:
await self.conductor.spv_node.server.synchronized.wait()
self.conductor.spv_node.server.synchronized.clear()
if self.conductor.spv_node.server.db.db_height < height:
continue
if self.conductor.spv_node.server._es_height < height:
continue
break
class FakeExchangeRateManager(ExchangeRateManager): class FakeExchangeRateManager(ExchangeRateManager):
@ -276,8 +348,8 @@ class FakeExchangeRateManager(ExchangeRateManager):
def get_fake_exchange_rate_manager(rates=None): def get_fake_exchange_rate_manager(rates=None):
return FakeExchangeRateManager( return FakeExchangeRateManager(
[LBRYFeed(), LBRYBTCFeed()], [BittrexBTCFeed(), BittrexUSDFeed()],
rates or {'BTCLBC': 3.0, 'USDBTC': 2.0} rates or {'BTCLBC': 3.0, 'USDLBC': 2.0}
) )
@ -315,26 +387,32 @@ class CommandTestCase(IntegrationTestCase):
self.server_blob_manager = None self.server_blob_manager = None
self.server = None self.server = None
self.reflector = None self.reflector = None
self.skip_libtorrent = True
async def asyncSetUp(self): async def asyncSetUp(self):
await super().asyncSetUp()
logging.getLogger('lbry.blob_exchange').setLevel(self.VERBOSITY) logging.getLogger('lbry.blob_exchange').setLevel(self.VERBOSITY)
logging.getLogger('lbry.daemon').setLevel(self.VERBOSITY) logging.getLogger('lbry.daemon').setLevel(self.VERBOSITY)
logging.getLogger('lbry.stream').setLevel(self.VERBOSITY) logging.getLogger('lbry.stream').setLevel(self.VERBOSITY)
logging.getLogger('lbry.wallet').setLevel(self.VERBOSITY) logging.getLogger('lbry.wallet').setLevel(self.VERBOSITY)
await super().asyncSetUp()
self.daemon = await self.add_daemon(self.wallet_node) self.daemon = await self.add_daemon(self.wallet_node)
await self.account.ensure_address_gap() await self.account.ensure_address_gap()
address = (await self.account.receiving.get_addresses(limit=1, only_usable=True))[0] address = (await self.account.receiving.get_addresses(limit=1, only_usable=True))[0]
sendtxid = await self.blockchain.send_to_address(address, 10) await self.send_to_address_and_wait(address, 10, 6)
await self.confirm_tx(sendtxid)
await self.generate(5)
server_tmp_dir = tempfile.mkdtemp() server_tmp_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, server_tmp_dir) self.addCleanup(shutil.rmtree, server_tmp_dir)
self.server_config = Config() self.server_config = Config(
data_dir=server_tmp_dir,
wallet_dir=server_tmp_dir,
save_files=True,
download_dir=server_tmp_dir
)
self.server_config.transaction_cache_size = 10000
self.server_storage = SQLiteStorage(self.server_config, ':memory:') self.server_storage = SQLiteStorage(self.server_config, ':memory:')
await self.server_storage.open() await self.server_storage.open()
@ -357,6 +435,7 @@ class CommandTestCase(IntegrationTestCase):
await daemon.stop() await daemon.stop()
async def add_daemon(self, wallet_node=None, seed=None): async def add_daemon(self, wallet_node=None, seed=None):
start_wallet_node = False
if wallet_node is None: if wallet_node is None:
wallet_node = WalletNode( wallet_node = WalletNode(
self.wallet_node.manager_class, self.wallet_node.manager_class,
@ -364,30 +443,42 @@ class CommandTestCase(IntegrationTestCase):
port=self.extra_wallet_node_port port=self.extra_wallet_node_port
) )
self.extra_wallet_node_port += 1 self.extra_wallet_node_port += 1
await wallet_node.start(self.conductor.spv_node, seed=seed) start_wallet_node = True
self.extra_wallet_nodes.append(wallet_node)
upload_dir = os.path.join(wallet_node.data_path, 'uploads') upload_dir = os.path.join(wallet_node.data_path, 'uploads')
os.mkdir(upload_dir) os.mkdir(upload_dir)
conf = Config() conf = Config(
conf.data_dir = wallet_node.data_path # needed during instantiation to access known_hubs path
conf.wallet_dir = wallet_node.data_path data_dir=wallet_node.data_path,
conf.download_dir = wallet_node.data_path wallet_dir=wallet_node.data_path,
save_files=True,
download_dir=wallet_node.data_path
)
conf.upload_dir = upload_dir # not a real conf setting conf.upload_dir = upload_dir # not a real conf setting
conf.share_usage_data = False conf.share_usage_data = False
conf.use_upnp = False conf.use_upnp = False
conf.reflect_streams = True conf.reflect_streams = True
conf.blockchain_name = 'lbrycrd_regtest' conf.blockchain_name = 'lbrycrd_regtest'
conf.lbryum_servers = [('127.0.0.1', 50001)] conf.lbryum_servers = [(self.conductor.spv_node.hostname, self.conductor.spv_node.port)]
conf.reflector_servers = [('127.0.0.1', 5566)] conf.reflector_servers = [('127.0.0.1', 5566)]
conf.fixed_peers = [('127.0.0.1', 5567)]
conf.known_dht_nodes = [] conf.known_dht_nodes = []
conf.blob_lru_cache_size = self.blob_lru_cache_size conf.blob_lru_cache_size = self.blob_lru_cache_size
conf.transaction_cache_size = 10000
conf.components_to_skip = [ conf.components_to_skip = [
DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT, DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
PEER_PROTOCOL_SERVER_COMPONENT PEER_PROTOCOL_SERVER_COMPONENT
] ]
if self.skip_libtorrent:
conf.components_to_skip.append(LIBTORRENT_COMPONENT)
if start_wallet_node:
await wallet_node.start(self.conductor.spv_node, seed=seed, config=conf)
self.extra_wallet_nodes.append(wallet_node)
else:
wallet_node.manager.config = conf wallet_node.manager.config = conf
wallet_node.manager.ledger.config['known_hubs'] = conf.known_hubs
def wallet_maker(component_manager): def wallet_maker(component_manager):
wallet_component = WalletComponent(component_manager) wallet_component = WalletComponent(component_manager)
@ -398,7 +489,7 @@ class CommandTestCase(IntegrationTestCase):
daemon = Daemon(conf, ComponentManager( daemon = Daemon(conf, ComponentManager(
conf, skip_components=conf.components_to_skip, wallet=wallet_maker, conf, skip_components=conf.components_to_skip, wallet=wallet_maker,
exchange_rate_manager=partial(ExchangeRateManagerComponent, rates={ exchange_rate_manager=partial(ExchangeRateManagerComponent, rates={
'BTCLBC': 1.0, 'USDBTC': 2.0 'BTCLBC': 1.0, 'USDLBC': 2.0
}) })
)) ))
await daemon.initialize() await daemon.initialize()
@ -408,9 +499,14 @@ class CommandTestCase(IntegrationTestCase):
async def confirm_tx(self, txid, ledger=None): async def confirm_tx(self, txid, ledger=None):
""" Wait for tx to be in mempool, then generate a block, wait for tx to be in a block. """ """ Wait for tx to be in mempool, then generate a block, wait for tx to be in a block. """
await self.on_transaction_id(txid, ledger) # await (ledger or self.ledger).on_transaction.where(lambda e: e.tx.id == txid)
await self.generate(1) on_tx = (ledger or self.ledger).on_transaction.where(lambda e: e.tx.id == txid)
await self.on_transaction_id(txid, ledger) await asyncio.wait([self.generate(1), on_tx], timeout=5)
# # actually, if it's in the mempool or in the block we're fine
# await self.generate_and_wait(1, [txid], ledger=ledger)
# return txid
return txid return txid
async def on_transaction_dict(self, tx): async def on_transaction_dict(self, tx):
@ -425,11 +521,6 @@ class CommandTestCase(IntegrationTestCase):
addresses.add(txo['address']) addresses.add(txo['address'])
return list(addresses) return list(addresses)
async def generate(self, blocks):
""" Ask lbrycrd to generate some blocks and wait until ledger has them. """
await self.blockchain.generate(blocks)
await self.ledger.on_header.where(self.blockchain.is_expected_block)
async def blockchain_claim_name(self, name: str, value: str, amount: str, confirm=True): async def blockchain_claim_name(self, name: str, value: str, amount: str, confirm=True):
txid = await self.blockchain._cli_cmnd('claimname', name, value, amount) txid = await self.blockchain._cli_cmnd('claimname', name, value, amount)
if confirm: if confirm:
@ -450,12 +541,27 @@ class CommandTestCase(IntegrationTestCase):
""" Synchronous version of `out` method. """ """ Synchronous version of `out` method. """
return json.loads(jsonrpc_dumps_pretty(value, ledger=self.ledger))['result'] return json.loads(jsonrpc_dumps_pretty(value, ledger=self.ledger))['result']
async def confirm_and_render(self, awaitable, confirm) -> Transaction: async def confirm_and_render(self, awaitable, confirm, return_tx=False) -> Transaction:
tx = await awaitable tx = await awaitable
if confirm: if confirm:
await self.ledger.wait(tx) await self.ledger.wait(tx)
await self.generate(1) await self.generate(1)
await self.ledger.wait(tx, self.blockchain.block_expected) await self.ledger.wait(tx, self.blockchain.block_expected)
if not return_tx:
return self.sout(tx)
return tx
async def create_nondeterministic_channel(self, name, price, pubkey_bytes, daemon=None, blocking=False):
account = (daemon or self.daemon).wallet_manager.default_account
claim_address = await account.receiving.get_or_create_usable_address()
claim = Claim()
claim.channel.public_key_bytes = pubkey_bytes
tx = await Transaction.claim_create(
name, claim, lbc_to_dewies(price),
claim_address, [self.account], self.account
)
await tx.sign([self.account])
await (daemon or self.daemon).broadcast_or_release(tx, blocking)
return self.sout(tx) return self.sout(tx)
def create_upload_file(self, data, prefix=None, suffix=None): def create_upload_file(self, data, prefix=None, suffix=None):
@ -467,26 +573,26 @@ class CommandTestCase(IntegrationTestCase):
async def stream_create( async def stream_create(
self, name='hovercraft', bid='1.0', file_path=None, self, name='hovercraft', bid='1.0', file_path=None,
data=b'hi!', confirm=True, prefix=None, suffix=None, **kwargs): data=b'hi!', confirm=True, prefix=None, suffix=None, return_tx=False, **kwargs):
if file_path is None: if file_path is None and data is not None:
file_path = self.create_upload_file(data=data, prefix=prefix, suffix=suffix) file_path = self.create_upload_file(data=data, prefix=prefix, suffix=suffix)
return await self.confirm_and_render( return await self.confirm_and_render(
self.daemon.jsonrpc_stream_create(name, bid, file_path=file_path, **kwargs), confirm self.daemon.jsonrpc_stream_create(name, bid, file_path=file_path, **kwargs), confirm, return_tx
) )
async def stream_update( async def stream_update(
self, claim_id, data=None, prefix=None, suffix=None, confirm=True, **kwargs): self, claim_id, data=None, prefix=None, suffix=None, confirm=True, return_tx=False, **kwargs):
if data is not None: if data is not None:
file_path = self.create_upload_file(data=data, prefix=prefix, suffix=suffix) file_path = self.create_upload_file(data=data, prefix=prefix, suffix=suffix)
return await self.confirm_and_render( return await self.confirm_and_render(
self.daemon.jsonrpc_stream_update(claim_id, file_path=file_path, **kwargs), confirm self.daemon.jsonrpc_stream_update(claim_id, file_path=file_path, **kwargs), confirm, return_tx
) )
return await self.confirm_and_render( return await self.confirm_and_render(
self.daemon.jsonrpc_stream_update(claim_id, **kwargs), confirm self.daemon.jsonrpc_stream_update(claim_id, **kwargs), confirm
) )
def stream_repost(self, claim_id, name='repost', bid='1.0', confirm=True, **kwargs): async def stream_repost(self, claim_id, name='repost', bid='1.0', confirm=True, **kwargs):
return self.confirm_and_render( return await self.confirm_and_render(
self.daemon.jsonrpc_stream_repost(claim_id=claim_id, name=name, bid=bid, **kwargs), confirm self.daemon.jsonrpc_stream_repost(claim_id=claim_id, name=name, bid=bid, **kwargs), confirm
) )
@ -497,6 +603,11 @@ class CommandTestCase(IntegrationTestCase):
self.daemon.jsonrpc_stream_abandon(*args, **kwargs), confirm self.daemon.jsonrpc_stream_abandon(*args, **kwargs), confirm
) )
async def purchase_create(self, *args, confirm=True, **kwargs):
return await self.confirm_and_render(
self.daemon.jsonrpc_purchase_create(*args, **kwargs), confirm
)
async def publish(self, name, *args, confirm=True, **kwargs): async def publish(self, name, *args, confirm=True, **kwargs):
return await self.confirm_and_render( return await self.confirm_and_render(
self.daemon.jsonrpc_publish(name, *args, **kwargs), confirm self.daemon.jsonrpc_publish(name, *args, **kwargs), confirm
@ -543,15 +654,58 @@ class CommandTestCase(IntegrationTestCase):
self.daemon.jsonrpc_support_create(claim_id, bid, **kwargs), confirm self.daemon.jsonrpc_support_create(claim_id, bid, **kwargs), confirm
) )
async def resolve(self, uri): async def support_abandon(self, *args, confirm=True, **kwargs):
return await self.out(self.daemon.jsonrpc_resolve(uri)) if 'blocking' not in kwargs:
kwargs['blocking'] = False
return await self.confirm_and_render(
self.daemon.jsonrpc_support_abandon(*args, **kwargs), confirm
)
async def account_send(self, *args, confirm=True, **kwargs):
return await self.confirm_and_render(
self.daemon.jsonrpc_account_send(*args, **kwargs), confirm
)
async def wallet_send(self, *args, confirm=True, **kwargs):
return await self.confirm_and_render(
self.daemon.jsonrpc_wallet_send(*args, **kwargs), confirm
)
async def txo_spend(self, *args, confirm=True, **kwargs):
txs = await self.daemon.jsonrpc_txo_spend(*args, **kwargs)
if confirm:
await asyncio.wait([self.ledger.wait(tx) for tx in txs])
await self.generate(1)
await asyncio.wait([self.ledger.wait(tx, self.blockchain.block_expected) for tx in txs])
return self.sout(txs)
async def blob_clean(self):
return await self.out(self.daemon.jsonrpc_blob_clean())
async def status(self):
return await self.out(self.daemon.jsonrpc_status())
async def resolve(self, uri, **kwargs):
return (await self.out(self.daemon.jsonrpc_resolve(uri, **kwargs)))[uri]
async def claim_search(self, **kwargs): async def claim_search(self, **kwargs):
return (await self.out(self.daemon.jsonrpc_claim_search(**kwargs)))['items'] return (await self.out(self.daemon.jsonrpc_claim_search(**kwargs)))['items']
async def get_claim_by_claim_id(self, claim_id):
return await self.out(self.ledger.get_claim_by_claim_id(claim_id))
async def file_list(self, *args, **kwargs): async def file_list(self, *args, **kwargs):
return (await self.out(self.daemon.jsonrpc_file_list(*args, **kwargs)))['items'] return (await self.out(self.daemon.jsonrpc_file_list(*args, **kwargs)))['items']
async def txo_list(self, *args, **kwargs):
return (await self.out(self.daemon.jsonrpc_txo_list(*args, **kwargs)))['items']
async def txo_sum(self, *args, **kwargs):
return await self.out(self.daemon.jsonrpc_txo_sum(*args, **kwargs))
async def txo_plot(self, *args, **kwargs):
return await self.out(self.daemon.jsonrpc_txo_plot(*args, **kwargs))
async def claim_list(self, *args, **kwargs): async def claim_list(self, *args, **kwargs):
return (await self.out(self.daemon.jsonrpc_claim_list(*args, **kwargs)))['items'] return (await self.out(self.daemon.jsonrpc_claim_list(*args, **kwargs)))['items']
@ -561,9 +715,15 @@ class CommandTestCase(IntegrationTestCase):
async def channel_list(self, *args, **kwargs): async def channel_list(self, *args, **kwargs):
return (await self.out(self.daemon.jsonrpc_channel_list(*args, **kwargs)))['items'] return (await self.out(self.daemon.jsonrpc_channel_list(*args, **kwargs)))['items']
async def transaction_list(self, *args, **kwargs):
return (await self.out(self.daemon.jsonrpc_transaction_list(*args, **kwargs)))['items']
async def blob_list(self, *args, **kwargs):
return (await self.out(self.daemon.jsonrpc_blob_list(*args, **kwargs)))['items']
@staticmethod @staticmethod
def get_claim_id(tx): def get_claim_id(tx):
return tx['outputs'][0]['claim_id'] return tx['outputs'][0]['claim_id']
def assertItemCount(self, result, count): # pylint: disable=invalid-name def assertItemCount(self, result, count): # pylint: disable=invalid-name
self.assertEqual(result['total_items'], count) self.assertEqual(count, result['total_items'])

254
lbry/torrent/session.py Normal file
View file

@ -0,0 +1,254 @@
import asyncio
import binascii
import os
import logging
import random
from hashlib import sha1
from tempfile import mkdtemp
from typing import Optional
import libtorrent
log = logging.getLogger(__name__)
DEFAULT_FLAGS = ( # fixme: somehow the logic here is inverted?
libtorrent.add_torrent_params_flags_t.flag_auto_managed
| libtorrent.add_torrent_params_flags_t.flag_update_subscribe
)
class TorrentHandle:
def __init__(self, loop, executor, handle):
self._loop = loop
self._executor = executor
self._handle: libtorrent.torrent_handle = handle
self.started = asyncio.Event(loop=loop)
self.finished = asyncio.Event(loop=loop)
self.metadata_completed = asyncio.Event(loop=loop)
self.size = 0
self.total_wanted_done = 0
self.name = ''
self.tasks = []
self.torrent_file: Optional[libtorrent.file_storage] = None
self._base_path = None
self._handle.set_sequential_download(1)
@property
def largest_file(self) -> Optional[str]:
if not self.torrent_file:
return None
index = self.largest_file_index
return os.path.join(self._base_path, self.torrent_file.at(index).path)
@property
def largest_file_index(self):
largest_size, index = 0, 0
for file_num in range(self.torrent_file.num_files()):
if self.torrent_file.file_size(file_num) > largest_size:
largest_size = self.torrent_file.file_size(file_num)
index = file_num
return index
def stop_tasks(self):
while self.tasks:
self.tasks.pop().cancel()
def _show_status(self):
# fixme: cleanup
if not self._handle.is_valid():
return
status = self._handle.status()
if status.has_metadata:
self.size = status.total_wanted
self.total_wanted_done = status.total_wanted_done
self.name = status.name
if not self.metadata_completed.is_set():
self.metadata_completed.set()
log.info("Metadata completed for btih:%s - %s", status.info_hash, self.name)
self.torrent_file = self._handle.get_torrent_info().files()
self._base_path = status.save_path
first_piece = self.torrent_file.at(self.largest_file_index).offset
if not self.started.is_set():
if self._handle.have_piece(first_piece):
self.started.set()
else:
# prioritize it
self._handle.set_piece_deadline(first_piece, 100)
if not status.is_seeding:
log.debug('%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d seeds: %d) %s - %s',
status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000,
status.num_peers, status.num_seeds, status.state, status.save_path)
elif not self.finished.is_set():
self.finished.set()
log.info("Torrent finished: %s", self.name)
async def status_loop(self):
while True:
self._show_status()
if self.finished.is_set():
break
await asyncio.sleep(0.1)
async def pause(self):
await self._loop.run_in_executor(
self._executor, self._handle.pause
)
async def resume(self):
await self._loop.run_in_executor(
self._executor, lambda: self._handle.resume() # pylint: disable=unnecessary-lambda
)
class TorrentSession:
def __init__(self, loop, executor):
self._loop = loop
self._executor = executor
self._session: Optional[libtorrent.session] = None
self._handles = {}
self.tasks = []
self.wait_start = True
async def add_fake_torrent(self):
tmpdir = mkdtemp()
info, btih = _create_fake_torrent(tmpdir)
flags = libtorrent.add_torrent_params_flags_t.flag_seed_mode
handle = self._session.add_torrent({
'ti': info, 'save_path': tmpdir, 'flags': flags
})
self._handles[btih] = TorrentHandle(self._loop, self._executor, handle)
return btih
async def bind(self, interface: str = '0.0.0.0', port: int = 10889):
settings = {
'listen_interfaces': f"{interface}:{port}",
'enable_natpmp': False,
'enable_upnp': False
}
self._session = await self._loop.run_in_executor(
self._executor, libtorrent.session, settings # pylint: disable=c-extension-no-member
)
self.tasks.append(self._loop.create_task(self.process_alerts()))
def stop(self):
while self.tasks:
self.tasks.pop().cancel()
self._session.save_state()
self._session.pause()
self._session.stop_dht()
self._session.stop_lsd()
self._session.stop_natpmp()
self._session.stop_upnp()
self._session = None
def _pop_alerts(self):
for alert in self._session.pop_alerts():
log.info("torrent alert: %s", alert)
async def process_alerts(self):
while True:
await self._loop.run_in_executor(
self._executor, self._pop_alerts
)
await asyncio.sleep(1)
async def pause(self):
await self._loop.run_in_executor(
self._executor, lambda: self._session.save_state() # pylint: disable=unnecessary-lambda
)
await self._loop.run_in_executor(
self._executor, lambda: self._session.pause() # pylint: disable=unnecessary-lambda
)
async def resume(self):
await self._loop.run_in_executor(
self._executor, self._session.resume
)
def _add_torrent(self, btih: str, download_directory: Optional[str]):
params = {'info_hash': binascii.unhexlify(btih.encode()), 'flags': DEFAULT_FLAGS}
if download_directory:
params['save_path'] = download_directory
handle = self._session.add_torrent(params)
handle.force_dht_announce()
self._handles[btih] = TorrentHandle(self._loop, self._executor, handle)
def full_path(self, btih):
return self._handles[btih].largest_file
async def add_torrent(self, btih, download_path):
await self._loop.run_in_executor(
self._executor, self._add_torrent, btih, download_path
)
self._handles[btih].tasks.append(self._loop.create_task(self._handles[btih].status_loop()))
await self._handles[btih].metadata_completed.wait()
if self.wait_start:
# fixme: temporary until we add streaming support, otherwise playback fails!
await self._handles[btih].started.wait()
def remove_torrent(self, btih, remove_files=False):
if btih in self._handles:
handle = self._handles[btih]
handle.stop_tasks()
self._session.remove_torrent(handle._handle, 1 if remove_files else 0)
self._handles.pop(btih)
async def save_file(self, btih, download_directory):
handle = self._handles[btih]
await handle.resume()
def get_size(self, btih):
return self._handles[btih].size
def get_name(self, btih):
return self._handles[btih].name
def get_downloaded(self, btih):
return self._handles[btih].total_wanted_done
def is_completed(self, btih):
return self._handles[btih].finished.is_set()
def get_magnet_uri(btih):
return f"magnet:?xt=urn:btih:{btih}"
def _create_fake_torrent(tmpdir):
# beware, that's just for testing
path = os.path.join(tmpdir, 'tmp')
with open(path, 'wb') as myfile:
size = myfile.write(bytes([random.randint(0, 255) for _ in range(40)]) * 1024)
file_storage = libtorrent.file_storage()
file_storage.add_file('tmp', size)
t = libtorrent.create_torrent(file_storage, 0, 4 * 1024 * 1024)
libtorrent.set_piece_hashes(t, tmpdir)
info = libtorrent.torrent_info(t.generate())
btih = sha1(info.metadata()).hexdigest()
return info, btih
async def main():
if os.path.exists("~/Downloads/ubuntu-18.04.3-live-server-amd64.torrent"):
os.remove("~/Downloads/ubuntu-18.04.3-live-server-amd64.torrent")
if os.path.exists("~/Downloads/ubuntu-18.04.3-live-server-amd64.iso"):
os.remove("~/Downloads/ubuntu-18.04.3-live-server-amd64.iso")
btih = "dd8255ecdc7ca55fb0bbf81323d87062db1f6d1c"
executor = None
session = TorrentSession(asyncio.get_event_loop(), executor)
session2 = TorrentSession(asyncio.get_event_loop(), executor)
await session.bind('localhost', port=4040)
await session2.bind('localhost', port=4041)
btih = await session.add_fake_torrent()
session2._session.add_dht_node(('localhost', 4040))
await session2.add_torrent(btih, "/tmp/down")
while True:
await asyncio.sleep(100)
await session.pause()
executor.shutdown()
if __name__ == "__main__":
asyncio.run(main())

72
lbry/torrent/torrent.py Normal file
View file

@ -0,0 +1,72 @@
import asyncio
import logging
import typing
log = logging.getLogger(__name__)
class TorrentInfo:
__slots__ = ('dht_seeds', 'http_seeds', 'trackers', 'total_size')
def __init__(self, dht_seeds: typing.Tuple[typing.Tuple[str, int]],
http_seeds: typing.Tuple[typing.Dict[str, typing.Any]],
trackers: typing.Tuple[typing.Tuple[str, int]], total_size: int):
self.dht_seeds = dht_seeds
self.http_seeds = http_seeds
self.trackers = trackers
self.total_size = total_size
@classmethod
def from_libtorrent_info(cls, torrent_info):
return cls(
torrent_info.nodes(), tuple(
{
'url': web_seed['url'],
'type': web_seed['type'],
'auth': web_seed['auth']
} for web_seed in torrent_info.web_seeds()
), tuple(
(tracker.url, tracker.tier) for tracker in torrent_info.trackers()
), torrent_info.total_size()
)
class Torrent:
def __init__(self, loop, handle):
self._loop = loop
self._handle = handle
self.finished = asyncio.Event()
def _threaded_update_status(self):
status = self._handle.status()
if not status.is_seeding:
log.info(
'%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d) %s',
status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000,
status.num_peers, status.state
)
elif not self.finished.is_set():
self.finished.set()
async def wait_for_finished(self):
while True:
await self._loop.run_in_executor(
None, self._threaded_update_status
)
if self.finished.is_set():
log.info("finished downloading torrent!")
await self.pause()
break
await asyncio.sleep(1)
async def pause(self):
log.info("pause torrent")
await self._loop.run_in_executor(
None, self._handle.pause
)
async def resume(self):
await self._loop.run_in_executor(
None, self._handle.resume
)

View file

@ -0,0 +1,140 @@
import asyncio
import binascii
import logging
import os
import typing
from typing import Optional
from aiohttp.web import Request
from lbry.file.source_manager import SourceManager
from lbry.file.source import ManagedDownloadSource
if typing.TYPE_CHECKING:
from lbry.torrent.session import TorrentSession
from lbry.conf import Config
from lbry.wallet.transaction import Transaction
from lbry.extras.daemon.analytics import AnalyticsManager
from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim
from lbry.extras.daemon.storage import StoredContentClaim
log = logging.getLogger(__name__)
def path_or_none(encoded_path) -> Optional[str]:
if not encoded_path:
return
return binascii.unhexlify(encoded_path).decode()
class TorrentSource(ManagedDownloadSource):
STATUS_STOPPED = "stopped"
filter_fields = SourceManager.filter_fields
filter_fields.update({
'bt_infohash'
})
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage', identifier: str,
file_name: Optional[str] = None, download_directory: Optional[str] = None,
status: Optional[str] = STATUS_STOPPED, claim: Optional['StoredContentClaim'] = None,
download_id: Optional[str] = None, rowid: Optional[int] = None,
content_fee: Optional['Transaction'] = None,
analytics_manager: Optional['AnalyticsManager'] = None,
added_on: Optional[int] = None, torrent_session: Optional['TorrentSession'] = None):
super().__init__(loop, config, storage, identifier, file_name, download_directory, status, claim, download_id,
rowid, content_fee, analytics_manager, added_on)
self.torrent_session = torrent_session
@property
def full_path(self) -> Optional[str]:
full_path = self.torrent_session.full_path(self.identifier)
self.download_directory = os.path.dirname(full_path)
return full_path
async def start(self, timeout: Optional[float] = None, save_now: Optional[bool] = False):
await self.torrent_session.add_torrent(self.identifier, self.download_directory)
async def stop(self, finished: bool = False):
await self.torrent_session.remove_torrent(self.identifier)
async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
await self.torrent_session.save_file(self.identifier, download_directory)
@property
def torrent_length(self):
return self.torrent_session.get_size(self.identifier)
@property
def written_bytes(self):
return self.torrent_session.get_downloaded(self.identifier)
@property
def torrent_name(self):
return self.torrent_session.get_name(self.identifier)
@property
def bt_infohash(self):
return self.identifier
async def stop_tasks(self):
pass
@property
def completed(self):
return self.torrent_session.is_completed(self.identifier)
class TorrentManager(SourceManager):
_sources: typing.Dict[str, ManagedDownloadSource]
filter_fields = set(SourceManager.filter_fields)
filter_fields.update({
'bt_infohash',
'blobs_remaining', # TODO: here they call them "parts", but its pretty much the same concept
'blobs_in_stream'
})
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', torrent_session: 'TorrentSession',
storage: 'SQLiteStorage', analytics_manager: Optional['AnalyticsManager'] = None):
super().__init__(loop, config, storage, analytics_manager)
self.torrent_session: 'TorrentSession' = torrent_session
async def recover_streams(self, file_infos: typing.List[typing.Dict]):
raise NotImplementedError
async def _load_stream(self, rowid: int, bt_infohash: str, file_name: Optional[str],
download_directory: Optional[str], status: str,
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
added_on: Optional[int]):
stream = TorrentSource(
self.loop, self.config, self.storage, identifier=bt_infohash, file_name=file_name,
download_directory=download_directory, status=status, claim=claim, rowid=rowid,
content_fee=content_fee, analytics_manager=self.analytics_manager, added_on=added_on,
torrent_session=self.torrent_session
)
self.add(stream)
async def initialize_from_database(self):
pass
async def start(self):
await super().start()
async def stop(self):
await super().stop()
log.info("finished stopping the torrent manager")
async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
await super().delete(source, delete_file)
self.torrent_session.remove_torrent(source.identifier, delete_file)
async def create(self, file_path: str, key: Optional[bytes] = None,
iv_generator: Optional[typing.Generator[bytes, None, None]] = None):
raise NotImplementedError
async def _delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
raise NotImplementedError
# blob_hashes = [source.sd_hash] + [b.blob_hash for b in source.descriptor.blobs[:-1]]
# await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False)
# await self.storage.delete_stream(source.descriptor)
async def stream_partial_content(self, request: Request, sd_hash: str):
raise NotImplementedError

285
lbry/torrent/tracker.py Normal file
View file

@ -0,0 +1,285 @@
import random
import socket
import string
import struct
import asyncio
import logging
import time
import ipaddress
from collections import namedtuple
from functools import reduce
from typing import Optional
from lbry.dht.node import get_kademlia_peers_from_hosts
from lbry.utils import resolve_host, async_timed_cache, cache_concurrent
from lbry.wallet.stream import StreamController
from lbry import version
log = logging.getLogger(__name__)
CONNECTION_EXPIRES_AFTER_SECONDS = 50
PREFIX = 'LB' # todo: PR BEP20 to add ourselves
DEFAULT_TIMEOUT_SECONDS = 10.0
DEFAULT_CONCURRENCY_LIMIT = 100
# see: http://bittorrent.org/beps/bep_0015.html and http://xbtt.sourceforge.net/udp_tracker_protocol.html
ConnectRequest = namedtuple("ConnectRequest", ["connection_id", "action", "transaction_id"])
ConnectResponse = namedtuple("ConnectResponse", ["action", "transaction_id", "connection_id"])
AnnounceRequest = namedtuple("AnnounceRequest",
["connection_id", "action", "transaction_id", "info_hash", "peer_id", "downloaded", "left",
"uploaded", "event", "ip_addr", "key", "num_want", "port"])
AnnounceResponse = namedtuple("AnnounceResponse",
["action", "transaction_id", "interval", "leechers", "seeders", "peers"])
CompactIPv4Peer = namedtuple("CompactPeer", ["address", "port"])
ScrapeRequest = namedtuple("ScrapeRequest", ["connection_id", "action", "transaction_id", "infohashes"])
ScrapeResponse = namedtuple("ScrapeResponse", ["action", "transaction_id", "items"])
ScrapeResponseItem = namedtuple("ScrapeResponseItem", ["seeders", "completed", "leechers"])
ErrorResponse = namedtuple("ErrorResponse", ["action", "transaction_id", "message"])
structs = {
ConnectRequest: struct.Struct(">QII"),
ConnectResponse: struct.Struct(">IIQ"),
AnnounceRequest: struct.Struct(">QII20s20sQQQIIIiH"),
AnnounceResponse: struct.Struct(">IIIII"),
CompactIPv4Peer: struct.Struct(">IH"),
ScrapeRequest: struct.Struct(">QII"),
ScrapeResponse: struct.Struct(">II"),
ScrapeResponseItem: struct.Struct(">III"),
ErrorResponse: struct.Struct(">II")
}
def decode(cls, data, offset=0):
decoder = structs[cls]
if cls is AnnounceResponse:
return AnnounceResponse(*decoder.unpack_from(data, offset),
peers=[decode(CompactIPv4Peer, data, index) for index in range(20, len(data), 6)])
elif cls is ScrapeResponse:
return ScrapeResponse(*decoder.unpack_from(data, offset),
items=[decode(ScrapeResponseItem, data, index) for index in range(8, len(data), 12)])
elif cls is ErrorResponse:
return ErrorResponse(*decoder.unpack_from(data, offset), data[decoder.size:])
return cls(*decoder.unpack_from(data, offset))
def encode(obj):
if isinstance(obj, ScrapeRequest):
return structs[ScrapeRequest].pack(*obj[:-1]) + b''.join(obj.infohashes)
elif isinstance(obj, ErrorResponse):
return structs[ErrorResponse].pack(*obj[:-1]) + obj.message
elif isinstance(obj, AnnounceResponse):
return structs[AnnounceResponse].pack(*obj[:-1]) + b''.join([encode(peer) for peer in obj.peers])
return structs[type(obj)].pack(*obj)
def make_peer_id(random_part: Optional[str] = None) -> bytes:
# see https://wiki.theory.org/BitTorrentSpecification#peer_id and https://www.bittorrent.org/beps/bep_0020.html
# not to confuse with node id; peer id identifies uniquely the software, version and instance
random_part = random_part or ''.join(random.choice(string.ascii_letters) for _ in range(20))
return f"{PREFIX}-{'-'.join(map(str, version))}-{random_part}"[:20].encode()
class UDPTrackerClientProtocol(asyncio.DatagramProtocol):
def __init__(self, timeout: float = DEFAULT_TIMEOUT_SECONDS):
self.transport = None
self.data_queue = {}
self.timeout = timeout
self.semaphore = asyncio.Semaphore(DEFAULT_CONCURRENCY_LIMIT)
def connection_made(self, transport: asyncio.DatagramTransport) -> None:
self.transport = transport
async def request(self, obj, tracker_ip, tracker_port):
self.data_queue[obj.transaction_id] = asyncio.get_running_loop().create_future()
try:
async with self.semaphore:
self.transport.sendto(encode(obj), (tracker_ip, tracker_port))
return await asyncio.wait_for(self.data_queue[obj.transaction_id], self.timeout)
finally:
self.data_queue.pop(obj.transaction_id, None)
async def connect(self, tracker_ip, tracker_port):
transaction_id = random.getrandbits(32)
return decode(ConnectResponse,
await self.request(ConnectRequest(0x41727101980, 0, transaction_id), tracker_ip, tracker_port))
@cache_concurrent
@async_timed_cache(CONNECTION_EXPIRES_AFTER_SECONDS)
async def ensure_connection_id(self, peer_id, tracker_ip, tracker_port):
# peer_id is just to ensure cache coherency
return (await self.connect(tracker_ip, tracker_port)).connection_id
async def announce(self, info_hash, peer_id, port, tracker_ip, tracker_port, stopped=False):
connection_id = await self.ensure_connection_id(peer_id, tracker_ip, tracker_port)
# this should make the key deterministic but unique per info hash + peer id
key = int.from_bytes(info_hash[:4], "big") ^ int.from_bytes(peer_id[:4], "big") ^ port
transaction_id = random.getrandbits(32)
req = AnnounceRequest(
connection_id, 1, transaction_id, info_hash, peer_id, 0, 0, 0, 3 if stopped else 1, 0, key, -1, port)
return decode(AnnounceResponse, await self.request(req, tracker_ip, tracker_port))
async def scrape(self, infohashes, tracker_ip, tracker_port, connection_id=None):
connection_id = await self.ensure_connection_id(None, tracker_ip, tracker_port)
transaction_id = random.getrandbits(32)
reply = await self.request(
ScrapeRequest(connection_id, 2, transaction_id, infohashes), tracker_ip, tracker_port)
return decode(ScrapeResponse, reply), connection_id
def datagram_received(self, data: bytes, addr: (str, int)) -> None:
if len(data) < 8:
return
transaction_id = int.from_bytes(data[4:8], byteorder="big", signed=False)
if transaction_id in self.data_queue:
if not self.data_queue[transaction_id].done():
if data[3] == 3:
return self.data_queue[transaction_id].set_exception(Exception(decode(ErrorResponse, data).message))
return self.data_queue[transaction_id].set_result(data)
log.debug("unexpected packet (can be a response for a previously timed out request): %s", data.hex())
def connection_lost(self, exc: Exception = None) -> None:
self.transport = None
class TrackerClient:
event_controller = StreamController()
def __init__(self, node_id, announce_port, get_servers, timeout=10.0):
self.client = UDPTrackerClientProtocol(timeout=timeout)
self.transport = None
self.peer_id = make_peer_id(node_id.hex() if node_id else None)
self.announce_port = announce_port
self._get_servers = get_servers
self.results = {} # we can't probe the server before the interval, so we keep the result here until it expires
self.tasks = {}
async def start(self):
self.transport, _ = await asyncio.get_running_loop().create_datagram_endpoint(
lambda: self.client, local_addr=("0.0.0.0", 0))
self.event_controller.stream.listen(
lambda request: self.on_hash(request[1], request[2]) if request[0] == 'search' else None)
def stop(self):
while self.tasks:
self.tasks.popitem()[1].cancel()
if self.transport is not None:
self.transport.close()
self.client = None
self.transport = None
self.event_controller.close()
def on_hash(self, info_hash, on_announcement=None):
if info_hash not in self.tasks:
task = asyncio.create_task(self.get_peer_list(info_hash, on_announcement=on_announcement))
task.add_done_callback(lambda *_: self.tasks.pop(info_hash, None))
self.tasks[info_hash] = task
async def announce_many(self, *info_hashes, stopped=False):
await asyncio.gather(
*[self._announce_many(server, info_hashes, stopped=stopped) for server in self._get_servers()],
return_exceptions=True)
async def _announce_many(self, server, info_hashes, stopped=False):
tracker_ip = await resolve_host(*server, 'udp')
still_good_info_hashes = {
info_hash for (info_hash, (next_announcement, _)) in self.results.get(tracker_ip, {}).items()
if time.time() < next_announcement
}
results = await asyncio.gather(
*[self._probe_server(info_hash, tracker_ip, server[1], stopped=stopped)
for info_hash in info_hashes if info_hash not in still_good_info_hashes],
return_exceptions=True)
if results:
errors = sum([1 for result in results if result is None or isinstance(result, Exception)])
log.info("Tracker: finished announcing %d files to %s:%d, %d errors", len(results), *server, errors)
async def get_peer_list(self, info_hash, stopped=False, on_announcement=None, no_port=False):
found = []
probes = [self._probe_server(info_hash, *server, stopped, no_port) for server in self._get_servers()]
for done in asyncio.as_completed(probes):
result = await done
if result is not None:
await asyncio.gather(*filter(asyncio.iscoroutine, [on_announcement(result)] if on_announcement else []))
found.append(result)
return found
async def get_kademlia_peer_list(self, info_hash):
responses = await self.get_peer_list(info_hash, no_port=True)
return await announcement_to_kademlia_peers(*responses)
async def _probe_server(self, info_hash, tracker_host, tracker_port, stopped=False, no_port=False):
result = None
try:
tracker_host = await resolve_host(tracker_host, tracker_port, 'udp')
except socket.error:
log.warning("DNS failure while resolving tracker host: %s, skipping.", tracker_host)
return
self.results.setdefault(tracker_host, {})
if info_hash in self.results[tracker_host]:
next_announcement, result = self.results[tracker_host][info_hash]
if time.time() < next_announcement:
return result
try:
result = await self.client.announce(
info_hash, self.peer_id, 0 if no_port else self.announce_port, tracker_host, tracker_port, stopped)
self.results[tracker_host][info_hash] = (time.time() + result.interval, result)
except asyncio.TimeoutError: # todo: this is UDP, timeout is common, we need a better metric for failures
self.results[tracker_host][info_hash] = (time.time() + 60.0, result)
log.debug("Tracker timed out: %s:%d", tracker_host, tracker_port)
return None
log.debug("Announced: %s found %d peers for %s", tracker_host, len(result.peers), info_hash.hex()[:8])
return result
def enqueue_tracker_search(info_hash: bytes, peer_q: asyncio.Queue):
async def on_announcement(announcement: AnnounceResponse):
peers = await announcement_to_kademlia_peers(announcement)
log.info("Found %d peers from tracker for %s", len(peers), info_hash.hex()[:8])
peer_q.put_nowait(peers)
TrackerClient.event_controller.add(('search', info_hash, on_announcement))
def announcement_to_kademlia_peers(*announcements: AnnounceResponse):
peers = [
(str(ipaddress.ip_address(peer.address)), peer.port)
for announcement in announcements for peer in announcement.peers if peer.port > 1024 # no privileged or 0
]
return get_kademlia_peers_from_hosts(peers)
class UDPTrackerServerProtocol(asyncio.DatagramProtocol): # for testing. Not suitable for production
def __init__(self):
self.transport = None
self.known_conns = set()
self.peers = {}
def connection_made(self, transport: asyncio.DatagramTransport) -> None:
self.transport = transport
def add_peer(self, info_hash, ip_address: str, port: int):
self.peers.setdefault(info_hash, [])
self.peers[info_hash].append(encode_peer(ip_address, port))
def datagram_received(self, data: bytes, addr: (str, int)) -> None:
if len(data) < 16:
return
action = int.from_bytes(data[8:12], "big", signed=False)
if action == 0:
req = decode(ConnectRequest, data)
connection_id = random.getrandbits(32)
self.known_conns.add(connection_id)
return self.transport.sendto(encode(ConnectResponse(0, req.transaction_id, connection_id)), addr)
elif action == 1:
req = decode(AnnounceRequest, data)
if req.connection_id not in self.known_conns:
resp = encode(ErrorResponse(3, req.transaction_id, b'Connection ID missmatch.\x00'))
else:
compact_address = encode_peer(addr[0], req.port)
if req.event != 3:
self.add_peer(req.info_hash, addr[0], req.port)
elif compact_address in self.peers.get(req.info_hash, []):
self.peers[req.info_hash].remove(compact_address)
peers = [decode(CompactIPv4Peer, peer) for peer in self.peers[req.info_hash]]
resp = encode(AnnounceResponse(1, req.transaction_id, 1700, 0, len(peers), peers))
return self.transport.sendto(resp, addr)
def encode_peer(ip_address: str, port: int):
compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), ip_address.split('.'), bytearray())
return compact_ip + port.to_bytes(2, "big", signed=False)

View file

@ -3,7 +3,9 @@ import codecs
import datetime import datetime
import random import random
import socket import socket
import time
import string import string
import sys
import json import json
import typing import typing
import asyncio import asyncio
@ -18,8 +20,10 @@ import pkg_resources
import certifi import certifi
import aiohttp import aiohttp
from prometheus_client import Counter
from lbry.schema.claim import Claim from lbry.schema.claim import Claim
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -100,10 +104,6 @@ def check_connection(server="lbry.com", port=80, timeout=5) -> bool:
return False return False
async def async_check_connection(server="lbry.com", port=80, timeout=1) -> bool:
return await asyncio.get_event_loop().run_in_executor(None, check_connection, server, port, timeout)
def random_string(length=10, chars=string.ascii_lowercase): def random_string(length=10, chars=string.ascii_lowercase):
return ''.join([random.choice(chars) for _ in range(length)]) return ''.join([random.choice(chars) for _ in range(length)])
@ -130,21 +130,16 @@ def get_sd_hash(stream_info):
def json_dumps_pretty(obj, **kwargs): def json_dumps_pretty(obj, **kwargs):
return json.dumps(obj, sort_keys=True, indent=2, separators=(',', ': '), **kwargs) return json.dumps(obj, sort_keys=True, indent=2, separators=(',', ': '), **kwargs)
try:
def cancel_task(task: typing.Optional[asyncio.Task]): # the standard contextlib.aclosing() is available in 3.10+
if task and not task.done(): from contextlib import aclosing # pylint: disable=unused-import
task.cancel() except ImportError:
@contextlib.asynccontextmanager
async def aclosing(thing):
def cancel_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]): try:
for task in tasks: yield thing
cancel_task(task) finally:
await thing.aclose()
def drain_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
while tasks:
cancel_task(tasks.pop())
def async_timed_cache(duration: int): def async_timed_cache(duration: int):
def wrapper(func): def wrapper(func):
@ -155,7 +150,7 @@ def async_timed_cache(duration: int):
async def _inner(*args, **kwargs) -> typing.Any: async def _inner(*args, **kwargs) -> typing.Any:
loop = asyncio.get_running_loop() loop = asyncio.get_running_loop()
time_now = loop.time() time_now = loop.time()
key = tuple([args, tuple([tuple([k, kwargs[k]]) for k in kwargs])]) key = (args, tuple(kwargs.items()))
if key in cache and (time_now - cache[key][1] < duration): if key in cache and (time_now - cache[key][1] < duration):
return cache[key][0] return cache[key][0]
to_cache = await func(*args, **kwargs) to_cache = await func(*args, **kwargs)
@ -173,7 +168,7 @@ def cache_concurrent(async_fn):
@functools.wraps(async_fn) @functools.wraps(async_fn)
async def wrapper(*args, **kwargs): async def wrapper(*args, **kwargs):
key = tuple([args, tuple([tuple([k, kwargs[k]]) for k in kwargs])]) key = (args, tuple(kwargs.items()))
cache[key] = cache.get(key) or asyncio.create_task(async_fn(*args, **kwargs)) cache[key] = cache.get(key) or asyncio.create_task(async_fn(*args, **kwargs))
try: try:
return await cache[key] return await cache[key]
@ -187,6 +182,8 @@ def cache_concurrent(async_fn):
async def resolve_host(url: str, port: int, proto: str) -> str: async def resolve_host(url: str, port: int, proto: str) -> str:
if proto not in ['udp', 'tcp']: if proto not in ['udp', 'tcp']:
raise Exception("invalid protocol") raise Exception("invalid protocol")
if url.lower() == 'localhost':
return '127.0.0.1'
try: try:
if ipaddress.ip_address(url): if ipaddress.ip_address(url):
return url return url
@ -201,18 +198,44 @@ async def resolve_host(url: str, port: int, proto: str) -> str:
))[0][4][0] ))[0][4][0]
class LRUCache: class LRUCacheWithMetrics:
__slots__ = [ __slots__ = [
'capacity', 'capacity',
'cache' 'cache',
'_track_metrics',
'hits',
'misses'
] ]
def __init__(self, capacity): def __init__(self, capacity: int, metric_name: typing.Optional[str] = None, namespace: str = "daemon_cache"):
self.capacity = capacity self.capacity = capacity
self.cache = collections.OrderedDict() self.cache = collections.OrderedDict()
if metric_name is None:
self._track_metrics = False
self.hits = self.misses = None
else:
self._track_metrics = True
try:
self.hits = Counter(
f"{metric_name}_cache_hit_count", "Number of cache hits", namespace=namespace
)
self.misses = Counter(
f"{metric_name}_cache_miss_count", "Number of cache misses", namespace=namespace
)
except ValueError as err:
log.debug("failed to set up prometheus %s_cache_miss_count metric: %s", metric_name, err)
self._track_metrics = False
self.hits = self.misses = None
def get(self, key): def get(self, key, default=None):
try:
value = self.cache.pop(key) value = self.cache.pop(key)
if self._track_metrics:
self.hits.inc()
except KeyError:
if self._track_metrics:
self.misses.inc()
return default
self.cache[key] = value self.cache[key] = value
return value return value
@ -224,22 +247,97 @@ class LRUCache:
self.cache.popitem(last=False) self.cache.popitem(last=False)
self.cache[key] = value self.cache[key] = value
def clear(self):
self.cache.clear()
def pop(self, key):
return self.cache.pop(key)
def __setitem__(self, key, value):
return self.set(key, value)
def __getitem__(self, item):
return self.get(item)
def __contains__(self, item) -> bool: def __contains__(self, item) -> bool:
return item in self.cache return item in self.cache
def __len__(self):
return len(self.cache)
def __delitem__(self, key):
self.cache.pop(key)
def __del__(self):
self.clear()
class LRUCache:
__slots__ = [
'capacity',
'cache'
]
def __init__(self, capacity: int):
self.capacity = capacity
self.cache = collections.OrderedDict()
def get(self, key, default=None):
try:
value = self.cache.pop(key)
except KeyError:
return default
self.cache[key] = value
return value
def set(self, key, value):
try:
self.cache.pop(key)
except KeyError:
if len(self.cache) >= self.capacity:
self.cache.popitem(last=False)
self.cache[key] = value
def items(self):
return self.cache.items()
def clear(self):
self.cache.clear()
def pop(self, key, default=None):
return self.cache.pop(key, default)
def __setitem__(self, key, value):
return self.set(key, value)
def __getitem__(self, item):
return self.get(item)
def __contains__(self, item) -> bool:
return item in self.cache
def __len__(self):
return len(self.cache)
def __delitem__(self, key):
self.cache.pop(key)
def __del__(self):
self.clear()
def lru_cache_concurrent(cache_size: typing.Optional[int] = None, def lru_cache_concurrent(cache_size: typing.Optional[int] = None,
override_lru_cache: typing.Optional[LRUCache] = None): override_lru_cache: typing.Optional[LRUCacheWithMetrics] = None):
if not cache_size and override_lru_cache is None: if not cache_size and override_lru_cache is None:
raise ValueError("invalid cache size") raise ValueError("invalid cache size")
concurrent_cache = {} concurrent_cache = {}
lru_cache = override_lru_cache or LRUCache(cache_size) lru_cache = override_lru_cache if override_lru_cache is not None else LRUCacheWithMetrics(cache_size)
def wrapper(async_fn): def wrapper(async_fn):
@functools.wraps(async_fn) @functools.wraps(async_fn)
async def _inner(*args, **kwargs): async def _inner(*args, **kwargs):
key = tuple([args, tuple([tuple([k, kwargs[k]]) for k in kwargs])]) key = (args, tuple(kwargs.items()))
if key in lru_cache: if key in lru_cache:
return lru_cache.get(key) return lru_cache.get(key)
@ -264,15 +362,125 @@ def get_ssl_context() -> ssl.SSLContext:
@contextlib.asynccontextmanager @contextlib.asynccontextmanager
async def aiohttp_request(method, url, **kwargs) -> typing.AsyncContextManager[aiohttp.ClientResponse]: async def aiohttp_request(method, url, **kwargs) -> typing.AsyncContextManager[aiohttp.ClientResponse]:
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
async with session.request(method, url, ssl=get_ssl_context(), **kwargs) as response: async with session.request(method, url, **kwargs) as response:
yield response yield response
async def get_external_ip() -> typing.Optional[str]: # used if upnp is disabled or non-functioning # the ipaddress module does not show these subnets as reserved
CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
def is_valid_public_ipv4(address, allow_localhost: bool = False, allow_lan: bool = False):
try:
parsed_ip = ipaddress.ip_address(address)
if parsed_ip.is_loopback and allow_localhost:
return True
if allow_lan and parsed_ip.is_private:
return True
if any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local, parsed_ip.is_loopback,
parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private)):
return False
else:
return not any((CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
except (ipaddress.AddressValueError, ValueError):
return False
async def fallback_get_external_ip(): # used if spv servers can't be used for ip detection
try: try:
async with aiohttp_request("get", "https://api.lbry.com/ip") as resp: async with aiohttp_request("get", "https://api.lbry.com/ip") as resp:
response = await resp.json() response = await resp.json()
if response['success']: if response['success']:
return response['data']['ip'] return response['data']['ip'], None
except Exception: except Exception:
return return None, None
async def _get_external_ip(default_servers) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]:
# used if upnp is disabled or non-functioning
from lbry.wallet.udp import SPVStatusClientProtocol # pylint: disable=C0415
hostname_to_ip = {}
ip_to_hostnames = collections.defaultdict(list)
async def resolve_spv(server, port):
try:
server_addr = await resolve_host(server, port, 'udp')
hostname_to_ip[server] = (server_addr, port)
ip_to_hostnames[(server_addr, port)].append(server)
except Exception:
log.exception("error looking up dns for spv servers")
# accumulate the dns results
await asyncio.gather(*(resolve_spv(server, port) for (server, port) in default_servers))
loop = asyncio.get_event_loop()
pong_responses = asyncio.Queue()
connection = SPVStatusClientProtocol(pong_responses)
try:
await loop.create_datagram_endpoint(lambda: connection, ('0.0.0.0', 0))
# could raise OSError if it cant bind
randomized_servers = list(ip_to_hostnames.keys())
random.shuffle(randomized_servers)
for server in randomized_servers:
connection.ping(server)
try:
_, pong = await asyncio.wait_for(pong_responses.get(), 1)
if is_valid_public_ipv4(pong.ip_address):
return pong.ip_address, ip_to_hostnames[server][0]
except asyncio.TimeoutError:
pass
return None, None
finally:
connection.close()
async def get_external_ip(default_servers) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]:
ip_from_spv_servers = await _get_external_ip(default_servers)
if not ip_from_spv_servers[1]:
return await fallback_get_external_ip()
return ip_from_spv_servers
def is_running_from_bundle():
# see https://pyinstaller.readthedocs.io/en/stable/runtime-information.html
return getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS')
class LockWithMetrics(asyncio.Lock):
def __init__(self, acquire_metric, held_time_metric):
super().__init__()
self._acquire_metric = acquire_metric
self._lock_held_time_metric = held_time_metric
self._lock_acquired_time = None
async def acquire(self):
start = time.perf_counter()
try:
return await super().acquire()
finally:
self._lock_acquired_time = time.perf_counter()
self._acquire_metric.observe(self._lock_acquired_time - start)
def release(self):
try:
return super().release()
finally:
self._lock_held_time_metric.observe(time.perf_counter() - self._lock_acquired_time)
def get_colliding_prefix_bits(first_value: bytes, second_value: bytes):
"""
Calculates the amount of colliding prefix bits between <first_value> and <second_value>.
This is given by the amount of bits that are the same until the first different one (via XOR),
starting from the most significant bit to the least significant bit.
:param first_value: first value to compare, bigger than size.
:param second_value: second value to compare, bigger than size.
:return: amount of prefix colliding bits.
"""
assert len(first_value) == len(second_value), "length should be the same"
size = len(first_value) * 8
first_value, second_value = int.from_bytes(first_value, "big"), int.from_bytes(second_value, "big")
return size - (first_value ^ second_value).bit_length()

View file

@ -1,17 +1,23 @@
__node_daemon__ = 'lbrycrdd' __lbcd__ = 'lbcd'
__node_cli__ = 'lbrycrd-cli' __lbcctl__ = 'lbcctl'
__node_bin__ = '' __lbcwallet__ = 'lbcwallet'
__node_url__ = ( __lbcd_url__ = (
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.3.2/lbrycrd-linux-1732.zip' 'https://github.com/lbryio/lbcd/releases/download/' +
'v0.22.100-rc.0/lbcd_0.22.100-rc.0_TARGET_PLATFORM.tar.gz'
)
__lbcwallet_url__ = (
'https://github.com/lbryio/lbcwallet/releases/download/' +
'v0.13.100-alpha.0/lbcwallet_0.13.100-alpha.0_TARGET_PLATFORM.tar.gz'
) )
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest' __spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
from .wallet import Wallet, WalletStorage, TimestampedPreferences, ENCRYPT_ON_DISK from lbry.wallet.wallet import Wallet, WalletStorage, TimestampedPreferences, ENCRYPT_ON_DISK
from .manager import WalletManager from lbry.wallet.manager import WalletManager
from .network import Network from lbry.wallet.network import Network
from .ledger import Ledger, RegTestLedger, TestNetLedger, BlockHeightEvent from lbry.wallet.ledger import Ledger, RegTestLedger, TestNetLedger, BlockHeightEvent
from .account import Account, AddressManager, SingleKey, HierarchicalDeterministic from lbry.wallet.account import Account, AddressManager, SingleKey, HierarchicalDeterministic, \
from .transaction import Transaction, Output, Input DeterministicChannelKeyManager
from .script import OutputScript, InputScript from lbry.wallet.transaction import Transaction, Output, Input
from .database import SQLiteMixin, Database from lbry.wallet.script import OutputScript, InputScript
from .header import Headers from lbry.wallet.database import SQLiteMixin, Database
from lbry.wallet.header import Headers

View file

@ -5,18 +5,16 @@ import logging
import typing import typing
import asyncio import asyncio
import random import random
from functools import partial
from hashlib import sha256 from hashlib import sha256
from string import hexdigits from string import hexdigits
from typing import Type, Dict, Tuple, Optional, Any, List from typing import Type, Dict, Tuple, Optional, Any, List
import ecdsa
from lbry.error import InvalidPasswordError from lbry.error import InvalidPasswordError
from lbry.crypto.crypt import aes_encrypt, aes_decrypt from lbry.crypto.crypt import aes_encrypt, aes_decrypt
from .bip32 import PrivateKey, PubKey, from_extended_key_string from .bip32 import PrivateKey, PublicKey, KeyPath, from_extended_key_string
from .mnemonic import Mnemonic from .mnemonic import Mnemonic
from .constants import COIN, CLAIM_TYPES, TXO_TYPES from .constants import COIN, TXO_TYPES
from .transaction import Transaction, Input, Output from .transaction import Transaction, Input, Output
if typing.TYPE_CHECKING: if typing.TYPE_CHECKING:
@ -35,6 +33,49 @@ def validate_claim_id(claim_id):
raise Exception("Claim id is not hex encoded") raise Exception("Claim id is not hex encoded")
class DeterministicChannelKeyManager:
def __init__(self, account: 'Account'):
self.account = account
self.last_known = 0
self.cache = {}
self._private_key: Optional[PrivateKey] = None
@property
def private_key(self):
if self._private_key is None:
if self.account.private_key is not None:
self._private_key = self.account.private_key.child(KeyPath.CHANNEL)
return self._private_key
def maybe_generate_deterministic_key_for_channel(self, txo):
if self.private_key is None:
return
next_private_key = self.private_key.child(self.last_known)
public_key = next_private_key.public_key
public_key_bytes = public_key.pubkey_bytes
if txo.claim.channel.public_key_bytes == public_key_bytes:
self.cache[public_key.address] = next_private_key
self.last_known += 1
async def ensure_cache_primed(self):
if self.private_key is not None:
await self.generate_next_key()
async def generate_next_key(self) -> PrivateKey:
db = self.account.ledger.db
while True:
next_private_key = self.private_key.child(self.last_known)
public_key = next_private_key.public_key
self.cache[public_key.address] = next_private_key
if not await db.is_channel_key_used(self.account, public_key):
return next_private_key
self.last_known += 1
def get_private_key_from_pubkey_hash(self, pubkey_hash) -> PrivateKey:
return self.cache.get(pubkey_hash)
class AddressManager: class AddressManager:
name: str name: str
@ -71,6 +112,7 @@ class AddressManager:
def _query_addresses(self, **constraints): def _query_addresses(self, **constraints):
return self.account.ledger.db.get_addresses( return self.account.ledger.db.get_addresses(
read_only=constraints.pop("read_only", False),
accounts=[self.account], accounts=[self.account],
chain=self.chain_number, chain=self.chain_number,
**constraints **constraints
@ -79,7 +121,7 @@ class AddressManager:
def get_private_key(self, index: int) -> PrivateKey: def get_private_key(self, index: int) -> PrivateKey:
raise NotImplementedError raise NotImplementedError
def get_public_key(self, index: int) -> PubKey: def get_public_key(self, index: int) -> PublicKey:
raise NotImplementedError raise NotImplementedError
async def get_max_gap(self): async def get_max_gap(self):
@ -96,6 +138,7 @@ class AddressManager:
return [r['address'] for r in records] return [r['address'] for r in records]
async def get_or_create_usable_address(self) -> str: async def get_or_create_usable_address(self) -> str:
async with self.address_generator_lock:
addresses = await self.get_addresses(only_usable=True, limit=10) addresses = await self.get_addresses(only_usable=True, limit=10)
if addresses: if addresses:
return random.choice(addresses) return random.choice(addresses)
@ -118,8 +161,8 @@ class HierarchicalDeterministic(AddressManager):
@classmethod @classmethod
def from_dict(cls, account: 'Account', d: dict) -> Tuple[AddressManager, AddressManager]: def from_dict(cls, account: 'Account', d: dict) -> Tuple[AddressManager, AddressManager]:
return ( return (
cls(account, 0, **d.get('receiving', {'gap': 20, 'maximum_uses_per_address': 1})), cls(account, KeyPath.RECEIVE, **d.get('receiving', {'gap': 20, 'maximum_uses_per_address': 1})),
cls(account, 1, **d.get('change', {'gap': 6, 'maximum_uses_per_address': 1})) cls(account, KeyPath.CHANGE, **d.get('change', {'gap': 6, 'maximum_uses_per_address': 1}))
) )
def merge(self, d: dict): def merge(self, d: dict):
@ -132,7 +175,7 @@ class HierarchicalDeterministic(AddressManager):
def get_private_key(self, index: int) -> PrivateKey: def get_private_key(self, index: int) -> PrivateKey:
return self.account.private_key.child(self.chain_number).child(index) return self.account.private_key.child(self.chain_number).child(index)
def get_public_key(self, index: int) -> PubKey: def get_public_key(self, index: int) -> PublicKey:
return self.account.public_key.child(self.chain_number).child(index) return self.account.public_key.child(self.chain_number).child(index)
async def get_max_gap(self) -> int: async def get_max_gap(self) -> int:
@ -192,7 +235,7 @@ class SingleKey(AddressManager):
@classmethod @classmethod
def from_dict(cls, account: 'Account', d: dict) \ def from_dict(cls, account: 'Account', d: dict) \
-> Tuple[AddressManager, AddressManager]: -> Tuple[AddressManager, AddressManager]:
same_address_manager = cls(account, account.public_key, 0) same_address_manager = cls(account, account.public_key, KeyPath.RECEIVE)
return same_address_manager, same_address_manager return same_address_manager, same_address_manager
def to_dict_instance(self): def to_dict_instance(self):
@ -201,7 +244,7 @@ class SingleKey(AddressManager):
def get_private_key(self, index: int) -> PrivateKey: def get_private_key(self, index: int) -> PrivateKey:
return self.account.private_key return self.account.private_key
def get_public_key(self, index: int) -> PubKey: def get_public_key(self, index: int) -> PublicKey:
return self.account.public_key return self.account.public_key
async def get_max_gap(self) -> int: async def get_max_gap(self) -> int:
@ -223,9 +266,6 @@ class SingleKey(AddressManager):
class Account: class Account:
mnemonic_class = Mnemonic
private_key_class = PrivateKey
public_key_class = PubKey
address_generators: Dict[str, Type[AddressManager]] = { address_generators: Dict[str, Type[AddressManager]] = {
SingleKey.name: SingleKey, SingleKey.name: SingleKey,
HierarchicalDeterministic.name: HierarchicalDeterministic, HierarchicalDeterministic.name: HierarchicalDeterministic,
@ -233,7 +273,7 @@ class Account:
def __init__(self, ledger: 'Ledger', wallet: 'Wallet', name: str, def __init__(self, ledger: 'Ledger', wallet: 'Wallet', name: str,
seed: str, private_key_string: str, encrypted: bool, seed: str, private_key_string: str, encrypted: bool,
private_key: Optional[PrivateKey], public_key: PubKey, private_key: Optional[PrivateKey], public_key: PublicKey,
address_generator: dict, modified_on: float, channel_keys: dict) -> None: address_generator: dict, modified_on: float, channel_keys: dict) -> None:
self.ledger = ledger self.ledger = ledger
self.wallet = wallet self.wallet = wallet
@ -244,13 +284,14 @@ class Account:
self.private_key_string = private_key_string self.private_key_string = private_key_string
self.init_vectors: Dict[str, bytes] = {} self.init_vectors: Dict[str, bytes] = {}
self.encrypted = encrypted self.encrypted = encrypted
self.private_key = private_key self.private_key: Optional[PrivateKey] = private_key
self.public_key = public_key self.public_key: PublicKey = public_key
generator_name = address_generator.get('name', HierarchicalDeterministic.name) generator_name = address_generator.get('name', HierarchicalDeterministic.name)
self.address_generator = self.address_generators[generator_name] self.address_generator = self.address_generators[generator_name]
self.receiving, self.change = self.address_generator.from_dict(self, address_generator) self.receiving, self.change = self.address_generator.from_dict(self, address_generator)
self.address_managers = {am.chain_number: am for am in {self.receiving, self.change}} self.address_managers = {am.chain_number: am for am in (self.receiving, self.change)}
self.channel_keys = channel_keys self.channel_keys = channel_keys
self.deterministic_channel_keys = DeterministicChannelKeyManager(self)
ledger.add_account(self) ledger.add_account(self)
wallet.add_account(self) wallet.add_account(self)
@ -265,19 +306,19 @@ class Account:
name: str = None, address_generator: dict = None): name: str = None, address_generator: dict = None):
return cls.from_dict(ledger, wallet, { return cls.from_dict(ledger, wallet, {
'name': name, 'name': name,
'seed': cls.mnemonic_class().make_seed(), 'seed': Mnemonic().make_seed(),
'address_generator': address_generator or {} 'address_generator': address_generator or {}
}) })
@classmethod @classmethod
def get_private_key_from_seed(cls, ledger: 'Ledger', seed: str, password: str): def get_private_key_from_seed(cls, ledger: 'Ledger', seed: str, password: str):
return cls.private_key_class.from_seed( return PrivateKey.from_seed(
ledger, cls.mnemonic_class.mnemonic_to_seed(seed, password or 'lbryum') ledger, Mnemonic.mnemonic_to_seed(seed, password or 'lbryum')
) )
@classmethod @classmethod
def keys_from_dict(cls, ledger: 'Ledger', d: dict) \ def keys_from_dict(cls, ledger: 'Ledger', d: dict) \
-> Tuple[str, Optional[PrivateKey], PubKey]: -> Tuple[str, Optional[PrivateKey], PublicKey]:
seed = d.get('seed', '') seed = d.get('seed', '')
private_key_string = d.get('private_key', '') private_key_string = d.get('private_key', '')
private_key = None private_key = None
@ -310,7 +351,7 @@ class Account:
private_key=private_key, private_key=private_key,
public_key=public_key, public_key=public_key,
address_generator=d.get('address_generator', {}), address_generator=d.get('address_generator', {}),
modified_on=d.get('modified_on', time.time()), modified_on=int(d.get('modified_on', time.time())),
channel_keys=d.get('certificates', {}) channel_keys=d.get('certificates', {})
) )
@ -342,7 +383,7 @@ class Account:
def merge(self, d: dict): def merge(self, d: dict):
if d.get('modified_on', 0) > self.modified_on: if d.get('modified_on', 0) > self.modified_on:
self.name = d['name'] self.name = d['name']
self.modified_on = d.get('modified_on', time.time()) self.modified_on = int(d.get('modified_on', time.time()))
assert self.address_generator.name == d['address_generator']['name'] assert self.address_generator.name == d['address_generator']['name']
for chain_name in ('change', 'receiving'): for chain_name in ('change', 'receiving'):
if chain_name in d['address_generator']: if chain_name in d['address_generator']:
@ -434,9 +475,9 @@ class Account:
addresses.extend(new_addresses) addresses.extend(new_addresses)
return addresses return addresses
async def get_addresses(self, **constraints) -> List[str]: async def get_addresses(self, read_only=False, **constraints) -> List[str]:
rows = await self.ledger.db.select_addresses('address', accounts=[self], **constraints) rows = await self.ledger.db.select_addresses('address', read_only=read_only, accounts=[self], **constraints)
return [r[0] for r in rows] return [r['address'] for r in rows]
def get_address_records(self, **constraints): def get_address_records(self, **constraints):
return self.ledger.db.get_addresses(accounts=[self], **constraints) return self.ledger.db.get_addresses(accounts=[self], **constraints)
@ -448,16 +489,16 @@ class Account:
assert not self.encrypted, "Cannot get private key on encrypted wallet account." assert not self.encrypted, "Cannot get private key on encrypted wallet account."
return self.address_managers[chain].get_private_key(index) return self.address_managers[chain].get_private_key(index)
def get_public_key(self, chain: int, index: int) -> PubKey: def get_public_key(self, chain: int, index: int) -> PublicKey:
return self.address_managers[chain].get_public_key(index) return self.address_managers[chain].get_public_key(index)
def get_balance(self, confirmations: int = 0, include_claims=False, **constraints): def get_balance(self, confirmations=0, include_claims=False, read_only=False, **constraints):
if not include_claims: if not include_claims:
constraints.update({'txo_type__in': (0, TXO_TYPES['purchase'])}) constraints.update({'txo_type__in': (TXO_TYPES['other'], TXO_TYPES['purchase'])})
if confirmations > 0: if confirmations > 0:
height = self.ledger.headers.height - (confirmations-1) height = self.ledger.headers.height - (confirmations-1)
constraints.update({'height__lte': height, 'height__gt': 0}) constraints.update({'height__lte': height, 'height__gt': 0})
return self.ledger.db.get_balance(accounts=[self], **constraints) return self.ledger.db.get_balance(accounts=[self], read_only=read_only, **constraints)
async def get_max_gap(self): async def get_max_gap(self):
change_gap = await self.change.get_max_gap() change_gap = await self.change.get_max_gap()
@ -467,6 +508,12 @@ class Account:
'max_receiving_gap': receiving_gap, 'max_receiving_gap': receiving_gap,
} }
def get_txos(self, **constraints):
return self.ledger.get_txos(wallet=self.wallet, accounts=[self], **constraints)
def get_txo_count(self, **constraints):
return self.ledger.get_txo_count(wallet=self.wallet, accounts=[self], **constraints)
def get_utxos(self, **constraints): def get_utxos(self, **constraints):
return self.ledger.get_utxos(wallet=self.wallet, accounts=[self], **constraints) return self.ledger.get_utxos(wallet=self.wallet, accounts=[self], **constraints)
@ -513,16 +560,18 @@ class Account:
return tx return tx
def add_channel_private_key(self, private_key): async def generate_channel_private_key(self):
public_key_bytes = private_key.get_verifying_key().to_der() return await self.deterministic_channel_keys.generate_next_key()
channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes)
self.channel_keys[channel_pubkey_hash] = private_key.to_pem().decode()
def get_channel_private_key(self, public_key_bytes): def add_channel_private_key(self, private_key: PrivateKey):
self.channel_keys[private_key.address] = private_key.to_pem().decode()
async def get_channel_private_key(self, public_key_bytes) -> PrivateKey:
channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes) channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes)
private_key_pem = self.channel_keys.get(channel_pubkey_hash) private_key_pem = self.channel_keys.get(channel_pubkey_hash)
if private_key_pem: if private_key_pem:
return ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=sha256) return PrivateKey.from_pem(self.ledger, private_key_pem)
return self.deterministic_channel_keys.get_private_key_from_pubkey_hash(channel_pubkey_hash)
async def maybe_migrate_certificates(self): async def maybe_migrate_certificates(self):
if not self.channel_keys: if not self.channel_keys:
@ -531,11 +580,10 @@ class Account:
for private_key_pem in self.channel_keys.values(): for private_key_pem in self.channel_keys.values():
if not isinstance(private_key_pem, str): if not isinstance(private_key_pem, str):
continue continue
if "-----BEGIN EC PRIVATE KEY-----" not in private_key_pem: if not private_key_pem.startswith("-----BEGIN"):
continue continue
private_key = ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=sha256) private_key = PrivateKey.from_pem(self.ledger, private_key_pem)
public_key_der = private_key.get_verifying_key().to_der() channel_keys[private_key.address] = private_key_pem
channel_keys[self.ledger.public_key_to_address(public_key_der)] = private_key_pem
if self.channel_keys != channel_keys: if self.channel_keys != channel_keys:
self.channel_keys = channel_keys self.channel_keys = channel_keys
self.wallet.save() self.wallet.save()
@ -555,41 +603,24 @@ class Account:
if gap_changed: if gap_changed:
self.wallet.save() self.wallet.save()
async def get_detailed_balance(self, confirmations=0, reserved_subtotals=False): async def get_detailed_balance(self, confirmations=0, read_only=False):
tips_balance, supports_balance, claims_balance = 0, 0, 0 constraints = {}
get_total_balance = partial(self.get_balance, confirmations=confirmations, include_claims=True) if confirmations > 0:
total = await get_total_balance() height = self.ledger.headers.height - (confirmations-1)
if reserved_subtotals: constraints.update({'height__lte': height, 'height__gt': 0})
claims_balance = await get_total_balance(txo_type__in=CLAIM_TYPES) return await self.ledger.db.get_detailed_balance(
for amount, spent, from_me, to_me, height in await self.get_support_summary(): accounts=[self], read_only=read_only, **constraints
if confirmations > 0 and not 0 < height <= self.ledger.headers.height - (confirmations - 1):
continue
if not spent and to_me:
if from_me:
supports_balance += amount
else:
tips_balance += amount
reserved = claims_balance + supports_balance + tips_balance
else:
reserved = await self.get_balance(
confirmations=confirmations, include_claims=True, txo_type__gt=0
) )
return {
'total': total,
'available': total - reserved,
'reserved': reserved,
'reserved_subtotals': {
'claims': claims_balance,
'supports': supports_balance,
'tips': tips_balance
} if reserved_subtotals else None
}
def get_transaction_history(self, **constraints): def get_transaction_history(self, read_only=False, **constraints):
return self.ledger.get_transaction_history(wallet=self.wallet, accounts=[self], **constraints) return self.ledger.get_transaction_history(
read_only=read_only, wallet=self.wallet, accounts=[self], **constraints
)
def get_transaction_history_count(self, **constraints): def get_transaction_history_count(self, read_only=False, **constraints):
return self.ledger.get_transaction_history_count(wallet=self.wallet, accounts=[self], **constraints) return self.ledger.get_transaction_history_count(
read_only=read_only, wallet=self.wallet, accounts=[self], **constraints
)
def get_claims(self, **constraints): def get_claims(self, **constraints):
return self.ledger.get_claims(wallet=self.wallet, accounts=[self], **constraints) return self.ledger.get_claims(wallet=self.wallet, accounts=[self], **constraints)
@ -622,7 +653,7 @@ class Account:
return self.ledger.get_support_count(wallet=self.wallet, accounts=[self], **constraints) return self.ledger.get_support_count(wallet=self.wallet, accounts=[self], **constraints)
def get_support_summary(self): def get_support_summary(self):
return self.ledger.db.get_supports_summary(account_id=self.id) return self.ledger.db.get_supports_summary(wallet=self.wallet, accounts=[self])
async def release_all_outputs(self): async def release_all_outputs(self):
await self.ledger.db.release_all_outputs(self) await self.ledger.db.release_all_outputs(self)

View file

@ -1,10 +1,21 @@
from coincurve import PublicKey, PrivateKey as _PrivateKey from asn1crypto.keys import PrivateKeyInfo, ECPrivateKey
from coincurve import PublicKey as cPublicKey, PrivateKey as cPrivateKey
from coincurve.utils import (
pem_to_der, lib as libsecp256k1, ffi as libsecp256k1_ffi
)
from coincurve.ecdsa import CDATA_SIG_LENGTH
from lbry.crypto.hash import hmac_sha512, hash160, double_sha256 from lbry.crypto.hash import hmac_sha512, hash160, double_sha256
from lbry.crypto.base58 import Base58 from lbry.crypto.base58 import Base58
from .util import cachedproperty from .util import cachedproperty
class KeyPath:
RECEIVE = 0
CHANGE = 1
CHANNEL = 2
class DerivationError(Exception): class DerivationError(Exception):
""" Raised when an invalid derivation occurs. """ """ Raised when an invalid derivation occurs. """
@ -46,9 +57,11 @@ class _KeyBase:
if len(raw_serkey) != 33: if len(raw_serkey) != 33:
raise ValueError('raw_serkey must have length 33') raise ValueError('raw_serkey must have length 33')
return (ver_bytes + bytes((self.depth,)) return (
ver_bytes + bytes((self.depth,))
+ self.parent_fingerprint() + self.n.to_bytes(4, 'big') + self.parent_fingerprint() + self.n.to_bytes(4, 'big')
+ self.chain_code + raw_serkey) + self.chain_code + raw_serkey
)
def identifier(self): def identifier(self):
raise NotImplementedError raise NotImplementedError
@ -69,26 +82,30 @@ class _KeyBase:
return Base58.encode_check(self.extended_key()) return Base58.encode_check(self.extended_key())
class PubKey(_KeyBase): class PublicKey(_KeyBase):
""" A BIP32 public key. """ """ A BIP32 public key. """
def __init__(self, ledger, pubkey, chain_code, n, depth, parent=None): def __init__(self, ledger, pubkey, chain_code, n, depth, parent=None):
super().__init__(ledger, chain_code, n, depth, parent) super().__init__(ledger, chain_code, n, depth, parent)
if isinstance(pubkey, PublicKey): if isinstance(pubkey, cPublicKey):
self.verifying_key = pubkey self.verifying_key = pubkey
else: else:
self.verifying_key = self._verifying_key_from_pubkey(pubkey) self.verifying_key = self._verifying_key_from_pubkey(pubkey)
@classmethod
def from_compressed(cls, public_key_bytes, ledger=None) -> 'PublicKey':
return cls(ledger, public_key_bytes, bytes((0,)*32), 0, 0)
@classmethod @classmethod
def _verifying_key_from_pubkey(cls, pubkey): def _verifying_key_from_pubkey(cls, pubkey):
""" Converts a 33-byte compressed pubkey into an PublicKey object. """ """ Converts a 33-byte compressed pubkey into an coincurve.PublicKey object. """
if not isinstance(pubkey, (bytes, bytearray)): if not isinstance(pubkey, (bytes, bytearray)):
raise TypeError('pubkey must be raw bytes') raise TypeError('pubkey must be raw bytes')
if len(pubkey) != 33: if len(pubkey) != 33:
raise ValueError('pubkey must be 33 bytes') raise ValueError('pubkey must be 33 bytes')
if pubkey[0] not in (2, 3): if pubkey[0] not in (2, 3):
raise ValueError('invalid pubkey prefix byte') raise ValueError('invalid pubkey prefix byte')
return PublicKey(pubkey) return cPublicKey(pubkey)
@cachedproperty @cachedproperty
def pubkey_bytes(self): def pubkey_bytes(self):
@ -103,7 +120,7 @@ class PubKey(_KeyBase):
def ec_point(self): def ec_point(self):
return self.verifying_key.point() return self.verifying_key.point()
def child(self, n: int): def child(self, n: int) -> 'PublicKey':
""" Return the derived child extended pubkey at index N. """ """ Return the derived child extended pubkey at index N. """
if not 0 <= n < (1 << 31): if not 0 <= n < (1 << 31):
raise ValueError('invalid BIP32 public key child number') raise ValueError('invalid BIP32 public key child number')
@ -111,7 +128,7 @@ class PubKey(_KeyBase):
msg = self.pubkey_bytes + n.to_bytes(4, 'big') msg = self.pubkey_bytes + n.to_bytes(4, 'big')
L_b, R_b = self._hmac_sha512(msg) # pylint: disable=invalid-name L_b, R_b = self._hmac_sha512(msg) # pylint: disable=invalid-name
derived_key = self.verifying_key.add(L_b) derived_key = self.verifying_key.add(L_b)
return PubKey(self.ledger, derived_key, R_b, n, self.depth + 1, self) return PublicKey(self.ledger, derived_key, R_b, n, self.depth + 1, self)
def identifier(self): def identifier(self):
""" Return the key's identifier as 20 bytes. """ """ Return the key's identifier as 20 bytes. """
@ -124,6 +141,36 @@ class PubKey(_KeyBase):
self.pubkey_bytes self.pubkey_bytes
) )
def verify(self, signature, digest) -> bool:
""" Verify that a signature is valid for a 32 byte digest. """
if len(signature) != 64:
raise ValueError('Signature must be 64 bytes long.')
if len(digest) != 32:
raise ValueError('Digest must be 32 bytes long.')
key = self.verifying_key
raw_signature = libsecp256k1_ffi.new('secp256k1_ecdsa_signature *')
parsed = libsecp256k1.secp256k1_ecdsa_signature_parse_compact(
key.context.ctx, raw_signature, signature
)
assert parsed == 1
normalized_signature = libsecp256k1_ffi.new('secp256k1_ecdsa_signature *')
libsecp256k1.secp256k1_ecdsa_signature_normalize(
key.context.ctx, normalized_signature, raw_signature
)
verified = libsecp256k1.secp256k1_ecdsa_verify(
key.context.ctx, normalized_signature, digest, key.public_key
)
return bool(verified)
class PrivateKey(_KeyBase): class PrivateKey(_KeyBase):
"""A BIP32 private key.""" """A BIP32 private key."""
@ -132,7 +179,7 @@ class PrivateKey(_KeyBase):
def __init__(self, ledger, privkey, chain_code, n, depth, parent=None): def __init__(self, ledger, privkey, chain_code, n, depth, parent=None):
super().__init__(ledger, chain_code, n, depth, parent) super().__init__(ledger, chain_code, n, depth, parent)
if isinstance(privkey, _PrivateKey): if isinstance(privkey, cPrivateKey):
self.signing_key = privkey self.signing_key = privkey
else: else:
self.signing_key = self._signing_key_from_privkey(privkey) self.signing_key = self._signing_key_from_privkey(privkey)
@ -140,7 +187,7 @@ class PrivateKey(_KeyBase):
@classmethod @classmethod
def _signing_key_from_privkey(cls, private_key): def _signing_key_from_privkey(cls, private_key):
""" Converts a 32-byte private key into an coincurve.PrivateKey object. """ """ Converts a 32-byte private key into an coincurve.PrivateKey object. """
return _PrivateKey.from_int(PrivateKey._private_key_secret_exponent(private_key)) return cPrivateKey.from_int(PrivateKey._private_key_secret_exponent(private_key))
@classmethod @classmethod
def _private_key_secret_exponent(cls, private_key): def _private_key_secret_exponent(cls, private_key):
@ -152,24 +199,40 @@ class PrivateKey(_KeyBase):
return int.from_bytes(private_key, 'big') return int.from_bytes(private_key, 'big')
@classmethod @classmethod
def from_seed(cls, ledger, seed): def from_seed(cls, ledger, seed) -> 'PrivateKey':
# This hard-coded message string seems to be coin-independent... # This hard-coded message string seems to be coin-independent...
hmac = hmac_sha512(b'Bitcoin seed', seed) hmac = hmac_sha512(b'Bitcoin seed', seed)
privkey, chain_code = hmac[:32], hmac[32:] privkey, chain_code = hmac[:32], hmac[32:]
return cls(ledger, privkey, chain_code, 0, 0) return cls(ledger, privkey, chain_code, 0, 0)
@classmethod
def from_pem(cls, ledger, pem) -> 'PrivateKey':
der = pem_to_der(pem.encode())
try:
key_int = ECPrivateKey.load(der).native['private_key']
except ValueError:
key_int = PrivateKeyInfo.load(der).native['private_key']['private_key']
private_key = cPrivateKey.from_int(key_int)
return cls(ledger, private_key, bytes((0,)*32), 0, 0)
@classmethod
def from_bytes(cls, ledger, key_bytes) -> 'PrivateKey':
return cls(ledger, cPrivateKey(key_bytes), bytes((0,)*32), 0, 0)
@cachedproperty @cachedproperty
def private_key_bytes(self): def private_key_bytes(self):
""" Return the serialized private key (no leading zero byte). """ """ Return the serialized private key (no leading zero byte). """
return self.signing_key.secret return self.signing_key.secret
@cachedproperty @cachedproperty
def public_key(self): def public_key(self) -> PublicKey:
""" Return the corresponding extended public key. """ """ Return the corresponding extended public key. """
verifying_key = self.signing_key.public_key verifying_key = self.signing_key.public_key
parent_pubkey = self.parent.public_key if self.parent else None parent_pubkey = self.parent.public_key if self.parent else None
return PubKey(self.ledger, verifying_key, self.chain_code, self.n, self.depth, return PublicKey(
parent_pubkey) self.ledger, verifying_key, self.chain_code,
self.n, self.depth, parent_pubkey
)
def ec_point(self): def ec_point(self):
return self.public_key.ec_point() return self.public_key.ec_point()
@ -182,11 +245,12 @@ class PrivateKey(_KeyBase):
""" Return the private key encoded in Wallet Import Format. """ """ Return the private key encoded in Wallet Import Format. """
return self.ledger.private_key_to_wif(self.private_key_bytes) return self.ledger.private_key_to_wif(self.private_key_bytes)
@property
def address(self): def address(self):
""" The public key as a P2PKH address. """ """ The public key as a P2PKH address. """
return self.public_key.address return self.public_key.address
def child(self, n): def child(self, n) -> 'PrivateKey':
""" Return the derived child extended private key at index N.""" """ Return the derived child extended private key at index N."""
if not 0 <= n < (1 << 32): if not 0 <= n < (1 << 32):
raise ValueError('invalid BIP32 private key child number') raise ValueError('invalid BIP32 private key child number')
@ -205,6 +269,28 @@ class PrivateKey(_KeyBase):
""" Produce a signature for piece of data by double hashing it and signing the hash. """ """ Produce a signature for piece of data by double hashing it and signing the hash. """
return self.signing_key.sign(data, hasher=double_sha256) return self.signing_key.sign(data, hasher=double_sha256)
def sign_compact(self, digest):
""" Produce a compact signature. """
key = self.signing_key
signature = libsecp256k1_ffi.new('secp256k1_ecdsa_signature *')
signed = libsecp256k1.secp256k1_ecdsa_sign(
key.context.ctx, signature, digest, key.secret,
libsecp256k1_ffi.NULL, libsecp256k1_ffi.NULL
)
if not signed:
raise ValueError('The private key was invalid.')
serialized = libsecp256k1_ffi.new('unsigned char[%d]' % CDATA_SIG_LENGTH)
compacted = libsecp256k1.secp256k1_ecdsa_signature_serialize_compact(
key.context.ctx, serialized, signature
)
if compacted != 1:
raise ValueError('The signature could not be compacted.')
return bytes(libsecp256k1_ffi.buffer(serialized, CDATA_SIG_LENGTH))
def identifier(self): def identifier(self):
"""Return the key's identifier as 20 bytes.""" """Return the key's identifier as 20 bytes."""
return self.public_key.identifier() return self.public_key.identifier()
@ -216,9 +302,12 @@ class PrivateKey(_KeyBase):
b'\0' + self.private_key_bytes b'\0' + self.private_key_bytes
) )
def to_pem(self):
return self.signing_key.to_pem()
def _from_extended_key(ledger, ekey): def _from_extended_key(ledger, ekey):
"""Return a PubKey or PrivateKey from an extended key raw bytes.""" """Return a PublicKey or PrivateKey from an extended key raw bytes."""
if not isinstance(ekey, (bytes, bytearray)): if not isinstance(ekey, (bytes, bytearray)):
raise TypeError('extended key must be raw bytes') raise TypeError('extended key must be raw bytes')
if len(ekey) != 78: if len(ekey) != 78:
@ -230,7 +319,7 @@ def _from_extended_key(ledger, ekey):
if ekey[:4] == ledger.extended_public_key_prefix: if ekey[:4] == ledger.extended_public_key_prefix:
pubkey = ekey[45:] pubkey = ekey[45:]
key = PubKey(ledger, pubkey, chain_code, n, depth) key = PublicKey(ledger, pubkey, chain_code, n, depth)
elif ekey[:4] == ledger.extended_private_key_prefix: elif ekey[:4] == ledger.extended_private_key_prefix:
if ekey[45] != 0: if ekey[45] != 0:
raise ValueError('invalid extended private key prefix byte') raise ValueError('invalid extended private key prefix byte')
@ -248,6 +337,6 @@ def from_extended_key_string(ledger, ekey_str):
xpub6BsnM1W2Y7qLMiuhi7f7dbAwQZ5Cz5gYJCRzTNainXzQXYjFwtuQXHd xpub6BsnM1W2Y7qLMiuhi7f7dbAwQZ5Cz5gYJCRzTNainXzQXYjFwtuQXHd
3qfi3t3KJtHxshXezfjft93w4UE7BGMtKwhqEHae3ZA7d823DVrL 3qfi3t3KJtHxshXezfjft93w4UE7BGMtKwhqEHae3ZA7d823DVrL
return a PubKey or PrivateKey. return a PublicKey or PrivateKey.
""" """
return _from_extended_key(ledger, Base58.decode_check(ekey_str)) return _from_extended_key(ledger, Base58.decode_check(ekey_str))

1245
lbry/wallet/checkpoints.py Normal file

File diff suppressed because it is too large Load diff

View file

@ -5,7 +5,7 @@ from lbry.wallet.transaction import OutputEffectiveAmountEstimator
MAXIMUM_TRIES = 100000 MAXIMUM_TRIES = 100000
STRATEGIES = [] STRATEGIES = ['sqlite'] # sqlite coin chooser is in database.py
def strategy(method): def strategy(method):
@ -141,7 +141,7 @@ class CoinSelector:
_) -> List[OutputEffectiveAmountEstimator]: _) -> List[OutputEffectiveAmountEstimator]:
""" Accumulate UTXOs at random until there is enough to cover the target. """ """ Accumulate UTXOs at random until there is enough to cover the target. """
target = self.target + self.cost_of_change target = self.target + self.cost_of_change
self.random.shuffle(txos, self.random.random) self.random.shuffle(txos, random=self.random.random) # pylint: disable=deprecated-argument
selection = [] selection = []
amount = 0 amount = 0
for coin in txos: for coin in txos:

View file

@ -2,10 +2,12 @@ NULL_HASH32 = b'\x00'*32
CENT = 1000000 CENT = 1000000
COIN = 100*CENT COIN = 100*CENT
DUST = 1000
TIMEOUT = 30.0 TIMEOUT = 30.0
TXO_TYPES = { TXO_TYPES = {
"other": 0,
"stream": 1, "stream": 1,
"channel": 2, "channel": 2,
"support": 3, "support": 3,
@ -14,9 +16,13 @@ TXO_TYPES = {
"repost": 6, "repost": 6,
} }
CLAIM_TYPES = [ CLAIM_TYPE_NAMES = [
TXO_TYPES['stream'], 'stream',
TXO_TYPES['channel'], 'channel',
TXO_TYPES['collection'], 'collection',
TXO_TYPES['repost'], 'repost',
]
CLAIM_TYPES = [
TXO_TYPES[name] for name in CLAIM_TYPE_NAMES
] ]

File diff suppressed because it is too large Load diff

View file

@ -1,16 +1,18 @@
import base64
import os import os
import struct import struct
import asyncio import asyncio
import hashlib
import logging import logging
import zlib
from datetime import date
from io import BytesIO from io import BytesIO
from contextlib import asynccontextmanager from typing import Optional, Iterator, Tuple, Callable
from typing import Optional, Iterator, Tuple
from binascii import hexlify, unhexlify from binascii import hexlify, unhexlify
from lbry.crypto.hash import sha512, double_sha256, ripemd160 from lbry.crypto.hash import sha512, double_sha256, ripemd160
from lbry.wallet.util import ArithUint256 from lbry.wallet.util import ArithUint256, date_to_julian_day
from .checkpoints import HASHES
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -32,25 +34,50 @@ class Headers:
max_target = 0x0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff max_target = 0x0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
genesis_hash = b'9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463' genesis_hash = b'9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463'
target_timespan = 150 target_timespan = 150
checkpoint = (600_000, b'100b33ca3d0b86a48f0d6d6f30458a130ecb89d5affefe4afccb134d5a40f4c2') checkpoints = HASHES
first_block_timestamp = 1466646588 # block 1, as 0 is off by a lot
timestamp_average_offset = 160.6855883050695 # calculated at 733447
validate_difficulty: bool = True validate_difficulty: bool = True
def __init__(self, path) -> None: def __init__(self, path) -> None:
if path == ':memory:': self.io = None
self.io = BytesIO()
self.path = path self.path = path
self._size: Optional[int] = None self._size: Optional[int] = None
self.chunk_getter: Optional[Callable] = None
self.known_missing_checkpointed_chunks = set()
self.check_chunk_lock = asyncio.Lock()
async def open(self): async def open(self):
self.io = BytesIO()
if self.path != ':memory:': if self.path != ':memory:':
if not os.path.exists(self.path): def _readit():
self.io = open(self.path, 'w+b') if os.path.exists(self.path):
with open(self.path, 'r+b') as header_file:
self.io.seek(0)
self.io.write(header_file.read())
await asyncio.get_event_loop().run_in_executor(None, _readit)
bytes_size = self.io.seek(0, os.SEEK_END)
self._size = bytes_size // self.header_size
max_checkpointed_height = max(self.checkpoints.keys() or [-1]) + 1000
if bytes_size % self.header_size:
log.warning("Reader file size doesnt match header size. Repairing, might take a while.")
await self.repair()
else: else:
self.io = open(self.path, 'r+b') # try repairing any incomplete write on tip from previous runs (outside of checkpoints, that are ok)
await self.repair(start_height=max_checkpointed_height)
await self.ensure_checkpointed_size()
await self.get_all_missing_headers()
async def close(self): async def close(self):
if self.io is not None:
def _close():
flags = 'r+b' if os.path.exists(self.path) else 'w+b'
with open(self.path, flags) as header_file:
header_file.write(self.io.getbuffer())
await asyncio.get_event_loop().run_in_executor(None, _close)
self.io.close() self.io.close()
self.io = None
@staticmethod @staticmethod
def serialize(header): def serialize(header):
@ -97,23 +124,92 @@ class Headers:
return new_target return new_target
def __len__(self) -> int: def __len__(self) -> int:
if self._size is None:
self._size = self.io.seek(0, os.SEEK_END) // self.header_size
return self._size return self._size
def __bool__(self): def __bool__(self):
return True return True
def __getitem__(self, height) -> dict: async def get(self, height) -> dict:
if isinstance(height, slice): if isinstance(height, slice):
raise NotImplementedError("Slicing of header chain has not been implemented yet.") raise NotImplementedError("Slicing of header chain has not been implemented yet.")
try:
return self.deserialize(height, await self.get_raw_header(height))
except struct.error:
raise IndexError(f"failed to get {height}, at {len(self)}")
def estimated_timestamp(self, height, try_real_headers=True):
if height <= 0:
return
if try_real_headers and self.has_header(height):
offset = height * self.header_size
return struct.unpack('<I', self.io.getbuffer()[offset + 100: offset + 104])[0]
return int(self.first_block_timestamp + (height * self.timestamp_average_offset))
def estimated_julian_day(self, height):
return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height, False)))
async def get_raw_header(self, height) -> bytes:
if self.chunk_getter:
await self.ensure_chunk_at(height)
if not 0 <= height <= self.height: if not 0 <= height <= self.height:
raise IndexError(f"{height} is out of bounds, current height: {self.height}") raise IndexError(f"{height} is out of bounds, current height: {self.height}")
return self.deserialize(height, self.get_raw_header(height)) return self._read(height)
def get_raw_header(self, height) -> bytes: def _read(self, height, count=1):
self.io.seek(height * self.header_size, os.SEEK_SET) offset = height * self.header_size
return self.io.read(self.header_size) return bytes(self.io.getbuffer()[offset: offset + self.header_size * count])
def chunk_hash(self, start, count):
return self.hash_header(self._read(start, count)).decode()
async def ensure_checkpointed_size(self):
max_checkpointed_height = max(self.checkpoints.keys() or [-1])
if self.height < max_checkpointed_height:
self._write(max_checkpointed_height, bytes([0] * self.header_size * 1000))
async def ensure_chunk_at(self, height):
async with self.check_chunk_lock:
if self.has_header(height):
log.debug("has header %s", height)
return
return await self.fetch_chunk(height)
async def fetch_chunk(self, height):
log.info("on-demand fetching height %s", height)
start = (height // 1000) * 1000
headers = await self.chunk_getter(start) # pylint: disable=not-callable
chunk = (
zlib.decompress(base64.b64decode(headers['base64']), wbits=-15, bufsize=600_000)
)
chunk_hash = self.hash_header(chunk).decode()
if self.checkpoints.get(start) == chunk_hash:
self._write(start, chunk)
if start in self.known_missing_checkpointed_chunks:
self.known_missing_checkpointed_chunks.remove(start)
return
elif start not in self.checkpoints:
return # todo: fixme
raise Exception(
f"Checkpoint mismatch at height {start}. Expected {self.checkpoints[start]}, but got {chunk_hash} instead."
)
def has_header(self, height):
normalized_height = (height // 1000) * 1000
if normalized_height in self.checkpoints:
return normalized_height not in self.known_missing_checkpointed_chunks
empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d'
all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b'
return self.chunk_hash(height, 1) not in (empty, all_zeroes)
async def get_all_missing_headers(self):
# Heavy operation done in one optimized shot
for chunk_height, expected_hash in reversed(list(self.checkpoints.items())):
if chunk_height in self.known_missing_checkpointed_chunks:
continue
if self.chunk_hash(chunk_height, 1000) != expected_hash:
self.known_missing_checkpointed_chunks.add(chunk_height)
return self.known_missing_checkpointed_chunks
@property @property
def height(self) -> int: def height(self) -> int:
@ -123,9 +219,9 @@ class Headers:
def bytes_size(self): def bytes_size(self):
return len(self) * self.header_size return len(self) * self.header_size
def hash(self, height=None) -> bytes: async def hash(self, height=None) -> bytes:
return self.hash_header( return self.hash_header(
self.get_raw_header(height if height is not None else self.height) await self.get_raw_header(height if height is not None else self.height)
) )
@staticmethod @staticmethod
@ -134,44 +230,18 @@ class Headers:
return b'0' * 64 return b'0' * 64
return hexlify(double_sha256(header)[::-1]) return hexlify(double_sha256(header)[::-1])
@asynccontextmanager
async def checkpointed_connector(self):
buf = BytesIO()
try:
yield buf
finally:
await asyncio.sleep(0)
final_height = len(self) + buf.tell() // self.header_size
verifiable_bytes = (self.checkpoint[0] - len(self)) * self.header_size if self.checkpoint else 0
if verifiable_bytes > 0 and final_height >= self.checkpoint[0]:
buf.seek(0)
self.io.seek(0)
h = hashlib.sha256()
h.update(self.io.read())
h.update(buf.read(verifiable_bytes))
if h.hexdigest().encode() == self.checkpoint[1]:
buf.seek(0)
self._write(len(self), buf.read(verifiable_bytes))
remaining = buf.read()
buf.seek(0)
buf.write(remaining)
buf.truncate()
else:
log.warning("Checkpoint mismatch, connecting headers through slow method.")
if buf.tell() > 0:
await self.connect(len(self), buf.getvalue())
async def connect(self, start: int, headers: bytes) -> int: async def connect(self, start: int, headers: bytes) -> int:
added = 0 added = 0
bail = False bail = False
for height, chunk in self._iterate_chunks(start, headers): for height, chunk in self._iterate_chunks(start, headers):
try: try:
# validate_chunk() is CPU bound and reads previous chunks from file system # validate_chunk() is CPU bound and reads previous chunks from file system
self.validate_chunk(height, chunk) await self.validate_chunk(height, chunk)
except InvalidHeader as e: except InvalidHeader as e:
bail = True bail = True
chunk = chunk[:(height-e.height)*self.header_size] chunk = chunk[:(height-e.height)*self.header_size]
added += self._write(height, chunk) if chunk else 0 if chunk:
added += self._write(height, chunk)
if bail: if bail:
break break
return added return added
@ -179,20 +249,21 @@ class Headers:
def _write(self, height, verified_chunk): def _write(self, height, verified_chunk):
self.io.seek(height * self.header_size, os.SEEK_SET) self.io.seek(height * self.header_size, os.SEEK_SET)
written = self.io.write(verified_chunk) // self.header_size written = self.io.write(verified_chunk) // self.header_size
self.io.truncate() # self.io.truncate()
# .seek()/.write()/.truncate() might also .flush() when needed # .seek()/.write()/.truncate() might also .flush() when needed
# the goal here is mainly to ensure we're definitely flush()'ing # the goal here is mainly to ensure we're definitely flush()'ing
self.io.flush() self.io.flush()
self._size = self.io.tell() // self.header_size self._size = max(self._size or 0, self.io.tell() // self.header_size)
return written return written
def validate_chunk(self, height, chunk): async def validate_chunk(self, height, chunk):
previous_hash, previous_header, previous_previous_header = None, None, None previous_hash, previous_header, previous_previous_header = None, None, None
if height > 0: if height > 0:
previous_header = self[height-1] raw = await self.get_raw_header(height-1)
previous_hash = self.hash(height-1) previous_header = self.deserialize(height-1, raw)
previous_hash = self.hash_header(raw)
if height > 1: if height > 1:
previous_previous_header = self[height-2] previous_previous_header = await self.get(height-2)
chunk_target = self.get_next_chunk_target(height // 2016 - 1) chunk_target = self.get_next_chunk_target(height // 2016 - 1)
for current_hash, current_header in self._iterate_headers(height, chunk): for current_hash, current_header in self._iterate_headers(height, chunk):
block_target = self.get_next_block_target(chunk_target, previous_previous_header, previous_header) block_target = self.get_next_block_target(chunk_target, previous_previous_header, previous_header)
@ -231,28 +302,30 @@ class Headers:
height, f"insufficient proof of work: {proof_of_work.value} vs target {target.value}" height, f"insufficient proof of work: {proof_of_work.value} vs target {target.value}"
) )
async def repair(self): async def repair(self, start_height=0):
previous_header_hash = fail = None previous_header_hash = fail = None
batch_size = 36 batch_size = 36
for start_height in range(0, self.height, batch_size): for height in range(start_height, self.height, batch_size):
self.io.seek(self.header_size * start_height) headers = self._read(height, batch_size)
headers = self.io.read(self.header_size*batch_size)
if len(headers) % self.header_size != 0: if len(headers) % self.header_size != 0:
headers = headers[:(len(headers) // self.header_size) * self.header_size] headers = headers[:(len(headers) // self.header_size) * self.header_size]
for header_hash, header in self._iterate_headers(start_height, headers): for header_hash, header in self._iterate_headers(height, headers):
height = header['block_height'] height = header['block_height']
if height: if previous_header_hash:
if header['prev_block_hash'] != previous_header_hash: if header['prev_block_hash'] != previous_header_hash:
fail = True fail = True
else: elif height == 0:
if header_hash != self.genesis_hash: if header_hash != self.genesis_hash:
fail = True fail = True
else:
# for sanity and clarity, since it is the only way we can end up here
assert start_height > 0 and height == start_height
if fail: if fail:
log.warning("Header file corrupted at height %s, truncating it.", height - 1) log.warning("Header file corrupted at height %s, truncating it.", height - 1)
self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET) self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET)
self.io.truncate() self.io.truncate()
self.io.flush() self.io.flush()
self._size = None self._size = self.io.seek(0, os.SEEK_END) // self.header_size
return return
previous_header_hash = header_hash previous_header_hash = header_hash
@ -276,10 +349,6 @@ class Headers:
header = headers[start:end] header = headers[start:end]
yield self.hash_header(header), self.deserialize(height+idx, header) yield self.hash_header(header), self.deserialize(height+idx, header)
@property
def claim_trie_root(self):
return self[self.height]['claim_trie_root']
@staticmethod @staticmethod
def header_hash_to_pow_hash(header_hash: bytes): def header_hash_to_pow_hash(header_hash: bytes):
header_hash_bytes = unhexlify(header_hash)[::-1] header_hash_bytes = unhexlify(header_hash)[::-1]
@ -295,3 +364,4 @@ class UnvalidatedHeaders(Headers):
validate_difficulty = False validate_difficulty = False
max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
genesis_hash = b'6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556' genesis_hash = b'6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556'
checkpoints = {}

File diff suppressed because it is too large Load diff

View file

@ -3,20 +3,21 @@ import json
import typing import typing
import logging import logging
import asyncio import asyncio
from binascii import unhexlify from binascii import unhexlify
from decimal import Decimal from decimal import Decimal
from typing import List, Type, MutableSequence, MutableMapping, Optional from typing import List, Type, MutableSequence, MutableMapping, Optional
from lbry.error import KeyFeeAboveMaxAllowedError from lbry.error import KeyFeeAboveMaxAllowedError, WalletNotLoadedError
from lbry.conf import Config from lbry.conf import Config, NOT_SET
from .dewies import dewies_to_lbc from lbry.wallet.dewies import dewies_to_lbc
from .account import Account from lbry.wallet.account import Account
from .ledger import Ledger, LedgerRegistry from lbry.wallet.ledger import Ledger, LedgerRegistry
from .transaction import Transaction, Output from lbry.wallet.transaction import Transaction, Output
from .database import Database from lbry.wallet.database import Database
from .wallet import Wallet, WalletStorage, ENCRYPT_ON_DISK from lbry.wallet.wallet import Wallet, WalletStorage, ENCRYPT_ON_DISK
from .rpc.jsonrpc import CodeMessageError from lbry.wallet.rpc.jsonrpc import CodeMessageError
if typing.TYPE_CHECKING: if typing.TYPE_CHECKING:
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
@ -95,7 +96,7 @@ class WalletManager:
for wallet in self.wallets: for wallet in self.wallets:
if wallet.id == wallet_id: if wallet.id == wallet_id:
return wallet return wallet
raise ValueError(f"Couldn't find wallet: {wallet_id}.") raise WalletNotLoadedError(wallet_id)
@staticmethod @staticmethod
def get_balance(wallet): def get_balance(wallet):
@ -182,9 +183,17 @@ class WalletManager:
ledger_config = { ledger_config = {
'auto_connect': True, 'auto_connect': True,
'explicit_servers': [],
'hub_timeout': config.hub_timeout,
'default_servers': config.lbryum_servers, 'default_servers': config.lbryum_servers,
'known_hubs': config.known_hubs,
'jurisdiction': config.jurisdiction,
'concurrent_hub_requests': config.concurrent_hub_requests,
'data_path': config.wallet_dir, 'data_path': config.wallet_dir,
'tx_cache_size': config.transaction_cache_size
} }
if 'LBRY_FEE_PER_NAME_CHAR' in os.environ:
ledger_config['fee_per_name_char'] = int(os.environ.get('LBRY_FEE_PER_NAME_CHAR'))
wallets_directory = os.path.join(config.wallet_dir, 'wallets') wallets_directory = os.path.join(config.wallet_dir, 'wallets')
if not os.path.exists(wallets_directory): if not os.path.exists(wallets_directory):
@ -194,6 +203,10 @@ class WalletManager:
os.path.join(wallets_directory, 'default_wallet') os.path.join(wallets_directory, 'default_wallet')
) )
if Config.lbryum_servers.is_set_to_default(config):
with config.update_config() as c:
c.lbryum_servers = NOT_SET
manager = cls.from_config({ manager = cls.from_config({
'ledgers': {ledger_id: ledger_config}, 'ledgers': {ledger_id: ledger_config},
'wallets': [ 'wallets': [
@ -224,9 +237,16 @@ class WalletManager:
async def reset(self): async def reset(self):
self.ledger.config = { self.ledger.config = {
'auto_connect': True, 'auto_connect': True,
'default_servers': self.config.lbryum_servers, 'explicit_servers': [],
'default_servers': Config.lbryum_servers.default,
'known_hubs': self.config.known_hubs,
'jurisdiction': self.config.jurisdiction,
'hub_timeout': self.config.hub_timeout,
'concurrent_hub_requests': self.config.concurrent_hub_requests,
'data_path': self.config.wallet_dir, 'data_path': self.config.wallet_dir,
} }
if Config.lbryum_servers.is_set(self.config):
self.ledger.config['explicit_servers'] = self.config.lbryum_servers
await self.ledger.stop() await self.ledger.stop()
await self.ledger.start() await self.ledger.start()
@ -248,26 +268,28 @@ class WalletManager:
log.warning("Failed to migrate %s receiving addresses!", log.warning("Failed to migrate %s receiving addresses!",
len(set(receiving_addresses).difference(set(migrated_receiving)))) len(set(receiving_addresses).difference(set(migrated_receiving))))
def get_best_blockhash(self): async def get_best_blockhash(self):
if len(self.ledger.headers) <= 0: if len(self.ledger.headers) <= 0:
return self.ledger.genesis_hash return self.ledger.genesis_hash
return self.ledger.headers.hash(self.ledger.headers.height).decode() return (await self.ledger.headers.hash(self.ledger.headers.height)).decode()
def get_unused_address(self): def get_unused_address(self):
return self.default_account.receiving.get_or_create_usable_address() return self.default_account.receiving.get_or_create_usable_address()
async def get_transaction(self, txid): async def get_transaction(self, txid: str):
tx = await self.db.get_transaction(txid=txid) tx = await self.db.get_transaction(txid=txid)
if not tx: if tx:
return tx
try: try:
raw = await self.ledger.network.get_transaction(txid) raw, merkle = await self.ledger.network.get_transaction_and_merkle(txid)
height = await self.ledger.network.get_transaction_height(txid)
except CodeMessageError as e: except CodeMessageError as e:
if 'No such mempool or blockchain transaction.' in e.message: if 'No such mempool or blockchain transaction.' in e.message:
return {'success': False, 'code': 404, 'message': 'transaction not found'} return {'success': False, 'code': 404, 'message': 'transaction not found'}
return {'success': False, 'code': e.code, 'message': e.message} return {'success': False, 'code': e.code, 'message': e.message}
tx = Transaction(unhexlify(raw)) height = merkle.get('block_height')
await self.ledger.maybe_verify_transaction(tx, height) tx = Transaction(unhexlify(raw), height=height)
if height and height > 0:
await self.ledger.maybe_verify_transaction(tx, height, merkle)
return tx return tx
async def create_purchase_transaction( async def create_purchase_transaction(
@ -295,10 +317,4 @@ class WalletManager:
) )
async def broadcast_or_release(self, tx, blocking=False): async def broadcast_or_release(self, tx, blocking=False):
try: await self.ledger.broadcast_or_release(tx, blocking=blocking)
await self.ledger.broadcast(tx)
if blocking:
await self.ledger.wait(tx, timeout=None)
except:
await self.ledger.release_tx(tx)
raise

Some files were not shown because too many files have changed in this diff Show more