lbry-sdk/scripts/profile_get.py

# Script to download the sd_blobs of the front page
# Author: hackrush(hackrush@lbry.io)
import asyncio

from twisted.internet import asyncioreactor
asyncioreactor.install()

import keyring
import logging
import tempfile
import time
import treq
import typing
import shutil

from binascii import unhexlify
from twisted.internet import defer, reactor

from lbrynet import conf, log_support
from lbrynet.p2p.Peer import Peer
from lbrynet.p2p.BlobManager import DiskBlobManager
from lbrynet.p2p import SinglePeerDownloader
from lbrynet.extras.daemon.ComponentManager import ComponentManager
from lbrynet.extras.daemon.Components import DatabaseComponent
from lbrynet.extras.daemon.PeerFinder import DummyPeerFinder
from lbrynet.extras.daemon.storage import SQLiteStorage

log = logging.getLogger()

component_manager = None


def f2d(future):
    return defer.Deferred.fromFuture(asyncio.ensure_future(future))


class FakeAnalytics:
    @property
    def is_started(self):
        return True

    def send_server_startup_success(self):
        pass

    def send_server_startup(self):
        pass

    def shutdown(self):
        pass

    def send_upnp_setup_success_fail(self, success, status):
        pass


class TempDatabaseComponent(DatabaseComponent):
    @defer.inlineCallbacks
    def start(self):
        self.storage = SQLiteStorage(tempfile.tempdir)
        yield self.storage.setup()

    @defer.inlineCallbacks
    def stop(self):
        yield self.storage.stop()
        self.storage = None


class MultiplePeerFinder(DummyPeerFinder):
    def __init__(self, peer):
        super().__init__()
        # This is just a dud!! It has absolutely no use
        self.peer = peer

    @defer.inlineCallbacks
    def find_peers_for_blob(self, blob_hash, timeout=None, filter_self=False):
        dht = component_manager.get_component('dht')
        peers = yield dht.iterativeFindValue(unhexlify(blob_hash))
        peers = [Peer(host, port) for node_id, host, port in peers]
        return peers


# monkeypatching custom peer finder
SinglePeerDownloader.SinglePeerFinder = MultiplePeerFinder


@defer.inlineCallbacks
def download_blob_for_uri(blob_hash: typing.Text):
    tmp_blob_manager = DiskBlobManager(tempfile.tempdir, component_manager.get_component('database'))

    downloader = SinglePeerDownloader.SinglePeerDownloader()
    downloader.setup(component_manager.get_component('wallet'))

    peer = Peer(None, None)  # required for the log statements in SinglePeerDownloader
    result = yield downloader.download_blob_from_peer(peer, 180, blob_hash, tmp_blob_manager)
    tmp_blob_manager.stop()
    return result


async def get_sd_hash_from_uri(uri: typing.Text):
    wallet = component_manager.get_component("wallet")
    resolved = await wallet.resolve(uri)
    sd_hash = resolved[uri]["claim"]["value"]["stream"]["source"]["source"]
    print(sd_hash)
    return sd_hash


@defer.inlineCallbacks
def benchmark_performance(uris: list) -> dict:
    results = dict()
    for uri in uris:
        sd_hash = yield f2d(get_sd_hash_from_uri(uri))

        start = time.time()
        was_download_successful = yield download_blob_for_uri(sd_hash)
        end = time.time()

        if was_download_successful:
            results[uri] = end - start
        else:
            results[uri] = "Could not download"

        print(results[uri], uri, file=open({YOUR OUTPUT FILENAME HERE}, "a"))

    return results


def extract_uris(response):
    uris = list()
    for key in response:
        for value in response[key]:
            uris.append(value)

    return uris


@defer.inlineCallbacks
def get_frontpage_uris():
    kr = keyring.get_keyring()
    c = kr.get_preferred_collection()
    lbry_keyring = None
    for col in c.get_all_items():
        if col.get_label() == "LBRY/auth_token":
            lbry_keyring = col
    lbry_keyring = lbry_keyring.get_secret().decode("ascii")
    response = yield treq.get("https://api.lbry.io/file/list_homepage?auth_token={}".format(lbry_keyring))
    if response.code != 200:
        log.error("API returned non 200 code!!")
        reactor.callLater(0, reactor.stop)

    body = yield response.json()
    uris = extract_uris(body['data']['Uris'])
    return uris


@defer.inlineCallbacks
def main():
    global component_manager
    yield component_manager.setup()
    yield component_manager.get_component('dht')._join_deferred
    uris = yield get_frontpage_uris()
    # uris = [
    #     "linux-kernel-patch-replaces-naughty#e4fc36ac921970b3d15138781d697841dfb745f7",
    #     "linux-thursday-dec-1-2018-linux-y#062615ab5b974f5536b7e3d47731e7c74279ea23",
    #     "what"
    # ]
    results = yield benchmark_performance(uris)
    yield component_manager.stop()
    shutil.rmtree(tempfile.tempdir)

    max_len = len(max(uris, key=len))
    _sum = num = 0
    for result, value in results.items():
        if not isinstance(value, str):
            _sum += value
            num += 1
        print("{0:>{1:d}s}: {2}".format(result, max_len, value))

    avg = _sum / num
    print("Average for {0:d} downloaded files is: {1:f}".format(num, avg))
    could_not_download = len(uris) - num
    smiley = ":-)" if could_not_download == 0 else ":-("
    print("{0:d} files could not be downloaded {1:s}".format(could_not_download, smiley))

    reactor.callLater(0, reactor.stop)


if __name__ == "__main__":
    log_support.configure_console(level='INFO')
    log_support.configure_twisted()

    tempfile.tempdir = tempfile.mkdtemp()

    conf.initialize_settings()
    conf.settings.set('download_directory', tempfile.tempdir)
    conf.settings.set('lbryum_wallet_dir', {YOUR WALLET DIRECTORY HERE})
    conf.settings.set('data_dir', tempfile.tempdir)
    conf.settings.set('use_upnp', False)

    skip_components = ["blob_manager", "hash_announcer", "file_manager", "peer_protocol_server", "reflector",
                       "exchange_rate_manager", "rate_limiter", "payment_rate_manager"]
    component_manager = ComponentManager(
        analytics_manager=FakeAnalytics,
        skip_components=skip_components,
        database=TempDatabaseComponent
    )

    main()
    reactor.run()