Revert "TEMP: Try python 3.8."

This reverts commit 8def4d5177.

.github/workflows/main.yml

@@ -1,24 +1,24 @@
 name: ci
-on: ["push", "pull_request"]
+on: ["push", "pull_request", "workflow_dispatch"]
 
 jobs:
 
   lint:
     name: lint
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v1
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         with:
-          python-version: '3.7'
+          python-version: '3.9'
       - name: extract pip cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
           restore-keys: ${{ runner.os }}-pip-
       - run: pip install --user --upgrade pip wheel
-      - run: pip install -e .[torrent,lint]
+      - run: pip install -e .[lint]
       - run: make lint
 
   tests-unit:
@@ -26,31 +26,31 @@ jobs:
     strategy:
       matrix:
         os:
-          - ubuntu-latest
+          - ubuntu-20.04
           - macos-latest
           - windows-latest
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v1
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         with:
-          python-version: '3.7'
+          python-version: '3.9'
       - name: set pip cache dir
-        id: pip-cache
-        run: echo "::set-output name=dir::$(pip cache dir)"
+        shell: bash
+        run: echo "PIP_CACHE_DIR=$(pip cache dir)" >> $GITHUB_ENV
       - name: extract pip cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
-          path: ${{ steps.pip-cache.outputs.dir }}
+          path: ${{ env.PIP_CACHE_DIR }}
           key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
           restore-keys: ${{ runner.os }}-pip-
       - id: os-name
-        uses: ASzc/change-string-case-action@v1
+        uses: ASzc/change-string-case-action@v5
         with:
           string: ${{ runner.os }}
-      - run: pip install --user --upgrade pip wheel
+      - run: python -m pip install --user --upgrade pip wheel
       - if: startsWith(runner.os, 'linux')
-        run: pip install -e .[torrent,test]
+        run: pip install -e .[test]
       - if: startsWith(runner.os, 'linux')
         env:
           HOME: /tmp
@@ -72,7 +72,7 @@ jobs:
 
   tests-integration:
     name: "tests / integration"
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         test:
@@ -81,8 +81,6 @@ jobs:
           - claims
           - takeovers
           - transactions
-          - claims_legacy_search
-          - takeovers_legacy_search
           - other
     steps:
       - name: Configure sysctl limits
@@ -95,16 +93,16 @@ jobs:
         uses: elastic/elastic-github-actions/elasticsearch@master
         with:
           stack-version: 7.12.1
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v1
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         with:
-          python-version: '3.7'
+          python-version: '3.9'
       - if: matrix.test == 'other'
         run: |
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends ffmpeg
       - name: extract pip cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ./.tox
           key: tox-integration-${{ matrix.test }}-${{ hashFiles('setup.py') }}
@@ -125,7 +123,7 @@ jobs:
 
   coverage:
     needs: ["tests-unit", "tests-integration"]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - name: finalize coverage report submission
         env:
@@ -140,29 +138,29 @@ jobs:
     strategy:
       matrix:
         os:
-          - ubuntu-18.04
+          - ubuntu-20.04
           - macos-latest
           - windows-latest
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v1
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         with:
-          python-version: '3.7'
+          python-version: '3.9'
       - id: os-name
-        uses: ASzc/change-string-case-action@v1
+        uses: ASzc/change-string-case-action@v5
         with:
           string: ${{ runner.os }}
       - name: set pip cache dir
-        id: pip-cache
-        run: echo "::set-output name=dir::$(pip cache dir)"
+        shell: bash
+        run: echo "PIP_CACHE_DIR=$(pip cache dir)" >> $GITHUB_ENV
       - name: extract pip cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
-          path: ${{ steps.pip-cache.outputs.dir }}
+          path: ${{ env.PIP_CACHE_DIR }}
           key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
           restore-keys: ${{ runner.os }}-pip-
-      - run: pip install pyinstaller==4.4
+      - run: pip install pyinstaller==4.6
       - run: pip install -e .
       - if: startsWith(github.ref, 'refs/tags/v')
         run: python docker/set_build.py
@@ -177,7 +175,7 @@ jobs:
           pip install pywin32==301
           pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico --onefile --name lbrynet lbry/extras/cli.py
           dist/lbrynet.exe --version
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: lbrynet-${{ steps.os-name.outputs.lowercase }}
           path: dist/
@@ -186,7 +184,7 @@ jobs:
     name: "release"
     if: startsWith(github.ref, 'refs/tags/v')
     needs: ["build"]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v1
       - uses: actions/download-artifact@v2

.github/workflows/release.yml

@@ -7,7 +7,7 @@ on:
 jobs:
   release:
     name: "slack notification"
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - uses: LoveToKnow/slackify-markdown-action@v1.0.0
         id: markdown

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2015-2020 LBRY Inc
+Copyright (c) 2015-2022 LBRY Inc
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
 "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,

docker/Dockerfile.dht_node

@@ -2,6 +2,7 @@ FROM debian:10-slim
 
 ARG user=lbry
 ARG projects_dir=/home/$user
+ARG db_dir=/database
 
 ARG DOCKER_TAG
 ARG DOCKER_COMMIT=docker
@@ -27,12 +28,16 @@ RUN groupadd -g 999 $user && useradd -m -u 999 -g $user $user
 
 COPY . $projects_dir
 RUN chown -R $user:$user $projects_dir
+RUN mkdir -p $db_dir
+RUN chown -R $user:$user $db_dir
 
 USER $user
 WORKDIR $projects_dir
 
+RUN python3 -m pip install -U setuptools pip
 RUN make install
 RUN python3 docker/set_build.py
 RUN rm ~/.cache -rf
+VOLUME $db_dir
 ENTRYPOINT ["python3", "scripts/dht_node.py"]
 

docker/Dockerfile.wallet_server

@@ -15,6 +15,7 @@ RUN apt-get update && \
       build-essential \
       automake libtool \
       pkg-config \
+      libleveldb-dev \
       python3.7 \
       python3-dev \
       python3-pip \

docker/Dockerfile.wallet_server_deploy

@@ -1,48 +0,0 @@
-# FROM debian:10-slim
-FROM python:3.7.12-slim-buster
-
-ARG user=lbry
-ARG db_dir=/database
-ARG projects_dir=/home/$user
-
-ARG DOCKER_TAG
-ARG DOCKER_COMMIT=docker
-ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT
-
-RUN apt-get update && \
-    apt-get -y --no-install-recommends install \
-      wget \
-      tar unzip \
-      build-essential \
-      automake libtool \
-      pkg-config
-
-RUN pip install uvloop
-RUN groupadd -g 999 $user && useradd -m -u 999 -g $user $user
-RUN mkdir -p $db_dir
-RUN chown -R $user:$user $db_dir
-
-COPY . $projects_dir
-RUN chown -R $user:$user $projects_dir
-
-USER $user
-WORKDIR $projects_dir
-
-RUN make install
-RUN python3 docker/set_build.py
-RUN rm ~/.cache -rf
-
-# entry point
-ARG host=0.0.0.0
-ARG tcp_port=50001
-ARG daemon_url=http://lbry:lbry@192.99.151.178:9245/
-VOLUME $db_dir
-ENV TCP_PORT=$tcp_port
-ENV HOST=$host
-ENV DAEMON_URL=$daemon_url
-ENV DB_DIRECTORY=$db_dir
-ENV MAX_SESSIONS=1000000000
-ENV MAX_SEND=1000000000000000000
-ENV EVENT_LOOP_POLICY=uvloop
-COPY ./docker/wallet_server_entrypoint.sh /entrypoint.sh
-ENTRYPOINT ["/entrypoint.sh"]

docker/wallet_server_entrypoint.sh

@@ -20,14 +20,6 @@ if [[ -n "$SNAPSHOT_URL" ]] && [[ ! -f /database/lbry-leveldb ]]; then
   rm "$filename"
 fi
 
-if [ -z "$HUB_COMMAND" ]; then
-  echo "HUB_COMMAND env variable must be writer, reader, or es_sync"
-  exit 1
-fi
-
-case "$HUB_COMMAND" in
-  writer ) /home/lbry/.local/bin/lbry-hub-writer "$@" ;;
-  reader ) /home/lbry/.local/bin/lbry-hub-server "$@" ;;
-  es_sync ) /home/lbry/.local/bin/lbry-hub-elastic-sync ;;
-  * ) "HUB_COMMAND env variable must be writer, reader, or es_sync" && exit 1 ;;
-esac
+/home/lbry/.local/bin/lbry-hub-elastic-sync
+echo 'starting server'
+/home/lbry/.local/bin/lbry-hub "$@"

lbry/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.106.0"
+__version__ = "0.113.0"
 version = tuple(map(int, __version__.split('.')))  # pylint: disable=invalid-name

lbry/blob/blob_file.py

@@ -87,8 +87,8 @@ class AbstractBlob:
         self.blob_completed_callback = blob_completed_callback
         self.blob_directory = blob_directory
         self.writers: typing.Dict[typing.Tuple[typing.Optional[str], typing.Optional[int]], HashBlobWriter] = {}
-        self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
-        self.writing: asyncio.Event = asyncio.Event(loop=self.loop)
+        self.verified: asyncio.Event = asyncio.Event()
+        self.writing: asyncio.Event = asyncio.Event()
         self.readers: typing.List[typing.BinaryIO] = []
         self.added_on = added_on or time.time()
         self.is_mine = is_mine
@@ -201,7 +201,7 @@ class AbstractBlob:
         writer = blob.get_blob_writer()
         writer.write(blob_bytes)
         await blob.verified.wait()
-        return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash, added_on, is_mine)
+        return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), added_on, blob_hash, is_mine)
 
     def save_verified_blob(self, verified_bytes: bytes):
         if self.verified.is_set():
@@ -222,7 +222,7 @@ class AbstractBlob:
                         peer_port: typing.Optional[int] = None) -> HashBlobWriter:
         if (peer_address, peer_port) in self.writers and not self.writers[(peer_address, peer_port)].closed():
             raise OSError(f"attempted to download blob twice from {peer_address}:{peer_port}")
-        fut = asyncio.Future(loop=self.loop)
+        fut = asyncio.Future()
         writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
         self.writers[(peer_address, peer_port)] = writer
 

lbry/blob/blob_info.py

@@ -12,8 +12,8 @@ class BlobInfo:
     ]
 
     def __init__(
-            self, blob_num: int, length: int, iv: str,
-             blob_hash: typing.Optional[str] = None, added_on=0, is_mine=False):
+            self, blob_num: int, length: int, iv: str, added_on,
+             blob_hash: typing.Optional[str] = None, is_mine=False):
         self.blob_hash = blob_hash
         self.blob_num = blob_num
         self.length = length

lbry/blob/blob_manager.py

@@ -83,6 +83,8 @@ class BlobManager:
         to_add = await self.storage.sync_missing_blobs(in_blobfiles_dir)
         if to_add:
             self.completed_blob_hashes.update(to_add)
+        # check blobs that aren't set as finished but were seen on disk
+        await self.ensure_completed_blobs_status(in_blobfiles_dir - to_add)
         if self.config.track_bandwidth:
             self.connection_manager.start()
         return True
@@ -113,9 +115,18 @@ class BlobManager:
                 (blob.blob_hash, blob.length, blob.added_on, blob.is_mine), finished=False)
             )
 
-    def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]:
-        """Returns of the blobhashes_to_check, which are valid"""
-        return [blob_hash for blob_hash in blob_hashes if self.is_blob_verified(blob_hash)]
+    async def ensure_completed_blobs_status(self, blob_hashes: typing.Iterable[str]):
+        """Ensures that completed blobs from a given list of blob hashes are set as 'finished' in the database."""
+        to_add = []
+        for blob_hash in blob_hashes:
+            if not self.is_blob_verified(blob_hash):
+                continue
+            blob = self.get_blob(blob_hash)
+            to_add.append((blob.blob_hash, blob.length, blob.added_on, blob.is_mine))
+            if len(to_add) > 500:
+                await self.storage.add_blobs(*to_add, finished=True)
+                to_add.clear()
+        return await self.storage.add_blobs(*to_add, finished=True)
 
     def delete_blob(self, blob_hash: str):
         if not is_valid_blobhash(blob_hash):

lbry/blob_exchange/client.py

@@ -32,7 +32,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
         self.buf = b''
 
         # this is here to handle the race when the downloader is closed right as response_fut gets a result
-        self.closed = asyncio.Event(loop=self.loop)
+        self.closed = asyncio.Event()
 
     def data_received(self, data: bytes):
         if self.connection_manager:
@@ -111,7 +111,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
             self.transport.write(msg)
             if self.connection_manager:
                 self.connection_manager.sent_data(f"{self.peer_address}:{self.peer_port}", len(msg))
-            response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop)
+            response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout)
             availability_response = response.get_availability_response()
             price_response = response.get_price_response()
             blob_response = response.get_blob_response()
@@ -151,7 +151,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
                 f" timeout in {self.peer_timeout}"
             log.debug(msg)
             msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
-            await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
+            await asyncio.wait_for(self.writer.finished, self.peer_timeout)
             # wait for the io to finish
             await self.blob.verified.wait()
             log.info("%s at %fMB/s", msg,
@@ -187,7 +187,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
         try:
             self._blob_bytes_received = 0
             self.blob, self.writer = blob, blob.get_blob_writer(self.peer_address, self.peer_port)
-            self._response_fut = asyncio.Future(loop=self.loop)
+            self._response_fut = asyncio.Future()
             return await self._download_blob()
         except OSError:
             # i'm not sure how to fix this race condition - jack
@@ -244,7 +244,7 @@ async def request_blob(loop: asyncio.AbstractEventLoop, blob: Optional['Abstract
     try:
         if not connected_protocol:
             await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
-                                   peer_connect_timeout, loop=loop)
+                                   peer_connect_timeout)
             connected_protocol = protocol
         if blob is None or blob.get_is_verified() or not blob.is_writeable():
             # blob is None happens when we are just opening a connection

lbry/blob_exchange/downloader.py

@@ -30,7 +30,7 @@ class BlobDownloader:
         self.failures: typing.Dict['KademliaPeer', int] = {}
         self.connection_failures: typing.Set['KademliaPeer'] = set()
         self.connections: typing.Dict['KademliaPeer', 'BlobExchangeClientProtocol'] = {}
-        self.is_running = asyncio.Event(loop=self.loop)
+        self.is_running = asyncio.Event()
 
     def should_race_continue(self, blob: 'AbstractBlob'):
         max_probes = self.config.max_connections_per_download * (1 if self.connections else 10)
@@ -64,8 +64,8 @@ class BlobDownloader:
             self.scores[peer] = bytes_received / elapsed if bytes_received and elapsed else 1
 
     async def new_peer_or_finished(self):
-        active_tasks = list(self.active_connections.values()) + [asyncio.sleep(1)]
-        await asyncio.wait(active_tasks, loop=self.loop, return_when='FIRST_COMPLETED')
+        active_tasks = list(self.active_connections.values()) + [asyncio.create_task(asyncio.sleep(1))]
+        await asyncio.wait(active_tasks, return_when='FIRST_COMPLETED')
 
     def cleanup_active(self):
         if not self.active_connections and not self.connections:
@@ -126,7 +126,7 @@ class BlobDownloader:
 
 async def download_blob(loop, config: 'Config', blob_manager: 'BlobManager', dht_node: 'Node',
                         blob_hash: str) -> 'AbstractBlob':
-    search_queue = asyncio.Queue(loop=loop, maxsize=config.max_connections_per_download)
+    search_queue = asyncio.Queue(maxsize=config.max_connections_per_download)
     search_queue.put_nowait(blob_hash)
     peer_queue, accumulate_task = dht_node.accumulate_peers(search_queue)
     fixed_peers = None if not config.fixed_peers else await get_kademlia_peers_from_hosts(config.fixed_peers)

lbry/blob_exchange/server.py

@@ -1,6 +1,7 @@
 import asyncio
 import binascii
 import logging
+import socket
 import typing
 from json.decoder import JSONDecodeError
 from lbry.blob_exchange.serialization import BlobResponse, BlobRequest, blob_response_types
@@ -24,19 +25,19 @@ class BlobServerProtocol(asyncio.Protocol):
         self.idle_timeout = idle_timeout
         self.transfer_timeout = transfer_timeout
         self.server_task: typing.Optional[asyncio.Task] = None
-        self.started_listening = asyncio.Event(loop=self.loop)
+        self.started_listening = asyncio.Event()
         self.buf = b''
         self.transport: typing.Optional[asyncio.Transport] = None
         self.lbrycrd_address = lbrycrd_address
         self.peer_address_and_port: typing.Optional[str] = None
-        self.started_transfer = asyncio.Event(loop=self.loop)
-        self.transfer_finished = asyncio.Event(loop=self.loop)
+        self.started_transfer = asyncio.Event()
+        self.transfer_finished = asyncio.Event()
         self.close_on_idle_task: typing.Optional[asyncio.Task] = None
 
     async def close_on_idle(self):
         while self.transport:
             try:
-                await asyncio.wait_for(self.started_transfer.wait(), self.idle_timeout, loop=self.loop)
+                await asyncio.wait_for(self.started_transfer.wait(), self.idle_timeout)
             except asyncio.TimeoutError:
                 log.debug("closing idle connection from %s", self.peer_address_and_port)
                 return self.close()
@@ -100,7 +101,7 @@ class BlobServerProtocol(asyncio.Protocol):
                 log.debug("send %s to %s:%i", blob_hash, peer_address, peer_port)
                 self.started_transfer.set()
                 try:
-                    sent = await asyncio.wait_for(blob.sendfile(self), self.transfer_timeout, loop=self.loop)
+                    sent = await asyncio.wait_for(blob.sendfile(self), self.transfer_timeout)
                     if sent and sent > 0:
                         self.blob_manager.connection_manager.sent_data(self.peer_address_and_port, sent)
                         log.info("sent %s (%i bytes) to %s:%i", blob_hash, sent, peer_address, peer_port)
@@ -137,7 +138,7 @@ class BlobServerProtocol(asyncio.Protocol):
             try:
                 request = BlobRequest.deserialize(self.buf + data)
                 self.buf = remainder
-            except JSONDecodeError:
+            except (UnicodeDecodeError, JSONDecodeError):
                 log.error("request from %s is not valid json (%i bytes): %s", self.peer_address_and_port,
                           len(self.buf + data), '' if not data else binascii.hexlify(self.buf + data).decode())
                 self.close()
@@ -156,7 +157,7 @@ class BlobServer:
         self.loop = loop
         self.blob_manager = blob_manager
         self.server_task: typing.Optional[asyncio.Task] = None
-        self.started_listening = asyncio.Event(loop=self.loop)
+        self.started_listening = asyncio.Event()
         self.lbrycrd_address = lbrycrd_address
         self.idle_timeout = idle_timeout
         self.transfer_timeout = transfer_timeout
@@ -167,6 +168,13 @@ class BlobServer:
             raise Exception("already running")
 
         async def _start_server():
+            # checking if the port is in use
+            # thx https://stackoverflow.com/a/52872579
+            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+                if s.connect_ex(('localhost', port)) == 0:
+                    # the port is already in use!
+                    log.error("Failed to bind TCP %s:%d", interface, port)
+
             server = await self.loop.create_server(
                 lambda: self.server_protocol_class(self.loop, self.blob_manager, self.lbrycrd_address,
                                                    self.idle_timeout, self.transfer_timeout),

lbry/conf.py

@@ -622,7 +622,11 @@ class Config(CLIConfig):
         "Routing table bucket index below which we always split the bucket if given a new key to add to it and "
         "the bucket is full. As this value is raised the depth of the routing table (and number of peers in it) "
         "will increase. This setting is used by seed nodes, you probably don't want to change it during normal "
-        "use.", 1
+        "use.", 2
+    )
+    is_bootstrap_node = Toggle(
+        "When running as a bootstrap node, disable all logic related to balancing the routing table, so we can "
+        "add as many peers as possible and better help first-runs.", False
     )
 
     # protocol timeouts
@@ -681,6 +685,14 @@ class Config(CLIConfig):
         ('cdn.reflector.lbry.com', 5567)
     ])
 
+    tracker_servers = Servers("BitTorrent-compatible (BEP15) UDP trackers for helping P2P discovery", [
+        ('tracker.lbry.com', 9252),
+        ('tracker.lbry.grin.io', 9252),
+        ('tracker.lbry.pigg.es', 9252),
+        ('tracker.lizard.technology', 9252),
+        ('s1.lbry.network', 9252),
+    ])
+
     lbryum_servers = Servers("SPV wallet servers", [
         ('spv11.lbry.com', 50001),
         ('spv12.lbry.com', 50001),
@@ -691,14 +703,20 @@ class Config(CLIConfig):
         ('spv17.lbry.com', 50001),
         ('spv18.lbry.com', 50001),
         ('spv19.lbry.com', 50001),
+        ('hub.lbry.grin.io', 50001),
+        ('hub.lizard.technology', 50001),
+        ('s1.lbry.network', 50001),
     ])
     known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
         ('dht.lbry.grin.io', 4444),  # Grin
         ('dht.lbry.madiator.com', 4444),  # Madiator
+        ('dht.lbry.pigg.es', 4444), # Pigges
         ('lbrynet1.lbry.com', 4444),  # US EAST
         ('lbrynet2.lbry.com', 4444),  # US WEST
         ('lbrynet3.lbry.com', 4444),  # EU
-        ('lbrynet4.lbry.com', 4444)  # ASIA
+        ('lbrynet4.lbry.com', 4444),  # ASIA
+        ('dht.lizard.technology', 4444),  # Jack
+        ('s2.lbry.network', 4444),
     ])
 
     # blockchain

lbry/connection_manager.py

@@ -67,7 +67,7 @@ class ConnectionManager:
 
         while True:
             last = time.perf_counter()
-            await asyncio.sleep(0.1, loop=self.loop)
+            await asyncio.sleep(0.1)
             self._status['incoming_bps'].clear()
             self._status['outgoing_bps'].clear()
             now = time.perf_counter()

lbry/dht/blob_announcer.py

@@ -27,27 +27,28 @@ class BlobAnnouncer:
         self.storage = storage
         self.announce_task: asyncio.Task = None
         self.announce_queue: typing.List[str] = []
+        self._done = asyncio.Event()
+        self.announced = set()
 
-    async def _submit_announcement(self, blob_hash):
-        try:
-
-            peers = len(await self.node.announce_blob(blob_hash))
-            self.announcements_sent_metric.labels(peers=peers, error=False).inc()
-            if peers > 4:
-                return blob_hash
-            else:
-                log.debug("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
-        except Exception as err:
-            self.announcements_sent_metric.labels(peers=0, error=True).inc()
-            if isinstance(err, asyncio.CancelledError):  # TODO: remove when updated to 3.8
-                raise err
-            log.warning("error announcing %s: %s", blob_hash[:8], str(err))
+    async def _run_consumer(self):
+        while self.announce_queue:
+            try:
+                blob_hash = self.announce_queue.pop()
+                peers = len(await self.node.announce_blob(blob_hash))
+                self.announcements_sent_metric.labels(peers=peers, error=False).inc()
+                if peers > 4:
+                    self.announced.add(blob_hash)
+                else:
+                    log.debug("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
+            except Exception as err:
+                self.announcements_sent_metric.labels(peers=0, error=True).inc()
+                log.warning("error announcing %s: %s", blob_hash[:8], str(err))
 
     async def _announce(self, batch_size: typing.Optional[int] = 10):
         while batch_size:
             if not self.node.joined.is_set():
                 await self.node.joined.wait()
-            await asyncio.sleep(60, loop=self.loop)
+            await asyncio.sleep(60)
             if not self.node.protocol.routing_table.get_peers():
                 log.warning("No peers in DHT, announce round skipped")
                 continue
@@ -56,14 +57,14 @@ class BlobAnnouncer:
             log.debug("announcer task wake up, %d blobs to announce", len(self.announce_queue))
             while len(self.announce_queue) > 0:
                 log.info("%i blobs to announce", len(self.announce_queue))
-                announced = await asyncio.gather(*[
-                    self._submit_announcement(
-                        self.announce_queue.pop()) for _ in range(batch_size) if self.announce_queue
-                ], loop=self.loop)
-                announced = list(filter(None, announced))
+                await asyncio.gather(*[self._run_consumer() for _ in range(batch_size)])
+                announced = list(filter(None, self.announced))
                 if announced:
                     await self.storage.update_last_announced_blobs(announced)
                     log.info("announced %i blobs", len(announced))
+                    self.announced.clear()
+            self._done.set()
+            self._done.clear()
 
     def start(self, batch_size: typing.Optional[int] = 10):
         assert not self.announce_task or self.announce_task.done(), "already running"
@@ -72,3 +73,6 @@ class BlobAnnouncer:
     def stop(self):
         if self.announce_task and not self.announce_task.done():
             self.announce_task.cancel()
+
+    def wait(self):
+        return self._done.wait()

lbry/dht/constants.py

@@ -20,7 +20,6 @@ MAYBE_PING_DELAY = 300  # 5 minutes
 CHECK_REFRESH_INTERVAL = REFRESH_INTERVAL / 5
 RPC_ID_LENGTH = 20
 PROTOCOL_VERSION = 1
-BOTTOM_OUT_LIMIT = 3
 MSG_SIZE_LIMIT = 1400
 
 

lbry/dht/node.py

@@ -5,7 +5,7 @@ import socket
 
 from prometheus_client import Gauge
 
-from lbry.utils import resolve_host
+from lbry.utils import aclosing, resolve_host
 from lbry.dht import constants
 from lbry.dht.peer import make_kademlia_peer
 from lbry.dht.protocol.distance import Distance
@@ -30,14 +30,14 @@ class Node:
     )
     def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int,
                  internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: float = constants.RPC_TIMEOUT,
-                 split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX,
+                 split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX, is_bootstrap_node: bool = False,
                  storage: typing.Optional['SQLiteStorage'] = None):
         self.loop = loop
         self.internal_udp_port = internal_udp_port
         self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port, rpc_timeout,
-                                         split_buckets_under_index)
+                                         split_buckets_under_index, is_bootstrap_node)
         self.listening_port: asyncio.DatagramTransport = None
-        self.joined = asyncio.Event(loop=self.loop)
+        self.joined = asyncio.Event()
         self._join_task: asyncio.Task = None
         self._refresh_task: asyncio.Task = None
         self._storage = storage
@@ -70,13 +70,6 @@ class Node:
 
             # get ids falling in the midpoint of each bucket that hasn't been recently updated
             node_ids = self.protocol.routing_table.get_refresh_list(0, True)
-            # if we have 3 or fewer populated buckets get two random ids in the range of each to try and
-            # populate/split the buckets further
-            buckets_with_contacts = self.protocol.routing_table.buckets_with_contacts()
-            if buckets_with_contacts <= 3:
-                for i in range(buckets_with_contacts):
-                    node_ids.append(self.protocol.routing_table.random_id_in_bucket_range(i))
-                    node_ids.append(self.protocol.routing_table.random_id_in_bucket_range(i))
 
             if self.protocol.routing_table.get_peers():
                 # if we have node ids to look up, perform the iterative search until we have k results
@@ -86,7 +79,7 @@ class Node:
             else:
                 if force_once:
                     break
-                fut = asyncio.Future(loop=self.loop)
+                fut = asyncio.Future()
                 self.loop.call_later(constants.REFRESH_INTERVAL // 4, fut.set_result, None)
                 await fut
                 continue
@@ -100,7 +93,7 @@ class Node:
             if force_once:
                 break
 
-            fut = asyncio.Future(loop=self.loop)
+            fut = asyncio.Future()
             self.loop.call_later(constants.REFRESH_INTERVAL, fut.set_result, None)
             await fut
 
@@ -115,7 +108,7 @@ class Node:
         for peer in peers:
             log.debug("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
         stored_to_tup = await asyncio.gather(
-            *(self.protocol.store_to_peer(hash_value, peer) for peer in peers), loop=self.loop
+            *(self.protocol.store_to_peer(hash_value, peer) for peer in peers)
         )
         stored_to = [node_id for node_id, contacted in stored_to_tup if contacted]
         if stored_to:
@@ -189,39 +182,36 @@ class Node:
                             for address, udp_port in known_node_urls or []
                         ]))
                     except socket.gaierror:
-                        await asyncio.sleep(30, loop=self.loop)
+                        await asyncio.sleep(30)
                         continue
 
                 self.protocol.peer_manager.reset()
                 self.protocol.ping_queue.enqueue_maybe_ping(*seed_peers, delay=0.0)
                 await self.peer_search(self.protocol.node_id, shortlist=seed_peers, count=32)
 
-            await asyncio.sleep(1, loop=self.loop)
+            await asyncio.sleep(1)
 
     def start(self, interface: str, known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
         self._join_task = self.loop.create_task(self.join_network(interface, known_node_urls))
 
     def get_iterative_node_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
-                                  bottom_out_limit: int = constants.BOTTOM_OUT_LIMIT,
                                   max_results: int = constants.K) -> IterativeNodeFinder:
-
-        return IterativeNodeFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
-                                   key, bottom_out_limit, max_results, None, shortlist)
+        shortlist = shortlist or self.protocol.routing_table.find_close_peers(key)
+        return IterativeNodeFinder(self.loop, self.protocol, key, max_results, shortlist)
 
     def get_iterative_value_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
-                                   bottom_out_limit: int = 40,
                                    max_results: int = -1) -> IterativeValueFinder:
-
-        return IterativeValueFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
-                                    key, bottom_out_limit, max_results, None, shortlist)
+        shortlist = shortlist or self.protocol.routing_table.find_close_peers(key)
+        return IterativeValueFinder(self.loop, self.protocol, key, max_results, shortlist)
 
     async def peer_search(self, node_id: bytes, count=constants.K, max_results=constants.K * 2,
-                          bottom_out_limit=20, shortlist: typing.Optional[typing.List['KademliaPeer']] = None
+                          shortlist: typing.Optional[typing.List['KademliaPeer']] = None
                           ) -> typing.List['KademliaPeer']:
         peers = []
-        async for iteration_peers in self.get_iterative_node_finder(
-                node_id, shortlist=shortlist, bottom_out_limit=bottom_out_limit, max_results=max_results):
-            peers.extend(iteration_peers)
+        async with aclosing(self.get_iterative_node_finder(
+                node_id, shortlist=shortlist, max_results=max_results)) as node_finder:
+            async for iteration_peers in node_finder:
+                peers.extend(iteration_peers)
         distance = Distance(node_id)
         peers.sort(key=lambda peer: distance(peer.node_id))
         return peers[:count]
@@ -247,41 +237,41 @@ class Node:
 
         # prioritize peers who reply to a dht ping first
         # this minimizes attempting to make tcp connections that won't work later to dead or unreachable peers
-
-        async for results in self.get_iterative_value_finder(bytes.fromhex(blob_hash)):
-            to_put = []
-            for peer in results:
-                if peer.address == self.protocol.external_ip and self.protocol.peer_port == peer.tcp_port:
-                    continue
-                is_good = self.protocol.peer_manager.peer_is_good(peer)
-                if is_good:
-                    # the peer has replied recently over UDP, it can probably be reached on the TCP port
-                    to_put.append(peer)
-                elif is_good is None:
-                    if not peer.udp_port:
-                        # TODO: use the same port for TCP and UDP
-                        # the udp port must be guessed
-                        # default to the ports being the same. if the TCP port appears to be <=0.48.0 default,
-                        # including on a network with several nodes, then assume the udp port is proportionately
-                        # based on a starting port of 4444
-                        udp_port_to_try = peer.tcp_port
-                        if 3400 > peer.tcp_port > 3332:
-                            udp_port_to_try = (peer.tcp_port - 3333) + 4444
-                        self.loop.create_task(put_into_result_queue_after_pong(
-                            make_kademlia_peer(peer.node_id, peer.address, udp_port_to_try, peer.tcp_port)
-                        ))
+        async with aclosing(self.get_iterative_value_finder(bytes.fromhex(blob_hash))) as value_finder:
+            async for results in value_finder:
+                to_put = []
+                for peer in results:
+                    if peer.address == self.protocol.external_ip and self.protocol.peer_port == peer.tcp_port:
+                        continue
+                    is_good = self.protocol.peer_manager.peer_is_good(peer)
+                    if is_good:
+                        # the peer has replied recently over UDP, it can probably be reached on the TCP port
+                        to_put.append(peer)
+                    elif is_good is None:
+                        if not peer.udp_port:
+                            # TODO: use the same port for TCP and UDP
+                            # the udp port must be guessed
+                            # default to the ports being the same. if the TCP port appears to be <=0.48.0 default,
+                            # including on a network with several nodes, then assume the udp port is proportionately
+                            # based on a starting port of 4444
+                            udp_port_to_try = peer.tcp_port
+                            if 3400 > peer.tcp_port > 3332:
+                                udp_port_to_try = (peer.tcp_port - 3333) + 4444
+                            self.loop.create_task(put_into_result_queue_after_pong(
+                                make_kademlia_peer(peer.node_id, peer.address, udp_port_to_try, peer.tcp_port)
+                            ))
+                        else:
+                            self.loop.create_task(put_into_result_queue_after_pong(peer))
                     else:
-                        self.loop.create_task(put_into_result_queue_after_pong(peer))
-                else:
-                    # the peer is known to be bad/unreachable, skip trying to connect to it over TCP
-                    log.debug("skip bad peer %s:%i for %s", peer.address, peer.tcp_port, blob_hash)
-            if to_put:
-                result_queue.put_nowait(to_put)
+                        # the peer is known to be bad/unreachable, skip trying to connect to it over TCP
+                        log.debug("skip bad peer %s:%i for %s", peer.address, peer.tcp_port, blob_hash)
+                if to_put:
+                    result_queue.put_nowait(to_put)
 
     def accumulate_peers(self, search_queue: asyncio.Queue,
                          peer_queue: typing.Optional[asyncio.Queue] = None
                          ) -> typing.Tuple[asyncio.Queue, asyncio.Task]:
-        queue = peer_queue or asyncio.Queue(loop=self.loop)
+        queue = peer_queue or asyncio.Queue()
         return queue, self.loop.create_task(self._accumulate_peers_for_value(search_queue, queue))
 
 

lbry/dht/peer.py

@@ -100,6 +100,9 @@ class PeerManager:
         self._node_id_reverse_mapping[node_id] = (address, udp_port)
         self.peer_manager_keys_metric.labels("global").set(self.count_cache_keys())
 
+    def get_node_id_for_endpoint(self, address, port):
+        return self._node_id_mapping.get((address, port))
+
     def prune(self):  # TODO: periodically call this
         now = self._loop.time()
         to_pop = []
@@ -150,9 +153,10 @@ class PeerManager:
     def peer_is_good(self, peer: 'KademliaPeer'):
         return self.contact_triple_is_good(peer.node_id, peer.address, peer.udp_port)
 
-    def decode_tcp_peer_from_compact_address(self, compact_address: bytes) -> 'KademliaPeer':  # pylint: disable=no-self-use
-        node_id, address, tcp_port = decode_compact_address(compact_address)
-        return make_kademlia_peer(node_id, address, udp_port=None, tcp_port=tcp_port)
+
+def decode_tcp_peer_from_compact_address(compact_address: bytes) -> 'KademliaPeer':  # pylint: disable=no-self-use
+    node_id, address, tcp_port = decode_compact_address(compact_address)
+    return make_kademlia_peer(node_id, address, udp_port=None, tcp_port=tcp_port)
 
 
 @dataclass(unsafe_hash=True)
@@ -190,3 +194,6 @@ class KademliaPeer:
 
     def compact_ip(self):
         return make_compact_ip(self.address)
+
+    def __str__(self):
+        return f"{self.__class__.__name__}({self.node_id.hex()[:8]}@{self.address}:{self.udp_port}-{self.tcp_port})"

lbry/dht/protocol/iterative_find.py

@@ -1,17 +1,17 @@
 import asyncio
 from itertools import chain
-from collections import defaultdict
+from collections import defaultdict, OrderedDict
+from collections.abc import AsyncIterator
 import typing
 import logging
 from typing import TYPE_CHECKING
 from lbry.dht import constants
 from lbry.dht.error import RemoteException, TransportNotConnected
 from lbry.dht.protocol.distance import Distance
-from lbry.dht.peer import make_kademlia_peer
+from lbry.dht.peer import make_kademlia_peer, decode_tcp_peer_from_compact_address
 from lbry.dht.serialization.datagram import PAGE_KEY
 
 if TYPE_CHECKING:
-    from lbry.dht.protocol.routing_table import TreeRoutingTable
     from lbry.dht.protocol.protocol import KademliaProtocol
     from lbry.dht.peer import PeerManager, KademliaPeer
 
@@ -26,6 +26,15 @@ class FindResponse:
     def get_close_triples(self) -> typing.List[typing.Tuple[bytes, str, int]]:
         raise NotImplementedError()
 
+    def get_close_kademlia_peers(self, peer_info) -> typing.Generator[typing.Iterator['KademliaPeer'], None, None]:
+        for contact_triple in self.get_close_triples():
+            node_id, address, udp_port = contact_triple
+            try:
+                yield make_kademlia_peer(node_id, address, udp_port)
+            except ValueError:
+                log.warning("misbehaving peer %s:%i returned peer with reserved ip %s:%i", peer_info.address,
+                            peer_info.udp_port, address, udp_port)
+
 
 class FindNodeResponse(FindResponse):
     def __init__(self, key: bytes, close_triples: typing.List[typing.Tuple[bytes, str, int]]):
@@ -56,57 +65,33 @@ class FindValueResponse(FindResponse):
         return [(node_id, address.decode(), port) for node_id, address, port in self.close_triples]
 
 
-def get_shortlist(routing_table: 'TreeRoutingTable', key: bytes,
-                  shortlist: typing.Optional[typing.List['KademliaPeer']]) -> typing.List['KademliaPeer']:
-    """
-    If not provided, initialize the shortlist of peers to probe to the (up to) k closest peers in the routing table
-
-    :param routing_table: a TreeRoutingTable
-    :param key: a 48 byte hash
-    :param shortlist: optional manually provided shortlist, this is done during bootstrapping when there are no
-                      peers in the routing table. During bootstrap the shortlist is set to be the seed nodes.
-    """
-    if len(key) != constants.HASH_LENGTH:
-        raise ValueError("invalid key length: %i" % len(key))
-    return shortlist or routing_table.find_close_peers(key)
-
-
-class IterativeFinder:
-    def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager',
-                 routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
-                 bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.K,
-                 exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
+class IterativeFinder(AsyncIterator):
+    def __init__(self, loop: asyncio.AbstractEventLoop,
+                 protocol: 'KademliaProtocol', key: bytes,
+                 max_results: typing.Optional[int] = constants.K,
                  shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
         if len(key) != constants.HASH_LENGTH:
             raise ValueError("invalid key length: %i" % len(key))
         self.loop = loop
-        self.peer_manager = peer_manager
-        self.routing_table = routing_table
+        self.peer_manager = protocol.peer_manager
         self.protocol = protocol
 
         self.key = key
-        self.bottom_out_limit = bottom_out_limit
-        self.max_results = max_results
-        self.exclude = exclude or []
+        self.max_results = max(constants.K, max_results)
 
-        self.active: typing.Set['KademliaPeer'] = set()
+        self.active: typing.Dict['KademliaPeer', int] = OrderedDict()  # peer: distance, sorted
         self.contacted: typing.Set['KademliaPeer'] = set()
         self.distance = Distance(key)
 
-        self.closest_peer: typing.Optional['KademliaPeer'] = None
-        self.prev_closest_peer: typing.Optional['KademliaPeer'] = None
+        self.iteration_queue = asyncio.Queue()
 
-        self.iteration_queue = asyncio.Queue(loop=self.loop)
-
-        self.running_probes: typing.Set[asyncio.Task] = set()
+        self.running_probes: typing.Dict['KademliaPeer', asyncio.Task] = {}
         self.iteration_count = 0
-        self.bottom_out_count = 0
         self.running = False
         self.tasks: typing.List[asyncio.Task] = []
-        self.delayed_calls: typing.List[asyncio.Handle] = []
-        for peer in get_shortlist(routing_table, key, shortlist):
+        for peer in shortlist:
             if peer.node_id:
-                self._add_active(peer)
+                self._add_active(peer, force=True)
             else:
                 # seed nodes
                 self._schedule_probe(peer)
@@ -138,66 +123,79 @@ class IterativeFinder:
         """
         return []
 
-    def _is_closer(self, peer: 'KademliaPeer') -> bool:
-        return not self.closest_peer or self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
-
-    def _add_active(self, peer):
+    def _add_active(self, peer, force=False):
+        if not force and self.peer_manager.peer_is_good(peer) is False:
+            return
+        if peer in self.contacted:
+            return
         if peer not in self.active and peer.node_id and peer.node_id != self.protocol.node_id:
-            self.active.add(peer)
-            if self._is_closer(peer):
-                self.prev_closest_peer = self.closest_peer
-                self.closest_peer = peer
+            self.active[peer] = self.distance(peer.node_id)
+            self.active = OrderedDict(sorted(self.active.items(), key=lambda item: item[1]))
 
     async def _handle_probe_result(self, peer: 'KademliaPeer', response: FindResponse):
         self._add_active(peer)
-        for contact_triple in response.get_close_triples():
-            node_id, address, udp_port = contact_triple
-            try:
-                self._add_active(make_kademlia_peer(node_id, address, udp_port))
-            except ValueError:
-                log.warning("misbehaving peer %s:%i returned peer with reserved ip %s:%i", peer.address,
-                            peer.udp_port, address, udp_port)
+        for new_peer in response.get_close_kademlia_peers(peer):
+            self._add_active(new_peer)
         self.check_result_ready(response)
+        self._log_state(reason="check result")
+
+    def _reset_closest(self, peer):
+        if peer in self.active:
+            del self.active[peer]
 
     async def _send_probe(self, peer: 'KademliaPeer'):
         try:
             response = await self.send_probe(peer)
         except asyncio.TimeoutError:
-            self.active.discard(peer)
+            self._reset_closest(peer)
             return
+        except asyncio.CancelledError:
+            log.debug("%s[%x] cancelled probe",
+                      type(self).__name__, id(self))
+            raise
         except ValueError as err:
             log.warning(str(err))
-            self.active.discard(peer)
+            self._reset_closest(peer)
             return
         except TransportNotConnected:
-            return self.aclose()
+            await self._aclose(reason="not connected")
+            return
         except RemoteException:
+            self._reset_closest(peer)
             return
         return await self._handle_probe_result(peer, response)
 
-    async def _search_round(self):
+    def _search_round(self):
         """
         Send up to constants.alpha (5) probes to closest active peers
         """
 
         added = 0
-        to_probe = list(self.active - self.contacted)
-        to_probe.sort(key=lambda peer: self.distance(self.key))
-        for peer in to_probe:
-            if added >= constants.ALPHA:
+        for index, peer in enumerate(self.active.keys()):
+            if index == 0:
+                log.debug("%s[%x] closest to probe: %s",
+                          type(self).__name__, id(self),
+                          peer.node_id.hex()[:8])
+            if peer in self.contacted:
+                continue
+            if len(self.running_probes) >= constants.ALPHA:
+                break
+            if index > (constants.K + len(self.running_probes)):
                 break
             origin_address = (peer.address, peer.udp_port)
-            if origin_address in self.exclude:
-                continue
             if peer.node_id == self.protocol.node_id:
                 continue
             if origin_address == (self.protocol.external_ip, self.protocol.udp_port):
                 continue
             self._schedule_probe(peer)
             added += 1
-        log.debug("running %d probes for key %s", len(self.running_probes), self.key.hex()[:8])
+        log.debug("%s[%x] running %d probes for key %s",
+                  type(self).__name__, id(self),
+                  len(self.running_probes), self.key.hex()[:8])
         if not added and not self.running_probes:
-            log.debug("search for %s exhausted", self.key.hex()[:8])
+            log.debug("%s[%x] search for %s exhausted",
+                      type(self).__name__, id(self),
+                      self.key.hex()[:8])
             self.search_exhausted()
 
     def _schedule_probe(self, peer: 'KademliaPeer'):
@@ -206,33 +204,24 @@ class IterativeFinder:
         t = self.loop.create_task(self._send_probe(peer))
 
         def callback(_):
-            self.running_probes.difference_update({
-                probe for probe in self.running_probes if probe.done() or probe == t
-            })
-            if not self.running_probes:
-                self.tasks.append(self.loop.create_task(self._search_task(0.0)))
+            self.running_probes.pop(peer, None)
+            if self.running:
+                self._search_round()
 
         t.add_done_callback(callback)
-        self.running_probes.add(t)
+        self.running_probes[peer] = t
 
-    async def _search_task(self, delay: typing.Optional[float] = constants.ITERATIVE_LOOKUP_DELAY):
-        try:
-            if self.running:
-                await self._search_round()
-            if self.running:
-                self.delayed_calls.append(self.loop.call_later(delay, self._search))
-        except (asyncio.CancelledError, StopAsyncIteration, TransportNotConnected):
-            if self.running:
-                self.loop.call_soon(self.aclose)
-
-    def _search(self):
-        self.tasks.append(self.loop.create_task(self._search_task()))
+    def _log_state(self, reason="?"):
+        log.debug("%s[%x] [%s] %s: %i active nodes %i contacted %i produced %i queued",
+                  type(self).__name__, id(self), self.key.hex()[:8],
+                  reason, len(self.active), len(self.contacted),
+                  self.iteration_count, self.iteration_queue.qsize())
 
     def __aiter__(self):
         if self.running:
             raise Exception("already running")
         self.running = True
-        self._search()
+        self.loop.call_soon(self._search_round)
         return self
 
     async def __anext__(self) -> typing.List['KademliaPeer']:
@@ -245,28 +234,37 @@ class IterativeFinder:
                 raise StopAsyncIteration
             self.iteration_count += 1
             return result
-        except (asyncio.CancelledError, StopAsyncIteration):
-            self.loop.call_soon(self.aclose)
+        except asyncio.CancelledError:
+            await self._aclose(reason="cancelled")
+            raise
+        except StopAsyncIteration:
+            await self._aclose(reason="no more results")
             raise
 
-    def aclose(self):
+    async def _aclose(self, reason="?"):
+        log.debug("%s[%x] [%s] shutdown because %s: %i active nodes %i contacted %i produced %i queued",
+                  type(self).__name__, id(self), self.key.hex()[:8],
+                  reason, len(self.active), len(self.contacted),
+                  self.iteration_count, self.iteration_queue.qsize())
         self.running = False
         self.iteration_queue.put_nowait(None)
-        for task in chain(self.tasks, self.running_probes, self.delayed_calls):
+        for task in chain(self.tasks, self.running_probes.values()):
             task.cancel()
         self.tasks.clear()
         self.running_probes.clear()
-        self.delayed_calls.clear()
 
+    async def aclose(self):
+        if self.running:
+            await self._aclose(reason="aclose")
+        log.debug("%s[%x] [%s] async close completed",
+                  type(self).__name__, id(self), self.key.hex()[:8])
 
 class IterativeNodeFinder(IterativeFinder):
-    def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager',
-                 routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
-                 bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.K,
-                 exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
+    def __init__(self, loop: asyncio.AbstractEventLoop,
+                 protocol: 'KademliaProtocol', key: bytes,
+                 max_results: typing.Optional[int] = constants.K,
                  shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
-        super().__init__(loop, peer_manager, routing_table, protocol, key, bottom_out_limit, max_results, exclude,
-                         shortlist)
+        super().__init__(loop, protocol, key, max_results, shortlist)
         self.yielded_peers: typing.Set['KademliaPeer'] = set()
 
     async def send_probe(self, peer: 'KademliaPeer') -> FindNodeResponse:
@@ -276,14 +274,14 @@ class IterativeNodeFinder(IterativeFinder):
         return FindNodeResponse(self.key, response)
 
     def search_exhausted(self):
-        self.put_result(self.active, finish=True)
+        self.put_result(self.active.keys(), finish=True)
 
     def put_result(self, from_iter: typing.Iterable['KademliaPeer'], finish=False):
         not_yet_yielded = [
             peer for peer in from_iter
             if peer not in self.yielded_peers
             and peer.node_id != self.protocol.node_id
-            and self.peer_manager.peer_is_good(peer) is not False
+            and self.peer_manager.peer_is_good(peer) is True  # return only peers who answered
         ]
         not_yet_yielded.sort(key=lambda peer: self.distance(peer.node_id))
         to_yield = not_yet_yielded[:max(constants.K, self.max_results)]
@@ -298,27 +296,15 @@ class IterativeNodeFinder(IterativeFinder):
 
         if found:
             log.debug("found")
-            return self.put_result(self.active, finish=True)
-        if self.prev_closest_peer and self.closest_peer and not self._is_closer(self.prev_closest_peer):
-            # log.info("improving, %i %i %i %i %i", len(self.shortlist), len(self.active), len(self.contacted),
-            #          self.bottom_out_count, self.iteration_count)
-            self.bottom_out_count = 0
-        elif self.prev_closest_peer and self.closest_peer:
-            self.bottom_out_count += 1
-            log.info("bottom out %i %i %i", len(self.active), len(self.contacted), self.bottom_out_count)
-        if self.bottom_out_count >= self.bottom_out_limit or self.iteration_count >= self.bottom_out_limit:
-            log.info("limit hit")
-            self.put_result(self.active, True)
+            return self.put_result(self.active.keys(), finish=True)
 
 
 class IterativeValueFinder(IterativeFinder):
-    def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager',
-                 routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
-                 bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.K,
-                 exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
+    def __init__(self, loop: asyncio.AbstractEventLoop,
+                 protocol: 'KademliaProtocol', key: bytes,
+                 max_results: typing.Optional[int] = constants.K,
                  shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
-        super().__init__(loop, peer_manager, routing_table, protocol, key, bottom_out_limit, max_results, exclude,
-                         shortlist)
+        super().__init__(loop, protocol, key, max_results, shortlist)
         self.blob_peers: typing.Set['KademliaPeer'] = set()
         # this tracks the index of the most recent page we requested from each peer
         self.peer_pages: typing.DefaultDict['KademliaPeer', int] = defaultdict(int)
@@ -337,7 +323,7 @@ class IterativeValueFinder(IterativeFinder):
         decoded_peers = set()
         for compact_addr in parsed.found_compact_addresses:
             try:
-                decoded_peers.add(self.peer_manager.decode_tcp_peer_from_compact_address(compact_addr))
+                decoded_peers.add(decode_tcp_peer_from_compact_address(compact_addr))
             except ValueError:
                 log.warning("misbehaving peer %s:%i returned invalid peer for blob",
                             peer.address, peer.udp_port)
@@ -359,26 +345,15 @@ class IterativeValueFinder(IterativeFinder):
 
     def check_result_ready(self, response: FindValueResponse):
         if response.found:
-            blob_peers = [self.peer_manager.decode_tcp_peer_from_compact_address(compact_addr)
+            blob_peers = [decode_tcp_peer_from_compact_address(compact_addr)
                           for compact_addr in response.found_compact_addresses]
             to_yield = []
-            self.bottom_out_count = 0
             for blob_peer in blob_peers:
                 if blob_peer not in self.blob_peers:
                     self.blob_peers.add(blob_peer)
                     to_yield.append(blob_peer)
             if to_yield:
-                # log.info("found %i new peers for blob", len(to_yield))
                 self.iteration_queue.put_nowait(to_yield)
-                # if self.max_results and len(self.blob_peers) >= self.max_results:
-                #     log.info("enough blob peers found")
-                #     if not self.finished.is_set():
-                #         self.finished.set()
-        elif self.prev_closest_peer and self.closest_peer:
-            self.bottom_out_count += 1
-            if self.bottom_out_count >= self.bottom_out_limit:
-                log.info("blob peer search bottomed out")
-                self.iteration_queue.put_nowait(None)
 
     def get_initial_result(self) -> typing.List['KademliaPeer']:
         if self.protocol.data_store.has_peers_for_blob(self.key):

lbry/dht/protocol/protocol.py

@@ -218,6 +218,10 @@ class PingQueue:
     def running(self):
         return self._running
 
+    @property
+    def busy(self):
+        return self._running and (any(self._running_pings) or any(self._pending_contacts))
+
     def enqueue_maybe_ping(self, *peers: 'KademliaPeer', delay: typing.Optional[float] = None):
         delay = delay if delay is not None else self._default_delay
         now = self._loop.time()
@@ -229,7 +233,7 @@ class PingQueue:
         async def ping_task():
             try:
                 if self._protocol.peer_manager.peer_is_good(peer):
-                    if peer not in self._protocol.routing_table.get_peers():
+                    if not self._protocol.routing_table.get_peer(peer.node_id):
                         self._protocol.add_peer(peer)
                     return
                 await self._protocol.get_rpc_peer(peer).ping()
@@ -249,7 +253,7 @@ class PingQueue:
                     del self._pending_contacts[peer]
                     self.maybe_ping(peer)
                     break
-            await asyncio.sleep(1, loop=self._loop)
+            await asyncio.sleep(1)
 
     def start(self):
         assert not self._running
@@ -294,7 +298,7 @@ class KademliaProtocol(DatagramProtocol):
 
     def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', node_id: bytes, external_ip: str,
                  udp_port: int, peer_port: int, rpc_timeout: float = constants.RPC_TIMEOUT,
-                 split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX):
+                 split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX, is_boostrap_node: bool = False):
         self.peer_manager = peer_manager
         self.loop = loop
         self.node_id = node_id
@@ -309,15 +313,16 @@ class KademliaProtocol(DatagramProtocol):
         self.transport: DatagramTransport = None
         self.old_token_secret = constants.generate_id()
         self.token_secret = constants.generate_id()
-        self.routing_table = TreeRoutingTable(self.loop, self.peer_manager, self.node_id, split_buckets_under_index)
+        self.routing_table = TreeRoutingTable(
+            self.loop, self.peer_manager, self.node_id, split_buckets_under_index, is_bootstrap_node=is_boostrap_node)
         self.data_store = DictDataStore(self.loop, self.peer_manager)
         self.ping_queue = PingQueue(self.loop, self)
         self.node_rpc = KademliaRPC(self, self.loop, self.peer_port)
         self.rpc_timeout = rpc_timeout
-        self._split_lock = asyncio.Lock(loop=self.loop)
+        self._split_lock = asyncio.Lock()
         self._to_remove: typing.Set['KademliaPeer'] = set()
         self._to_add: typing.Set['KademliaPeer'] = set()
-        self._wakeup_routing_task = asyncio.Event(loop=self.loop)
+        self._wakeup_routing_task = asyncio.Event()
         self.maintaing_routing_task: typing.Optional[asyncio.Task] = None
 
     @functools.lru_cache(128)
@@ -356,72 +361,10 @@ class KademliaProtocol(DatagramProtocol):
         return args, {}
 
     async def _add_peer(self, peer: 'KademliaPeer'):
-        if not peer.node_id:
-            log.warning("Tried adding a peer with no node id!")
-            return False
-        for my_peer in self.routing_table.get_peers():
-            if (my_peer.address, my_peer.udp_port) == (peer.address, peer.udp_port) and my_peer.node_id != peer.node_id:
-                self.routing_table.remove_peer(my_peer)
-                self.routing_table.join_buckets()
-        bucket_index = self.routing_table.kbucket_index(peer.node_id)
-        if self.routing_table.buckets[bucket_index].add_peer(peer):
-            return True
-
-        # The bucket is full; see if it can be split (by checking if its range includes the host node's node_id)
-        if self.routing_table.should_split(bucket_index, peer.node_id):
-            self.routing_table.split_bucket(bucket_index)
-            # Retry the insertion attempt
-            result = await self._add_peer(peer)
-            self.routing_table.join_buckets()
-            return result
-        else:
-            # We can't split the k-bucket
-            #
-            # The 13 page kademlia paper specifies that the least recently contacted node in the bucket
-            # shall be pinged. If it fails to reply it is replaced with the new contact. If the ping is successful
-            # the new contact is ignored and not added to the bucket (sections 2.2 and 2.4).
-            #
-            # A reasonable extension to this is BEP 0005, which extends the above:
-            #
-            #    Not all nodes that we learn about are equal. Some are "good" and some are not.
-            #    Many nodes using the DHT are able to send queries and receive responses,
-            #    but are not able to respond to queries from other nodes. It is important that
-            #    each node's routing table must contain only known good nodes. A good node is
-            #    a node has responded to one of our queries within the last 15 minutes. A node
-            #    is also good if it has ever responded to one of our queries and has sent us a
-            #    query within the last 15 minutes. After 15 minutes of inactivity, a node becomes
-            #    questionable. Nodes become bad when they fail to respond to multiple queries
-            #    in a row. Nodes that we know are good are given priority over nodes with unknown status.
-            #
-            # When there are bad or questionable nodes in the bucket, the least recent is selected for
-            # potential replacement (BEP 0005). When all nodes in the bucket are fresh, the head (least recent)
-            # contact is selected as described in section 2.2 of the kademlia paper. In both cases the new contact
-            # is ignored if the pinged node replies.
-
-            not_good_contacts = self.routing_table.buckets[bucket_index].get_bad_or_unknown_peers()
-            not_recently_replied = []
-            for my_peer in not_good_contacts:
-                last_replied = self.peer_manager.get_last_replied(my_peer.address, my_peer.udp_port)
-                if not last_replied or last_replied + 60 < self.loop.time():
-                    not_recently_replied.append(my_peer)
-            if not_recently_replied:
-                to_replace = not_recently_replied[0]
-            else:
-                to_replace = self.routing_table.buckets[bucket_index].peers[0]
-                last_replied = self.peer_manager.get_last_replied(to_replace.address, to_replace.udp_port)
-                if last_replied and last_replied + 60 > self.loop.time():
-                    return False
-            log.debug("pinging %s:%s", to_replace.address, to_replace.udp_port)
-            try:
-                to_replace_rpc = self.get_rpc_peer(to_replace)
-                await to_replace_rpc.ping()
-                return False
-            except asyncio.TimeoutError:
-                log.debug("Replacing dead contact in bucket %i: %s:%i with %s:%i ", bucket_index,
-                          to_replace.address, to_replace.udp_port, peer.address, peer.udp_port)
-                if to_replace in self.routing_table.buckets[bucket_index]:
-                    self.routing_table.buckets[bucket_index].remove_peer(to_replace)
-                return await self._add_peer(peer)
+        async def probe(some_peer: 'KademliaPeer'):
+            rpc_peer = self.get_rpc_peer(some_peer)
+            await rpc_peer.ping()
+        return await self.routing_table.add_peer(peer, probe)
 
     def add_peer(self, peer: 'KademliaPeer'):
         if peer.node_id == self.node_id:
@@ -439,11 +382,10 @@ class KademliaProtocol(DatagramProtocol):
                 async with self._split_lock:
                     peer = self._to_remove.pop()
                     self.routing_table.remove_peer(peer)
-                    self.routing_table.join_buckets()
             while self._to_add:
                 async with self._split_lock:
                     await self._add_peer(self._to_add.pop())
-            await asyncio.gather(self._wakeup_routing_task.wait(), asyncio.sleep(.1, loop=self.loop), loop=self.loop)
+            await asyncio.gather(self._wakeup_routing_task.wait(), asyncio.sleep(.1))
             self._wakeup_routing_task.clear()
 
     def _handle_rpc(self, sender_contact: 'KademliaPeer', message: RequestDatagram):
@@ -482,9 +424,8 @@ class KademliaProtocol(DatagramProtocol):
         # This is an RPC method request
         self.received_request_metric.labels(method=request_datagram.method).inc()
         self.peer_manager.report_last_requested(address[0], address[1])
-        try:
-            peer = self.routing_table.get_peer(request_datagram.node_id)
-        except IndexError:
+        peer = self.routing_table.get_peer(request_datagram.node_id)
+        if not peer:
             try:
                 peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1])
             except ValueError as err:

lbry/dht/protocol/routing_table.py

@@ -6,7 +6,9 @@ import itertools
 
 from prometheus_client import Gauge
 
+from lbry import utils
 from lbry.dht import constants
+from lbry.dht.error import RemoteException
 from lbry.dht.protocol.distance import Distance
 if typing.TYPE_CHECKING:
     from lbry.dht.peer import KademliaPeer, PeerManager
@@ -27,7 +29,8 @@ class KBucket:
         namespace="dht_node", labelnames=("amount",)
     )
 
-    def __init__(self, peer_manager: 'PeerManager', range_min: int, range_max: int, node_id: bytes):
+    def __init__(self, peer_manager: 'PeerManager', range_min: int, range_max: int,
+                 node_id: bytes, capacity: int = constants.K):
         """
         @param range_min: The lower boundary for the range in the n-bit ID
                          space covered by this k-bucket
@@ -35,12 +38,12 @@ class KBucket:
                          covered by this k-bucket
         """
         self._peer_manager = peer_manager
-        self.last_accessed = 0
         self.range_min = range_min
         self.range_max = range_max
         self.peers: typing.List['KademliaPeer'] = []
         self._node_id = node_id
         self._distance_to_self = Distance(node_id)
+        self.capacity = capacity
 
     def add_peer(self, peer: 'KademliaPeer') -> bool:
         """ Add contact to _contact list in the right order. This will move the
@@ -67,22 +70,19 @@ class KBucket:
                     self.peers.remove(local_peer)
                     self.peers.append(peer)
                     return True
-        if len(self.peers) < constants.K:
+        if len(self.peers) < self.capacity:
             self.peers.append(peer)
             self.peer_in_routing_table_metric.labels("global").inc()
-            if peer.node_id[0] == self._node_id[0]:
-                bits_colliding = 8 - (peer.node_id[1] ^ self._node_id[1]).bit_length()
-                self.peer_with_x_bit_colliding_metric.labels(amount=(bits_colliding + 8)).inc()
+            bits_colliding = utils.get_colliding_prefix_bits(peer.node_id, self._node_id)
+            self.peer_with_x_bit_colliding_metric.labels(amount=bits_colliding).inc()
             return True
         else:
             return False
-            # raise BucketFull("No space in bucket to insert contact")
 
     def get_peer(self, node_id: bytes) -> 'KademliaPeer':
         for peer in self.peers:
             if peer.node_id == node_id:
                 return peer
-        raise IndexError(node_id)
 
     def get_peers(self, count=-1, exclude_contact=None, sort_distance_to=None) -> typing.List['KademliaPeer']:
         """ Returns a list containing up to the first count number of contacts
@@ -140,9 +140,8 @@ class KBucket:
     def remove_peer(self, peer: 'KademliaPeer') -> None:
         self.peers.remove(peer)
         self.peer_in_routing_table_metric.labels("global").dec()
-        if peer.node_id[0] == self._node_id[0]:
-            bits_colliding = 8 - (peer.node_id[1] ^ self._node_id[1]).bit_length()
-            self.peer_with_x_bit_colliding_metric.labels(amount=(bits_colliding + 8)).dec()
+        bits_colliding = utils.get_colliding_prefix_bits(peer.node_id, self._node_id)
+        self.peer_with_x_bit_colliding_metric.labels(amount=bits_colliding).dec()
 
     def key_in_range(self, key: bytes) -> bool:
         """ Tests whether the specified key (i.e. node ID) is in the range
@@ -180,6 +179,13 @@ class TreeRoutingTable:
     version of the Kademlia paper, in section 2.4. It does, however, use the
     ping RPC-based k-bucket eviction algorithm described in section 2.2 of
     that paper.
+
+    BOOTSTRAP MODE: if set to True, we always add all peers. This is so a
+    bootstrap node does not get a bias towards its own node id and replies are
+    the best it can provide (joining peer knows its neighbors immediately).
+    Over time, this will need to be optimized so we use the disk as holding
+    everything in memory won't be feasible anymore.
+    See: https://github.com/bittorrent/bootstrap-dht
     """
     bucket_in_routing_table_metric = Gauge(
         "buckets_in_routing_table", "Number of buckets on routing table", namespace="dht_node",
@@ -187,21 +193,22 @@ class TreeRoutingTable:
     )
 
     def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', parent_node_id: bytes,
-                 split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX):
+                 split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX, is_bootstrap_node: bool = False):
         self._loop = loop
         self._peer_manager = peer_manager
         self._parent_node_id = parent_node_id
         self._split_buckets_under_index = split_buckets_under_index
         self.buckets: typing.List[KBucket] = [
             KBucket(
-                self._peer_manager, range_min=0, range_max=2 ** constants.HASH_BITS, node_id=self._parent_node_id
+                self._peer_manager, range_min=0, range_max=2 ** constants.HASH_BITS, node_id=self._parent_node_id,
+                capacity=1 << 32 if is_bootstrap_node else constants.K
             )
         ]
 
     def get_peers(self) -> typing.List['KademliaPeer']:
         return list(itertools.chain.from_iterable(map(lambda bucket: bucket.peers, self.buckets)))
 
-    def should_split(self, bucket_index: int, to_add: bytes) -> bool:
+    def _should_split(self, bucket_index: int, to_add: bytes) -> bool:
         #  https://stackoverflow.com/questions/32129978/highly-unbalanced-kademlia-routing-table/32187456#32187456
         if bucket_index < self._split_buckets_under_index:
             return True
@@ -226,39 +233,32 @@ class TreeRoutingTable:
         return []
 
     def get_peer(self, contact_id: bytes) -> 'KademliaPeer':
-        """
-        @raise IndexError: No contact with the specified contact ID is known
-                           by this node
-        """
-        return self.buckets[self.kbucket_index(contact_id)].get_peer(contact_id)
+        return self.buckets[self._kbucket_index(contact_id)].get_peer(contact_id)
 
     def get_refresh_list(self, start_index: int = 0, force: bool = False) -> typing.List[bytes]:
-        bucket_index = start_index
         refresh_ids = []
-        now = int(self._loop.time())
-        for bucket in self.buckets[start_index:]:
-            if force or now - bucket.last_accessed >= constants.REFRESH_INTERVAL:
-                to_search = self.midpoint_id_in_bucket_range(bucket_index)
-                refresh_ids.append(to_search)
-            bucket_index += 1
+        for offset, _ in enumerate(self.buckets[start_index:]):
+            refresh_ids.append(self._midpoint_id_in_bucket_range(start_index + offset))
+        # if we have 3 or fewer populated buckets get two random ids in the range of each to try and
+        # populate/split the buckets further
+        buckets_with_contacts = self.buckets_with_contacts()
+        if buckets_with_contacts <= 3:
+            for i in range(buckets_with_contacts):
+                refresh_ids.append(self._random_id_in_bucket_range(i))
+                refresh_ids.append(self._random_id_in_bucket_range(i))
         return refresh_ids
 
     def remove_peer(self, peer: 'KademliaPeer') -> None:
         if not peer.node_id:
             return
-        bucket_index = self.kbucket_index(peer.node_id)
+        bucket_index = self._kbucket_index(peer.node_id)
         try:
             self.buckets[bucket_index].remove_peer(peer)
+            self._join_buckets()
         except ValueError:
             return
 
-    def touch_kbucket(self, key: bytes) -> None:
-        self.touch_kbucket_by_index(self.kbucket_index(key))
-
-    def touch_kbucket_by_index(self, bucket_index: int):
-        self.buckets[bucket_index].last_accessed = int(self._loop.time())
-
-    def kbucket_index(self, key: bytes) -> int:
+    def _kbucket_index(self, key: bytes) -> int:
         i = 0
         for bucket in self.buckets:
             if bucket.key_in_range(key):
@@ -267,19 +267,19 @@ class TreeRoutingTable:
                 i += 1
         return i
 
-    def random_id_in_bucket_range(self, bucket_index: int) -> bytes:
+    def _random_id_in_bucket_range(self, bucket_index: int) -> bytes:
         random_id = int(random.randrange(self.buckets[bucket_index].range_min, self.buckets[bucket_index].range_max))
         return Distance(
             self._parent_node_id
         )(random_id.to_bytes(constants.HASH_LENGTH, 'big')).to_bytes(constants.HASH_LENGTH, 'big')
 
-    def midpoint_id_in_bucket_range(self, bucket_index: int) -> bytes:
+    def _midpoint_id_in_bucket_range(self, bucket_index: int) -> bytes:
         half = int((self.buckets[bucket_index].range_max - self.buckets[bucket_index].range_min) // 2)
         return Distance(self._parent_node_id)(
             int(self.buckets[bucket_index].range_min + half).to_bytes(constants.HASH_LENGTH, 'big')
         ).to_bytes(constants.HASH_LENGTH, 'big')
 
-    def split_bucket(self, old_bucket_index: int) -> None:
+    def _split_bucket(self, old_bucket_index: int) -> None:
         """ Splits the specified k-bucket into two new buckets which together
         cover the same range in the key/ID space
 
@@ -304,7 +304,7 @@ class TreeRoutingTable:
             old_bucket.remove_peer(contact)
         self.bucket_in_routing_table_metric.labels("global").set(len(self.buckets))
 
-    def join_buckets(self):
+    def _join_buckets(self):
         if len(self.buckets) == 1:
             return
         to_pop = [i for i, bucket in enumerate(self.buckets) if len(bucket) == 0]
@@ -327,14 +327,7 @@ class TreeRoutingTable:
             self.buckets[bucket_index_to_pop + 1].range_min = bucket.range_min
         self.buckets.remove(bucket)
         self.bucket_in_routing_table_metric.labels("global").set(len(self.buckets))
-        return self.join_buckets()
-
-    def contact_in_routing_table(self, address_tuple: typing.Tuple[str, int]) -> bool:
-        for bucket in self.buckets:
-            for contact in bucket.get_peers(sort_distance_to=False):
-                if address_tuple[0] == contact.address and address_tuple[1] == contact.udp_port:
-                    return True
-        return False
+        return self._join_buckets()
 
     def buckets_with_contacts(self) -> int:
         count = 0
@@ -342,3 +335,70 @@ class TreeRoutingTable:
             if len(bucket) > 0:
                 count += 1
         return count
+
+    async def add_peer(self, peer: 'KademliaPeer', probe: typing.Callable[['KademliaPeer'], typing.Awaitable]):
+        if not peer.node_id:
+            log.warning("Tried adding a peer with no node id!")
+            return False
+        for my_peer in self.get_peers():
+            if (my_peer.address, my_peer.udp_port) == (peer.address, peer.udp_port) and my_peer.node_id != peer.node_id:
+                self.remove_peer(my_peer)
+                self._join_buckets()
+        bucket_index = self._kbucket_index(peer.node_id)
+        if self.buckets[bucket_index].add_peer(peer):
+            return True
+
+        # The bucket is full; see if it can be split (by checking if its range includes the host node's node_id)
+        if self._should_split(bucket_index, peer.node_id):
+            self._split_bucket(bucket_index)
+            # Retry the insertion attempt
+            result = await self.add_peer(peer, probe)
+            self._join_buckets()
+            return result
+        else:
+            # We can't split the k-bucket
+            #
+            # The 13 page kademlia paper specifies that the least recently contacted node in the bucket
+            # shall be pinged. If it fails to reply it is replaced with the new contact. If the ping is successful
+            # the new contact is ignored and not added to the bucket (sections 2.2 and 2.4).
+            #
+            # A reasonable extension to this is BEP 0005, which extends the above:
+            #
+            #    Not all nodes that we learn about are equal. Some are "good" and some are not.
+            #    Many nodes using the DHT are able to send queries and receive responses,
+            #    but are not able to respond to queries from other nodes. It is important that
+            #    each node's routing table must contain only known good nodes. A good node is
+            #    a node has responded to one of our queries within the last 15 minutes. A node
+            #    is also good if it has ever responded to one of our queries and has sent us a
+            #    query within the last 15 minutes. After 15 minutes of inactivity, a node becomes
+            #    questionable. Nodes become bad when they fail to respond to multiple queries
+            #    in a row. Nodes that we know are good are given priority over nodes with unknown status.
+            #
+            # When there are bad or questionable nodes in the bucket, the least recent is selected for
+            # potential replacement (BEP 0005). When all nodes in the bucket are fresh, the head (least recent)
+            # contact is selected as described in section 2.2 of the kademlia paper. In both cases the new contact
+            # is ignored if the pinged node replies.
+
+            not_good_contacts = self.buckets[bucket_index].get_bad_or_unknown_peers()
+            not_recently_replied = []
+            for my_peer in not_good_contacts:
+                last_replied = self._peer_manager.get_last_replied(my_peer.address, my_peer.udp_port)
+                if not last_replied or last_replied + 60 < self._loop.time():
+                    not_recently_replied.append(my_peer)
+            if not_recently_replied:
+                to_replace = not_recently_replied[0]
+            else:
+                to_replace = self.buckets[bucket_index].peers[0]
+                last_replied = self._peer_manager.get_last_replied(to_replace.address, to_replace.udp_port)
+                if last_replied and last_replied + 60 > self._loop.time():
+                    return False
+            log.debug("pinging %s:%s", to_replace.address, to_replace.udp_port)
+            try:
+                await probe(to_replace)
+                return False
+            except (asyncio.TimeoutError, RemoteException):
+                log.debug("Replacing dead contact in bucket %i: %s:%i with %s:%i ", bucket_index,
+                          to_replace.address, to_replace.udp_port, peer.address, peer.udp_port)
+                if to_replace in self.buckets[bucket_index]:
+                    self.buckets[bucket_index].remove_peer(to_replace)
+                return await self.add_peer(peer, probe)

lbry/extras/daemon/client.py

@@ -1,5 +1,5 @@
-from lbry.conf import Config
 from lbry.extras.cli import execute_command
+from lbry.conf import Config
 
 
 def daemon_rpc(conf: Config, method: str, **kwargs):

lbry/extras/daemon/component.py

@@ -37,7 +37,7 @@ class Component(metaclass=ComponentType):
     def running(self):
         return self._running
 
-    async def get_status(self):
+    async def get_status(self): # pylint: disable=no-self-use
         return
 
     async def start(self):

lbry/extras/daemon/componentmanager.py

@@ -42,7 +42,7 @@ class ComponentManager:
         self.analytics_manager = analytics_manager
         self.component_classes = {}
         self.components = set()
-        self.started = asyncio.Event(loop=self.loop)
+        self.started = asyncio.Event()
         self.peer_manager = peer_manager or PeerManager(asyncio.get_event_loop_policy().get_event_loop())
 
         for component_name, component_class in self.default_component_classes.items():
@@ -118,7 +118,7 @@ class ComponentManager:
                 component._setup() for component in stage if not component.running
             ]
             if needing_start:
-                await asyncio.wait(needing_start)
+                await asyncio.wait(map(asyncio.create_task, needing_start))
         self.started.set()
 
     async def stop(self):
@@ -131,7 +131,7 @@ class ComponentManager:
                 component._stop() for component in stage if component.running
             ]
             if needing_stop:
-                await asyncio.wait(needing_stop)
+                await asyncio.wait(map(asyncio.create_task, needing_stop))
 
     def all_components_running(self, *component_names):
         """

lbry/extras/daemon/components.py

@@ -27,10 +27,8 @@ from lbry.extras.daemon.storage import SQLiteStorage
 from lbry.torrent.torrent_manager import TorrentManager
 from lbry.wallet import WalletManager
 from lbry.wallet.usage_payment import WalletServerPayer
-try:
-    from lbry.torrent.session import TorrentSession
-except ImportError:
-    TorrentSession = None
+from lbry.torrent.tracker import TrackerClient
+from lbry.torrent.session import TorrentSession
 
 log = logging.getLogger(__name__)
 
@@ -48,6 +46,7 @@ BACKGROUND_DOWNLOADER_COMPONENT = "background_downloader"
 PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
 UPNP_COMPONENT = "upnp"
 EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager"
+TRACKER_ANNOUNCER_COMPONENT = "tracker_announcer_component"
 LIBTORRENT_COMPONENT = "libtorrent_component"
 
 
@@ -294,6 +293,7 @@ class DHTComponent(Component):
             peer_port=self.external_peer_port,
             rpc_timeout=self.conf.node_rpc_timeout,
             split_buckets_under_index=self.conf.split_buckets_under_index,
+            is_bootstrap_node=self.conf.is_bootstrap_node,
             storage=storage
         )
         self.dht_node.start(self.conf.network_interface, self.conf.known_dht_nodes)
@@ -357,10 +357,6 @@ class FileManagerComponent(Component):
         wallet = self.component_manager.get_component(WALLET_COMPONENT)
         node = self.component_manager.get_component(DHT_COMPONENT) \
             if self.component_manager.has_component(DHT_COMPONENT) else None
-        try:
-            torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT) if TorrentSession else None
-        except NameError:
-            torrent = None
         log.info('Starting the file manager')
         loop = asyncio.get_event_loop()
         self.file_manager = FileManager(
@@ -369,7 +365,8 @@ class FileManagerComponent(Component):
         self.file_manager.source_managers['stream'] = StreamManager(
             loop, self.conf, blob_manager, wallet, storage, node,
         )
-        if TorrentSession and LIBTORRENT_COMPONENT not in self.conf.components_to_skip:
+        if self.component_manager.has_component(LIBTORRENT_COMPONENT):
+            torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT)
             self.file_manager.source_managers['torrent'] = TorrentManager(
                 loop, self.conf, torrent, storage, self.component_manager.analytics_manager
             )
@@ -377,10 +374,11 @@ class FileManagerComponent(Component):
         log.info('Done setting up file manager')
 
     async def stop(self):
-        self.file_manager.stop()
+        await self.file_manager.stop()
 
 
 class BackgroundDownloaderComponent(Component):
+    MIN_PREFIX_COLLIDING_BITS = 8
     component_name = BACKGROUND_DOWNLOADER_COMPONENT
     depends_on = [DATABASE_COMPONENT, BLOB_COMPONENT, DISK_SPACE_COMPONENT]
 
@@ -412,12 +410,18 @@ class BackgroundDownloaderComponent(Component):
         while True:
             self.space_available = await self.space_manager.get_free_space_mb(True)
             if not self.is_busy and self.space_available > 10:
-                blob_hash = next((key.hex() for key in self.dht_node.stored_blob_hashes if
-                                 key.hex() not in self.blob_manager.completed_blob_hashes), None)
-                if blob_hash:
-                    self.ongoing_download = asyncio.create_task(self.background_downloader.download_blobs(blob_hash))
+                self._download_next_close_blob_hash()
             await asyncio.sleep(self.download_loop_delay_seconds)
 
+    def _download_next_close_blob_hash(self):
+        node_id = self.dht_node.protocol.node_id
+        for blob_hash in self.dht_node.stored_blob_hashes:
+            if blob_hash.hex() in self.blob_manager.completed_blob_hashes:
+                continue
+            if utils.get_colliding_prefix_bits(node_id, blob_hash) >= self.MIN_PREFIX_COLLIDING_BITS:
+                self.ongoing_download = asyncio.create_task(self.background_downloader.download_blobs(blob_hash.hex()))
+                return
+
     async def start(self):
         self.space_manager: DiskSpaceManager = self.component_manager.get_component(DISK_SPACE_COMPONENT)
         if not self.component_manager.has_component(DHT_COMPONENT):
@@ -491,9 +495,8 @@ class TorrentComponent(Component):
         }
 
     async def start(self):
-        if TorrentSession:
-            self.torrent_session = TorrentSession(asyncio.get_event_loop(), None)
-            await self.torrent_session.bind()  # TODO: specify host/port
+        self.torrent_session = TorrentSession(asyncio.get_event_loop(), None)
+        await self.torrent_session.bind()  # TODO: specify host/port
 
     async def stop(self):
         if self.torrent_session:
@@ -548,7 +551,7 @@ class UPnPComponent(Component):
         while True:
             if now:
                 await self._maintain_redirects()
-            await asyncio.sleep(360, loop=self.component_manager.loop)
+            await asyncio.sleep(360)
 
     async def _maintain_redirects(self):
         # setup the gateway if necessary
@@ -557,8 +560,6 @@ class UPnPComponent(Component):
                 self.upnp = await UPnP.discover(loop=self.component_manager.loop)
                 log.info("found upnp gateway: %s", self.upnp.gateway.manufacturer_string)
             except Exception as err:
-                if isinstance(err, asyncio.CancelledError):  # TODO: remove when updated to 3.8
-                    raise
                 log.warning("upnp discovery failed: %s", err)
                 self.upnp = None
 
@@ -670,7 +671,7 @@ class UPnPComponent(Component):
             log.info("Removing upnp redirects: %s", self.upnp_redirects)
             await asyncio.wait([
                 self.upnp.delete_port_mapping(port, protocol) for protocol, port in self.upnp_redirects.items()
-            ], loop=self.component_manager.loop)
+            ])
         if self._maintain_redirects_task and not self._maintain_redirects_task.done():
             self._maintain_redirects_task.cancel()
 
@@ -701,3 +702,49 @@ class ExchangeRateManagerComponent(Component):
 
     async def stop(self):
         self.exchange_rate_manager.stop()
+
+
+class TrackerAnnouncerComponent(Component):
+    component_name = TRACKER_ANNOUNCER_COMPONENT
+    depends_on = [FILE_MANAGER_COMPONENT]
+
+    def __init__(self, component_manager):
+        super().__init__(component_manager)
+        self.file_manager = None
+        self.announce_task = None
+        self.tracker_client: typing.Optional[TrackerClient] = None
+
+    @property
+    def component(self):
+        return self.tracker_client
+
+    @property
+    def running(self):
+        return self._running and self.announce_task and not self.announce_task.done()
+
+    async def announce_forever(self):
+        while True:
+            sleep_seconds = 60.0
+            announce_sd_hashes = []
+            for file in self.file_manager.get_filtered():
+                if not file.downloader:
+                    continue
+                announce_sd_hashes.append(bytes.fromhex(file.sd_hash))
+            await self.tracker_client.announce_many(*announce_sd_hashes)
+            await asyncio.sleep(sleep_seconds)
+
+    async def start(self):
+        node = self.component_manager.get_component(DHT_COMPONENT) \
+            if self.component_manager.has_component(DHT_COMPONENT) else None
+        node_id = node.protocol.node_id if node else None
+        self.tracker_client = TrackerClient(node_id, self.conf.tcp_port, lambda: self.conf.tracker_servers)
+        await self.tracker_client.start()
+        self.file_manager = self.component_manager.get_component(FILE_MANAGER_COMPONENT)
+        self.announce_task = asyncio.create_task(self.announce_forever())
+
+    async def stop(self):
+        self.file_manager = None
+        if self.announce_task and not self.announce_task.done():
+            self.announce_task.cancel()
+        self.announce_task = None
+        self.tracker_client.stop()

lbry/extras/daemon/daemon.py

@@ -9,7 +9,7 @@ import inspect
 import typing
 import random
 import tracemalloc
-from decimal import Decimal
+import itertools
 from urllib.parse import urlencode, quote
 from typing import Callable, Optional, List
 from binascii import hexlify, unhexlify
@@ -27,7 +27,8 @@ from lbry.wallet import (
 )
 from lbry.wallet.dewies import dewies_to_lbc, lbc_to_dewies, dict_values_to_lbc
 from lbry.wallet.constants import TXO_TYPES, CLAIM_TYPE_NAMES
-from scribe.schema.bip32 import PrivateKey
+from lbry.wallet.bip32 import PrivateKey
+from lbry.crypto.base58 import Base58
 
 from lbry import utils
 from lbry.conf import Config, Setting, NOT_SET
@@ -43,7 +44,7 @@ from lbry.error import (
 from lbry.extras import system_info
 from lbry.extras.daemon import analytics
 from lbry.extras.daemon.components import WALLET_COMPONENT, DATABASE_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT
-from lbry.extras.daemon.components import FILE_MANAGER_COMPONENT, DISK_SPACE_COMPONENT
+from lbry.extras.daemon.components import FILE_MANAGER_COMPONENT, DISK_SPACE_COMPONENT, TRACKER_ANNOUNCER_COMPONENT
 from lbry.extras.daemon.components import EXCHANGE_RATE_MANAGER_COMPONENT, UPNP_COMPONENT
 from lbry.extras.daemon.componentmanager import RequiredCondition
 from lbry.extras.daemon.componentmanager import ComponentManager
@@ -51,8 +52,8 @@ from lbry.extras.daemon.json_response_encoder import JSONResponseEncoder
 from lbry.extras.daemon.undecorated import undecorated
 from lbry.extras.daemon.security import ensure_request_allowed
 from lbry.file_analysis import VideoFileAnalyzer
-from scribe.schema.claim import Claim
-from scribe.schema.url import URL, normalize_name
+from lbry.schema.claim import Claim
+from lbry.schema.url import URL
 
 
 if typing.TYPE_CHECKING:
@@ -194,61 +195,6 @@ def paginate_list(items: List, page: Optional[int], page_size: Optional[int]):
     }
 
 
-def fix_kwargs_for_hub(**kwargs):
-    repeated_fields = {"media_type", "stream_type", "claim_type"}
-    value_fields = {"tx_nout", "has_source", "is_signature_valid"}
-    opcodes = {'=': 0, '<=': 1, '>=': 2, '<': 3, '>': 4}
-    for key, value in list(kwargs.items()):
-        if value in (None, [], False):
-            kwargs.pop(key)
-            continue
-        if key in REPLACEMENTS:
-            kwargs[REPLACEMENTS[key]] = kwargs.pop(key)
-            key = REPLACEMENTS[key]
-
-        if key == "normalized_name":
-            kwargs[key] = normalize_name(value)
-        if key == "limit_claims_per_channel":
-            value = kwargs.pop("limit_claims_per_channel") or 0
-            if value > 0:
-                kwargs["limit_claims_per_channel"] = value
-        elif key == "invalid_channel_signature":
-            kwargs["is_signature_valid"] = {"value": not kwargs.pop("invalid_channel_signature")}
-        elif key == "has_no_source":
-            kwargs["has_source"] = {"value": not kwargs.pop("has_no_source")}
-        elif key in value_fields:
-            kwargs[key] = {"value": value} if not isinstance(value, dict) else value
-        elif key in repeated_fields and isinstance(value, str):
-            kwargs[key] = [value]
-        elif key in ("claim_id", "channel_id"):
-            kwargs[key] = {"invert": False, "value": [kwargs[key]]}
-        elif key in ("claim_ids", "channel_ids"):
-            kwargs[key[:-1]] = {"invert": False, "value": kwargs.pop(key)}
-        elif key == "not_channel_ids":
-            kwargs["channel_id"] = {"invert": True, "value": kwargs.pop("not_channel_ids")}
-        elif key in MY_RANGE_FIELDS:
-            constraints = []
-            for val in value if isinstance(value, list) else [value]:
-                operator = '='
-                if isinstance(val, str) and val[0] in opcodes:
-                    operator_length = 2 if val[:2] in opcodes else 1
-                    operator, val = val[:operator_length], val[operator_length:]
-                val = [int(val if key != 'fee_amount' else Decimal(val)*1000)]
-                constraints.append({"op": opcodes[operator], "value": val})
-            kwargs[key] = constraints
-        elif key == 'order_by':  # TODO: remove this after removing support for old trending args from the api
-            value = value if isinstance(value, list) else [value]
-            new_value = []
-            for new_v in value:
-                migrated = new_v if new_v not in (
-                    'trending_mixed', 'trending_local', 'trending_global', 'trending_group'
-                ) else 'trending_score'
-                if migrated not in new_value:
-                    new_value.append(migrated)
-            kwargs[key] = new_value
-    return kwargs
-
-
 DHT_HAS_CONTACTS = "dht_has_contacts"
 
 
@@ -668,7 +614,8 @@ class Daemon(metaclass=JSONRPCServerType):
             content_type='application/json'
         )
 
-    async def handle_metrics_get_request(self, request: web.Request):
+    @staticmethod
+    async def handle_metrics_get_request(request: web.Request):
         try:
             return web.Response(
                 text=prom_generate_latest().decode(),
@@ -1381,6 +1328,65 @@ class Daemon(metaclass=JSONRPCServerType):
                 c.wallets += [wallet_id]
         return wallet
 
+    @requires("wallet")
+    async def jsonrpc_wallet_export(self, password=None, wallet_id=None):
+        """
+        Exports encrypted wallet data if password is supplied; otherwise plain JSON.
+
+        Wallet must be unlocked to perform this operation.
+
+        Usage:
+            wallet_export [--password=<password>] [--wallet_id=<wallet_id>]
+
+        Options:
+            --password=<password>         : (str) password to encrypt outgoing data
+            --wallet_id=<wallet_id>       : (str) wallet being exported
+
+        Returns:
+            (str) data: base64-encoded encrypted wallet, or cleartext JSON
+
+        """
+        wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
+        if password is None:
+            return wallet.to_json()
+        return wallet.pack(password).decode()
+
+    @requires("wallet")
+    async def jsonrpc_wallet_import(self, data, password=None, wallet_id=None, blocking=False):
+        """
+        Import wallet data and merge accounts and preferences. Data is expected to be JSON if
+        password is not supplied.
+
+        Wallet must be unlocked to perform this operation.
+
+        Usage:
+            wallet_import (<data> | --data=<data>) [<password> | --password=<password>]
+                          [--wallet_id=<wallet_id>] [--blocking]
+
+        Options:
+            --data=<data>                 : (str) incoming wallet data
+            --password=<password>         : (str) password to decrypt incoming data
+            --wallet_id=<wallet_id>       : (str) wallet being merged into
+            --blocking                    : (bool) wait until any new accounts have merged
+
+        Returns:
+            (str) base64-encoded encrypted wallet, or cleartext JSON
+        """
+        wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
+        added_accounts, merged_accounts = wallet.merge(self.wallet_manager, password, data)
+        for new_account in itertools.chain(added_accounts, merged_accounts):
+            await new_account.maybe_migrate_certificates()
+        if added_accounts and self.ledger.network.is_connected:
+            if blocking:
+                await asyncio.wait([
+                    a.ledger.subscribe_account(a) for a in added_accounts
+                ])
+            else:
+                for new_account in added_accounts:
+                    asyncio.create_task(self.ledger.subscribe_account(new_account))
+        wallet.save()
+        return await self.jsonrpc_wallet_export(password=password, wallet_id=wallet_id)
+
     @requires("wallet")
     async def jsonrpc_wallet_add(self, wallet_id):
         """
@@ -1849,7 +1855,7 @@ class Daemon(metaclass=JSONRPCServerType):
         Options:
             --to_account=<to_account>     : (str) send to this account
             --from_account=<from_account> : (str) spend from this account
-            --amount=<amount>             : (str) the amount to transfer lbc
+            --amount=<amount>             : (decimal) the amount to transfer lbc
             --everything                  : (bool) transfer everything (excluding claims), default: false.
             --outputs=<outputs>           : (int) split payment across many outputs, default: 1.
             --wallet_id=<wallet_id>       : (str) limit operation to specific wallet.
@@ -1872,6 +1878,48 @@ class Daemon(metaclass=JSONRPCServerType):
             outputs=outputs, broadcast=broadcast
         )
 
+    @requires("wallet")
+    async def jsonrpc_account_deposit(
+        self, txid, nout, redeem_script, private_key,
+        to_account=None, wallet_id=None, preview=False, blocking=False
+    ):
+        """
+        Spend a time locked transaction into your account.
+
+        Usage:
+            account_deposit <txid> <nout> <redeem_script> <private_key>
+                [<to_account> | --to_account=<to_account>]
+                [--wallet_id=<wallet_id>] [--preview] [--blocking]
+
+        Options:
+            --txid=<txid>                   : (str) id of the transaction
+            --nout=<nout>                   : (int) output number in the transaction
+            --redeem_script=<redeem_script> : (str) redeem script for output
+            --private_key=<private_key>     : (str) private key to sign transaction
+            --to_account=<to_account>       : (str) deposit to this account
+            --wallet_id=<wallet_id>         : (str) limit operation to specific wallet.
+            --preview                       : (bool) do not broadcast the transaction
+            --blocking                      : (bool) wait until tx has synced
+
+        Returns: {Transaction}
+        """
+        wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
+        account = wallet.get_account_or_default(to_account)
+        other_tx = await self.wallet_manager.get_transaction(txid)
+        tx = await Transaction.spend_time_lock(
+            other_tx.outputs[nout], unhexlify(redeem_script), account
+        )
+        pk = PrivateKey.from_bytes(
+            account.ledger, Base58.decode_check(private_key)[1:-1]
+        )
+        await tx.sign([account], {pk.address: pk})
+        if not preview:
+            await self.broadcast_or_release(tx, blocking)
+            self.component_manager.loop.create_task(self.analytics_manager.send_credits_sent())
+        else:
+            await self.ledger.release_tx(tx)
+        return tx
+
     @requires(WALLET_COMPONENT)
     def jsonrpc_account_send(self, amount, addresses, account_id=None, wallet_id=None, preview=False, blocking=False):
         """
@@ -1943,7 +1991,9 @@ class Daemon(metaclass=JSONRPCServerType):
         wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
         wallet_changed = False
         if data is not None:
-            added_accounts = wallet.merge(self.wallet_manager, password, data)
+            added_accounts, merged_accounts = wallet.merge(self.wallet_manager, password, data)
+            for new_account in itertools.chain(added_accounts, merged_accounts):
+                await new_account.maybe_migrate_certificates()
             if added_accounts and self.ledger.network.is_connected:
                 if blocking:
                     await asyncio.wait([
@@ -2360,6 +2410,7 @@ class Daemon(metaclass=JSONRPCServerType):
 
         Usage:
             claim_list [--claim_type=<claim_type>...] [--claim_id=<claim_id>...] [--name=<name>...] [--is_spent]
+                       [--reposted_claim_id=<reposted_claim_id>...]
                        [--channel_id=<channel_id>...] [--account_id=<account_id>] [--wallet_id=<wallet_id>]
                        [--has_source | --has_no_source] [--page=<page>] [--page_size=<page_size>]
                        [--resolve] [--order_by=<order_by>] [--no_totals] [--include_received_tips]
@@ -2370,6 +2421,7 @@ class Daemon(metaclass=JSONRPCServerType):
             --channel_id=<channel_id>  : (str or list) streams in this channel
             --name=<name>              : (str or list) claim name
             --is_spent                 : (bool) shows previous claim updates and abandons
+            --reposted_claim_id=<reposted_claim_id> : (str or list) reposted claim id
             --account_id=<account_id>  : (str) id of the account to query
             --wallet_id=<wallet_id>    : (str) restrict results to specific wallet
             --has_source               : (bool) list claims containing a source field
@@ -2567,42 +2619,27 @@ class Daemon(metaclass=JSONRPCServerType):
 
         Returns: {Paginated[Output]}
         """
-        if self.ledger.config.get('use_go_hub'):
-            host = self.ledger.network.client.server[0]
-            port = "50051"
-            kwargs['new_sdk_server'] = f"{host}:{port}"
-            if kwargs.get("channel"):
-                channel = kwargs.pop("channel")
-                channel_obj = (await self.jsonrpc_resolve(channel))[channel]
-                if isinstance(channel_obj, dict):
-                    # This happens when the channel doesn't exist
-                    kwargs["channel_id"] = ""
-                else:
-                    kwargs["channel_id"] = channel_obj.claim_id
-            kwargs = fix_kwargs_for_hub(**kwargs)
-        else:
-            # Don't do this if using the hub server, it screws everything up
-            if "claim_ids" in kwargs and not kwargs["claim_ids"]:
-                kwargs.pop("claim_ids")
-            if {'claim_id', 'claim_ids'}.issubset(kwargs):
-                raise ConflictingInputValueError('claim_id', 'claim_ids')
-            if kwargs.pop('valid_channel_signature', False):
-                kwargs['signature_valid'] = 1
-            if kwargs.pop('invalid_channel_signature', False):
-                kwargs['signature_valid'] = 0
-            if 'has_no_source' in kwargs:
-                kwargs['has_source'] = not kwargs.pop('has_no_source')
-            if 'order_by' in kwargs:  # TODO: remove this after removing support for old trending args from the api
-                value = kwargs.pop('order_by')
-                value = value if isinstance(value, list) else [value]
-                new_value = []
-                for new_v in value:
-                    migrated = new_v if new_v not in (
-                        'trending_mixed', 'trending_local', 'trending_global', 'trending_group'
-                    ) else 'trending_score'
-                    if migrated not in new_value:
-                        new_value.append(migrated)
-                kwargs['order_by'] = new_value
+        if "claim_ids" in kwargs and not kwargs["claim_ids"]:
+            kwargs.pop("claim_ids")
+        if {'claim_id', 'claim_ids'}.issubset(kwargs):
+            raise ConflictingInputValueError('claim_id', 'claim_ids')
+        if kwargs.pop('valid_channel_signature', False):
+            kwargs['signature_valid'] = 1
+        if kwargs.pop('invalid_channel_signature', False):
+            kwargs['signature_valid'] = 0
+        if 'has_no_source' in kwargs:
+            kwargs['has_source'] = not kwargs.pop('has_no_source')
+        if 'order_by' in kwargs:  # TODO: remove this after removing support for old trending args from the api
+            value = kwargs.pop('order_by')
+            value = value if isinstance(value, list) else [value]
+            new_value = []
+            for new_v in value:
+                migrated = new_v if new_v not in (
+                    'trending_mixed', 'trending_local', 'trending_global', 'trending_group'
+                ) else 'trending_score'
+                if migrated not in new_value:
+                    new_value.append(migrated)
+            kwargs['order_by'] = new_value
         page_num, page_size = abs(kwargs.pop('page', 1)), min(abs(kwargs.pop('page_size', DEFAULT_PAGE_SIZE)), 50)
         wallet = self.wallet_manager.get_wallet_or_default(kwargs.pop('wallet_id', None))
         kwargs.update({'offset': page_size * (page_num - 1), 'limit': page_size})
@@ -2738,7 +2775,7 @@ class Daemon(metaclass=JSONRPCServerType):
             wallet.save()
             await self.broadcast_or_release(tx, blocking)
             self.component_manager.loop.create_task(self.storage.save_claims([self._old_get_temp_claim_info(
-                tx, txo, claim_address, claim, name, dewies_to_lbc(amount)
+                tx, txo, claim_address, claim, name
             )]))
             self.component_manager.loop.create_task(self.analytics_manager.send_new_channel())
         else:
@@ -2897,7 +2934,7 @@ class Daemon(metaclass=JSONRPCServerType):
             wallet.save()
             await self.broadcast_or_release(tx, blocking)
             self.component_manager.loop.create_task(self.storage.save_claims([self._old_get_temp_claim_info(
-                tx, new_txo, claim_address, new_txo.claim, new_txo.claim_name, dewies_to_lbc(amount)
+                tx, new_txo, claim_address, new_txo.claim, new_txo.claim_name
             )]))
             self.component_manager.loop.create_task(self.analytics_manager.send_new_channel())
         else:
@@ -2907,19 +2944,21 @@ class Daemon(metaclass=JSONRPCServerType):
 
     @requires(WALLET_COMPONENT)
     async def jsonrpc_channel_sign(
-            self, channel_name=None, channel_id=None, hexdata=None, channel_account_id=None, wallet_id=None):
+            self, channel_name=None, channel_id=None, hexdata=None, salt=None,
+            channel_account_id=None, wallet_id=None):
         """
         Signs data using the specified channel signing key.
 
         Usage:
-            channel_sign [<channel_name> | --channel_name=<channel_name>]
-                         [<channel_id> | --channel_id=<channel_id>] [<hexdata> | --hexdata=<hexdata>]
+            channel_sign [<channel_name> | --channel_name=<channel_name>] [<channel_id> | --channel_id=<channel_id>]
+                         [<hexdata> | --hexdata=<hexdata>] [<salt> | --salt=<salt>]
                          [--channel_account_id=<channel_account_id>...] [--wallet_id=<wallet_id>]
 
         Options:
             --channel_name=<channel_name>            : (str) name of channel used to sign (or use channel id)
             --channel_id=<channel_id>                : (str) claim id of channel used to sign (or use channel name)
             --hexdata=<hexdata>                      : (str) data to sign, encoded as hexadecimal
+            --salt=<salt>                            : (str) salt to use for signing, default is to use timestamp
             --channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in
                                                              for channel certificates, defaults to all accounts.
             --wallet_id=<wallet_id>                  : (str) restrict operation to specific wallet
@@ -2936,11 +2975,13 @@ class Daemon(metaclass=JSONRPCServerType):
         signing_channel = await self.get_channel_or_error(
             wallet, channel_account_id, channel_id, channel_name, for_signing=True
         )
-        timestamp = str(int(time.time()))
-        signature = signing_channel.sign_data(unhexlify(str(hexdata)), timestamp)
+        if salt is None:
+            salt = str(int(time.time()))
+        signature = signing_channel.sign_data(unhexlify(str(hexdata)), salt)
         return {
             'signature': signature,
-            'signing_ts': timestamp
+            'signing_ts': salt,  # DEPRECATED
+            'salt': salt,
         }
 
     @requires(WALLET_COMPONENT)
@@ -3258,15 +3299,17 @@ class Daemon(metaclass=JSONRPCServerType):
         )
 
     @requires(WALLET_COMPONENT, FILE_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT)
-    async def jsonrpc_stream_repost(self, name, bid, claim_id, allow_duplicate_name=False, channel_id=None,
-                                    channel_name=None, channel_account_id=None, account_id=None, wallet_id=None,
-                                    claim_address=None, funding_account_ids=None, preview=False, blocking=False):
+    async def jsonrpc_stream_repost(
+            self, name, bid, claim_id, allow_duplicate_name=False, channel_id=None,
+            channel_name=None, channel_account_id=None, account_id=None, wallet_id=None,
+            claim_address=None, funding_account_ids=None, preview=False, blocking=False, **kwargs):
         """
             Creates a claim that references an existing stream by its claim id.
 
             Usage:
                 stream_repost (<name> | --name=<name>) (<bid> | --bid=<bid>) (<claim_id> | --claim_id=<claim_id>)
                         [--allow_duplicate_name=<allow_duplicate_name>]
+                        [--title=<title>] [--description=<description>] [--tags=<tags>...]
                         [--channel_id=<channel_id> | --channel_name=<channel_name>]
                         [--channel_account_id=<channel_account_id>...]
                         [--account_id=<account_id>] [--wallet_id=<wallet_id>]
@@ -3279,6 +3322,9 @@ class Daemon(metaclass=JSONRPCServerType):
                 --claim_id=<claim_id>          : (str) id of the claim being reposted
                 --allow_duplicate_name=<allow_duplicate_name> : (bool) create new claim even if one already exists with
                                                                        given name. default: false.
+                --title=<title>                : (str) title of the repost
+                --description=<description>    : (str) description of the repost
+                --tags=<tags>                  : (list) add repost tags
                 --channel_id=<channel_id>      : (str) claim id of the publisher channel
                 --channel_name=<channel_name>  : (str) name of the publisher channel
                 --channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in
@@ -3313,6 +3359,7 @@ class Daemon(metaclass=JSONRPCServerType):
             raise Exception('Invalid claim id. It is expected to be a 40 characters long hexadecimal string.')
 
         claim = Claim()
+        claim.repost.update(**kwargs)
         claim.repost.reference.claim_id = claim_id
         tx = await Transaction.claim_create(
             name, claim, amount, claim_address, funding_accounts, funding_accounts[0], channel
@@ -3490,7 +3537,7 @@ class Daemon(metaclass=JSONRPCServerType):
 
             async def save_claims():
                 await self.storage.save_claims([self._old_get_temp_claim_info(
-                    tx, new_txo, claim_address, claim, name, dewies_to_lbc(amount)
+                    tx, new_txo, claim_address, claim, name
                 )])
                 if file_path is not None:
                     await self.storage.save_content_claim(file_stream.stream_hash, new_txo.id)
@@ -3693,6 +3740,8 @@ class Daemon(metaclass=JSONRPCServerType):
 
         if old_txo.claim.is_stream:
             claim.stream.update(file_path=file_path, **kwargs)
+        elif old_txo.claim.is_repost:
+            claim.repost.update(**kwargs)
 
         if clear_channel:
             claim.clear_signature()
@@ -3725,7 +3774,7 @@ class Daemon(metaclass=JSONRPCServerType):
 
             async def save_claims():
                 await self.storage.save_claims([self._old_get_temp_claim_info(
-                    tx, new_txo, claim_address, new_txo.claim, new_txo.claim_name, dewies_to_lbc(amount)
+                    tx, new_txo, claim_address, new_txo.claim, new_txo.claim_name
                 )])
                 if stream_hash:
                     await self.storage.save_content_claim(stream_hash, new_txo.id)
@@ -3987,6 +4036,8 @@ class Daemon(metaclass=JSONRPCServerType):
                            [--languages=<languages>...] [--clear_languages]
                            [--locations=<locations>...] [--clear_locations]
                            [--thumbnail_url=<thumbnail_url>] [--cover_url=<cover_url>]
+                           [--channel_id=<channel_id> | --channel_name=<channel_name>]
+                           [--channel_account_id=<channel_account_id>...]
                            [--account_id=<account_id>] [--wallet_id=<wallet_id>]
                            [--claim_address=<claim_address>]
                            [--funding_account_ids=<funding_account_ids>...]
@@ -4042,6 +4093,10 @@ class Daemon(metaclass=JSONRPCServerType):
 
             --clear_locations              : (bool) clear existing locations (prior to adding new ones)
             --thumbnail_url=<thumbnail_url>: (str) thumbnail url
+            --channel_id=<channel_id>      : (str) claim id of the publisher channel
+            --channel_name=<channel_name>  : (str) name of the publisher channel
+            --channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in
+                                                   for channel certificates, defaults to all accounts.
             --account_id=<account_id>      : (str) account in which to look for collection (default: all)
             --wallet_id=<wallet_id>        : (str) restrict operation to specific wallet
           --funding_account_ids=<funding_account_ids>: (list) ids of accounts to fund this transaction
@@ -4289,7 +4344,7 @@ class Daemon(metaclass=JSONRPCServerType):
                 'nout': tx.position,
                 'address': claim_address,
                 'claim_id': claim_id,
-                'amount': dewies_to_lbc(amount)
+                'amount': dewies_to_lbc(new_txo.amount)
             }]})
             self.component_manager.loop.create_task(self.analytics_manager.send_claim_action('new_support'))
         else:
@@ -4906,21 +4961,16 @@ class Daemon(metaclass=JSONRPCServerType):
     DHT / Blob Exchange peer commands.
     """
 
-    @requires(DHT_COMPONENT)
-    async def jsonrpc_peer_list(self, blob_hash, search_bottom_out_limit=None, page=None, page_size=None):
+    async def jsonrpc_peer_list(self, blob_hash, page=None, page_size=None):
         """
         Get peers for blob hash
 
         Usage:
             peer_list (<blob_hash> | --blob_hash=<blob_hash>)
-                [<search_bottom_out_limit> | --search_bottom_out_limit=<search_bottom_out_limit>]
                 [--page=<page>] [--page_size=<page_size>]
 
         Options:
             --blob_hash=<blob_hash>                                  : (str) find available peers for this blob hash
-            --search_bottom_out_limit=<search_bottom_out_limit>      : (int) the number of search probes in a row
-                                                                             that don't find any new peers
-                                                                             before giving up and returning
             --page=<page>                                            : (int) page to return during paginating
             --page_size=<page_size>                                  : (int) number of items on page during pagination
 
@@ -4932,28 +4982,29 @@ class Daemon(metaclass=JSONRPCServerType):
         if not is_valid_blobhash(blob_hash):
             # TODO: use error from lbry.error
             raise Exception("invalid blob hash")
-        if search_bottom_out_limit is not None:
-            search_bottom_out_limit = int(search_bottom_out_limit)
-            if search_bottom_out_limit <= 0:
-                # TODO: use error from lbry.error
-                raise Exception("invalid bottom out limit")
-        else:
-            search_bottom_out_limit = 4
-        peers = []
         peer_q = asyncio.Queue(loop=self.component_manager.loop)
-        await self.dht_node._peers_for_value_producer(blob_hash, peer_q)
+        if self.component_manager.has_component(TRACKER_ANNOUNCER_COMPONENT):
+            tracker = self.component_manager.get_component(TRACKER_ANNOUNCER_COMPONENT)
+            tracker_peers = await tracker.get_kademlia_peer_list(bytes.fromhex(blob_hash))
+            log.info("Found %d peers for %s from trackers.", len(tracker_peers), blob_hash[:8])
+            peer_q.put_nowait(tracker_peers)
+        elif not self.component_manager.has_component(DHT_COMPONENT):
+            raise Exception("Peer list needs, at least, either a DHT component or a Tracker component for discovery.")
+        peers = []
+        if self.component_manager.has_component(DHT_COMPONENT):
+            await self.dht_node._peers_for_value_producer(blob_hash, peer_q)
         while not peer_q.empty():
             peers.extend(peer_q.get_nowait())
-        results = [
-            {
-                "node_id": hexlify(peer.node_id).decode(),
+        results = {
+            (peer.address, peer.tcp_port): {
+                "node_id": hexlify(peer.node_id).decode() if peer.node_id else None,
                 "address": peer.address,
                 "udp_port": peer.udp_port,
                 "tcp_port": peer.tcp_port,
             }
             for peer in peers
-        ]
-        return paginate_list(results, page, page_size)
+        }
+        return paginate_list(list(results.values()), page, page_size)
 
     @requires(DATABASE_COMPONENT)
     async def jsonrpc_blob_announce(self, blob_hash=None, stream_hash=None, sd_hash=None):
@@ -5423,11 +5474,11 @@ class Daemon(metaclass=JSONRPCServerType):
         return results
 
     @staticmethod
-    def _old_get_temp_claim_info(tx, txo, address, claim_dict, name, bid):
+    def _old_get_temp_claim_info(tx, txo, address, claim_dict, name):
         return {
             "claim_id": txo.claim_id,
             "name": name,
-            "amount": bid,
+            "amount": dewies_to_lbc(txo.amount),
             "address": address,
             "txid": tx.id,
             "nout": txo.position,

lbry/extras/daemon/exchange_rate_manager.py

@@ -80,8 +80,6 @@ class MarketFeed:
             self.rate = ExchangeRate(self.market, rate, int(time.time()))
             self.last_check = time.time()
             return self.rate
-        except asyncio.CancelledError:
-            raise
         except asyncio.TimeoutError:
             log.warning("Timed out fetching exchange rate from %s.", self.name)
         except json.JSONDecodeError as e:
@@ -196,9 +194,9 @@ FEEDS: Iterable[Type[MarketFeed]] = (
     BittrexUSDFeed,
     CoinExBTCFeed,
     CoinExUSDFeed,
-    HotbitBTCFeed,
-    HotbitUSDFeed,
-    UPbitBTCFeed,
+#    HotbitBTCFeed,
+#    HotbitUSDFeed,
+#    UPbitBTCFeed,
 )
 
 

lbry/extras/daemon/json_response_encoder.py

@@ -6,11 +6,11 @@ from json import JSONEncoder
 
 from google.protobuf.message import DecodeError
 
-from scribe.schema.claim import Claim
-from scribe.schema.support import Support
-from scribe.schema.bip32 import PublicKey
+from lbry.schema.claim import Claim
+from lbry.schema.support import Support
 from lbry.torrent.torrent_manager import TorrentSource
 from lbry.wallet import Wallet, Ledger, Account, Transaction, Output
+from lbry.wallet.bip32 import PublicKey
 from lbry.wallet.dewies import dewies_to_lbc
 from lbry.stream.managed_stream import ManagedStream
 
@@ -328,8 +328,8 @@ class JSONResponseEncoder(JSONEncoder):
             result.update({
                 'streaming_url': managed_stream.stream_url,
                 'stream_hash': managed_stream.stream_hash,
-                'stream_name': managed_stream.descriptor.stream_name,
-                'suggested_file_name': managed_stream.descriptor.suggested_file_name,
+                'stream_name': managed_stream.stream_name,
+                'suggested_file_name': managed_stream.suggested_file_name,
                 'sd_hash': managed_stream.descriptor.sd_hash,
                 'mime_type': managed_stream.mime_type,
                 'key': managed_stream.descriptor.key,

lbry/extras/daemon/migrator/migrate5to6.py

@@ -3,7 +3,7 @@ import os
 import json
 import logging
 from binascii import hexlify
-from scribe.schema.claim import Claim
+from lbry.schema.claim import Claim
 
 log = logging.getLogger(__name__)
 

lbry/extras/daemon/migrator/migrate8to9.py

@@ -20,7 +20,7 @@ def do_migration(conf):
                            "left outer join blob b ON b.blob_hash=s.blob_hash order by s.position").fetchall()
     blobs_by_stream = {}
     for stream_hash, position, iv, blob_hash, blob_length in blobs:
-        blobs_by_stream.setdefault(stream_hash, []).append(BlobInfo(position, blob_length or 0, iv, blob_hash))
+        blobs_by_stream.setdefault(stream_hash, []).append(BlobInfo(position, blob_length or 0, iv, 0, blob_hash))
 
     for stream_name, stream_key, suggested_filename, sd_hash, stream_hash in streams:
         sd = StreamDescriptor(None, blob_dir, stream_name, stream_key, suggested_filename,

lbry/extras/daemon/storage.py

@@ -10,7 +10,7 @@ from lbry.wallet import SQLiteMixin
 from lbry.conf import Config
 from lbry.wallet.dewies import dewies_to_lbc, lbc_to_dewies
 from lbry.wallet.transaction import Transaction, Output
-from scribe.schema.claim import Claim
+from lbry.schema.claim import Claim
 from lbry.dht.constants import DATA_EXPIRATION
 from lbry.blob.blob_info import BlobInfo
 
@@ -449,7 +449,8 @@ class SQLiteStorage(SQLiteMixin):
             return await self.db.execute_fetchall(
                 "select blob.blob_hash, blob.blob_length, blob.added_on "
                 "from blob left join stream_blob using (blob_hash) "
-                "where stream_blob.stream_hash is null and blob.is_mine=? order by blob.added_on asc",
+                "where stream_blob.stream_hash is null and blob.is_mine=? and blob.status='finished'"
+                "order by blob.blob_length desc, blob.added_on asc",
                 (is_mine,)
             )
 
@@ -462,7 +463,8 @@ class SQLiteStorage(SQLiteMixin):
         content_blobs = await self.db.execute_fetchall(
             "select blob.blob_hash, blob.blob_length, blob.added_on "
             "from blob join stream_blob using (blob_hash) cross join stream using (stream_hash)"
-            "cross join file using (stream_hash) where blob.is_mine=? order by blob.added_on asc, blob.blob_length asc",
+            "cross join file using (stream_hash)"
+            "where blob.is_mine=? and blob.status='finished' order by blob.added_on asc, blob.blob_length asc",
             (is_mine,)
         )
         return content_blobs + sd_blobs
@@ -480,6 +482,7 @@ class SQLiteStorage(SQLiteMixin):
                    is_mine=1
                then blob_length else 0 end), 0) as private_storage
         from blob left join stream_blob using (blob_hash)
+        where blob_hash not in (select sd_hash from stream) and blob.status="finished"
         """)
         return {
             'network_storage': network_size,
@@ -531,7 +534,8 @@ class SQLiteStorage(SQLiteMixin):
         def _get_blobs_for_stream(transaction):
             crypt_blob_infos = []
             stream_blobs = transaction.execute(
-                "select blob_hash, position, iv from stream_blob where stream_hash=? "
+                "select s.blob_hash, s.position, s.iv, b.added_on "
+                "from stream_blob s left outer join blob b on b.blob_hash=s.blob_hash where stream_hash=? "
                 "order by position asc", (stream_hash, )
             ).fetchall()
             if only_completed:
@@ -551,9 +555,10 @@ class SQLiteStorage(SQLiteMixin):
             for blob_hash, length in lengths:
                 blob_length_dict[blob_hash] = length
 
-            for blob_hash, position, iv in stream_blobs:
+            current_time = time.time()
+            for blob_hash, position, iv, added_on in stream_blobs:
                 blob_length = blob_length_dict.get(blob_hash, 0)
-                crypt_blob_infos.append(BlobInfo(position, blob_length, iv, blob_hash))
+                crypt_blob_infos.append(BlobInfo(position, blob_length, iv, added_on or current_time, blob_hash))
                 if not blob_hash:
                     break
             return crypt_blob_infos
@@ -788,7 +793,7 @@ class SQLiteStorage(SQLiteMixin):
 
         await self.db.run(_save_claims)
         if update_file_callbacks:
-            await asyncio.wait(update_file_callbacks)
+            await asyncio.wait(map(asyncio.create_task, update_file_callbacks))
         if claim_id_to_supports:
             await self.save_supports(claim_id_to_supports)
 

lbry/file/file_manager.py

@@ -9,15 +9,16 @@ from lbry.error import InvalidStreamURLError
 from lbry.stream.managed_stream import ManagedStream
 from lbry.torrent.torrent_manager import TorrentSource
 from lbry.utils import cache_concurrent
-from scribe.schema.url import URL
+from lbry.schema.url import URL
 from lbry.wallet.dewies import dewies_to_lbc
 from lbry.file.source_manager import SourceManager
 from lbry.file.source import ManagedDownloadSource
+from lbry.extras.daemon.storage import StoredContentClaim
 if typing.TYPE_CHECKING:
     from lbry.conf import Config
     from lbry.extras.daemon.analytics import AnalyticsManager
     from lbry.extras.daemon.storage import SQLiteStorage
-    from lbry.wallet import WalletManager, Output
+    from lbry.wallet import WalletManager
     from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
 
 log = logging.getLogger(__name__)
@@ -49,10 +50,10 @@ class FileManager:
             await manager.started.wait()
         self.started.set()
 
-    def stop(self):
+    async def stop(self):
         for manager in self.source_managers.values():
             # fixme: pop or not?
-            manager.stop()
+            await manager.stop()
         self.started.clear()
 
     @cache_concurrent
@@ -98,8 +99,6 @@ class FileManager:
             except asyncio.TimeoutError:
                 raise ResolveTimeoutError(uri)
             except Exception as err:
-                if isinstance(err, asyncio.CancelledError):
-                    raise
                 log.exception("Unexpected error resolving stream:")
                 raise ResolveError(f"Unexpected error resolving stream: {str(err)}")
             if 'error' in resolved_result:
@@ -194,21 +193,24 @@ class FileManager:
             ####################
             # make downloader and wait for start
             ####################
+            # temporary with fields we know so downloader can start. Missing fields are populated later.
+            stored_claim = StoredContentClaim(outpoint=outpoint, claim_id=txo.claim_id, name=txo.claim_name,
+                                              amount=txo.amount, height=txo.tx_ref.height,
+                                              serialized=claim.to_bytes().hex())
 
             if not claim.stream.source.bt_infohash:
                 # fixme: this shouldnt be here
                 stream = ManagedStream(
                     self.loop, self.config, source_manager.blob_manager, claim.stream.source.sd_hash,
                     download_directory, file_name, ManagedStream.STATUS_RUNNING, content_fee=payment,
-                    analytics_manager=self.analytics_manager
+                    analytics_manager=self.analytics_manager, claim=stored_claim
                 )
                 stream.downloader.node = source_manager.node
             else:
                 stream = TorrentSource(
                     self.loop, self.config, self.storage, identifier=claim.stream.source.bt_infohash,
                     file_name=file_name, download_directory=download_directory or self.config.download_dir,
-                    status=ManagedStream.STATUS_RUNNING,
-                    analytics_manager=self.analytics_manager,
+                    status=ManagedStream.STATUS_RUNNING, claim=stored_claim, analytics_manager=self.analytics_manager,
                     torrent_session=source_manager.torrent_session
                 )
             log.info("starting download for %s", uri)
@@ -240,13 +242,12 @@ class FileManager:
                 claim_info = await self.storage.get_content_claim_for_torrent(stream.identifier)
                 stream.set_claim(claim_info, claim)
             if save_file:
-                await asyncio.wait_for(stream.save_file(), timeout - (self.loop.time() - before_download),
-                                       loop=self.loop)
+                await asyncio.wait_for(stream.save_file(), timeout - (self.loop.time() - before_download))
             return stream
         except asyncio.TimeoutError:
             error = DownloadDataTimeoutError(stream.sd_hash)
             raise error
-        except Exception as err:  # forgive data timeout, don't delete stream
+        except (Exception, asyncio.CancelledError) as err:  # forgive data timeout, don't delete stream
             expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError,
                         KeyFeeAboveMaxAllowedError, ResolveError, InvalidStreamURLError)
             if isinstance(err, expected):

lbry/file/source.py

@@ -45,11 +45,12 @@ class ManagedDownloadSource:
         self.purchase_receipt = None
         self._added_on = added_on
         self.analytics_manager = analytics_manager
+        self.downloader = None
 
-        self.saving = asyncio.Event(loop=self.loop)
-        self.finished_writing = asyncio.Event(loop=self.loop)
-        self.started_writing = asyncio.Event(loop=self.loop)
-        self.finished_write_attempt = asyncio.Event(loop=self.loop)
+        self.saving = asyncio.Event()
+        self.finished_writing = asyncio.Event()
+        self.started_writing = asyncio.Event()
+        self.finished_write_attempt = asyncio.Event()
 
     # @classmethod
     # async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', file_path: str,
@@ -66,7 +67,7 @@ class ManagedDownloadSource:
     async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
         raise NotImplementedError()
 
-    def stop_tasks(self):
+    async def stop_tasks(self):
         raise NotImplementedError()
 
     def set_claim(self, claim_info: typing.Dict, claim: 'Claim'):

lbry/file/source_manager.py

@@ -54,16 +54,16 @@ class SourceManager:
         self.storage = storage
         self.analytics_manager = analytics_manager
         self._sources: typing.Dict[str, ManagedDownloadSource] = {}
-        self.started = asyncio.Event(loop=self.loop)
+        self.started = asyncio.Event()
 
     def add(self, source: ManagedDownloadSource):
         self._sources[source.identifier] = source
 
-    def remove(self, source: ManagedDownloadSource):
+    async def remove(self, source: ManagedDownloadSource):
         if source.identifier not in self._sources:
             return
         self._sources.pop(source.identifier)
-        source.stop_tasks()
+        await source.stop_tasks()
 
     async def initialize_from_database(self):
         raise NotImplementedError()
@@ -72,10 +72,10 @@ class SourceManager:
         await self.initialize_from_database()
         self.started.set()
 
-    def stop(self):
+    async def stop(self):
         while self._sources:
             _, source = self._sources.popitem()
-            source.stop_tasks()
+            await source.stop_tasks()
         self.started.clear()
 
     async def create(self, file_path: str, key: Optional[bytes] = None,
@@ -83,7 +83,7 @@ class SourceManager:
         raise NotImplementedError()
 
     async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
-        self.remove(source)
+        await self.remove(source)
         if delete_file and source.output_file_exists:
             os.remove(source.full_path)
 

lbry/schema/Makefile

@@ -0,0 +1,6 @@
+build:
+	rm types/v2/* -rf
+	touch types/v2/__init__.py
+	cd types/v2/ && protoc --python_out=. -I ../../../../../types/v2/proto/ ../../../../../types/v2/proto/*.proto
+	cd types/v2/ && cp ../../../../../types/jsonschema/* ./
+	sed -e 's/^import\ \(.*\)_pb2\ /from . import\ \1_pb2\ /g' -i types/v2/*.py

lbry/schema/README.md

@@ -0,0 +1,24 @@
+Schema
+=====
+
+Those files are generated from the [types repo](https://github.com/lbryio/types). If you are modifying/adding a new type, make sure it is cloned in the same root folder as the SDK repo, like:
+
+```
+repos/
+    - lbry-sdk/
+    - types/
+```
+
+Then, [download protoc 3.2.0](https://github.com/protocolbuffers/protobuf/releases/tag/v3.2.0), add it to your PATH. On linux it is:
+
+```bash
+cd ~/.local/bin
+wget https://github.com/protocolbuffers/protobuf/releases/download/v3.2.0/protoc-3.2.0-linux-x86_64.zip
+unzip protoc-3.2.0-linux-x86_64.zip bin/protoc -d..
+```
+
+Finally, `make` should update everything in place.
+
+
+### Why protoc 3.2.0?
+Different/newer versions will generate larger diffs and we need to make sure they are good. In theory, we can just update to latest and it will all work, but it is a good practice to check blockchain data and retro compatibility before bumping versions (if you do, please update this section!).

lbry/schema/__init__.py

@@ -0,0 +1 @@
+from .claim import Claim

lbry/schema/attrs.py

@@ -0,0 +1,571 @@
+import json
+import logging
+import os.path
+import hashlib
+from typing import Tuple, List
+from string import ascii_letters
+from decimal import Decimal, ROUND_UP
+from binascii import hexlify, unhexlify
+from google.protobuf.json_format import MessageToDict
+
+from lbry.crypto.base58 import Base58
+from lbry.constants import COIN
+from lbry.error import MissingPublishedFileError, EmptyPublishedFileError
+
+from lbry.schema.mime_types import guess_media_type
+from lbry.schema.base import Metadata, BaseMessageList
+from lbry.schema.tags import clean_tags, normalize_tag
+from lbry.schema.types.v2.claim_pb2 import (
+    Fee as FeeMessage,
+    Location as LocationMessage,
+    Language as LanguageMessage
+)
+
+
+log = logging.getLogger(__name__)
+
+
+def calculate_sha384_file_hash(file_path):
+    sha384 = hashlib.sha384()
+    with open(file_path, 'rb') as f:
+        for chunk in iter(lambda: f.read(128 * sha384.block_size), b''):
+            sha384.update(chunk)
+    return sha384.digest()
+
+
+def country_int_to_str(country: int) -> str:
+    r = LocationMessage.Country.Name(country)
+    return r[1:] if r.startswith('R') else r
+
+
+def country_str_to_int(country: str) -> int:
+    if len(country) == 3:
+        country = 'R' + country
+    return LocationMessage.Country.Value(country)
+
+
+class Dimmensional(Metadata):
+
+    __slots__ = ()
+
+    @property
+    def width(self) -> int:
+        return self.message.width
+
+    @width.setter
+    def width(self, width: int):
+        self.message.width = width
+
+    @property
+    def height(self) -> int:
+        return self.message.height
+
+    @height.setter
+    def height(self, height: int):
+        self.message.height = height
+
+    @property
+    def dimensions(self) -> Tuple[int, int]:
+        return self.width, self.height
+
+    @dimensions.setter
+    def dimensions(self, dimensions: Tuple[int, int]):
+        self.message.width, self.message.height = dimensions
+
+    def _extract(self, file_metadata, field):
+        try:
+            setattr(self, field, file_metadata.getValues(field)[0])
+        except:
+            log.exception(f'Could not extract {field} from file metadata.')
+
+    def update(self, file_metadata=None, height=None, width=None):
+        if height is not None:
+            self.height = height
+        elif file_metadata:
+            self._extract(file_metadata, 'height')
+
+        if width is not None:
+            self.width = width
+        elif file_metadata:
+            self._extract(file_metadata, 'width')
+
+
+class Playable(Metadata):
+
+    __slots__ = ()
+
+    @property
+    def duration(self) -> int:
+        return self.message.duration
+
+    @duration.setter
+    def duration(self, duration: int):
+        self.message.duration = duration
+
+    def update(self, file_metadata=None, duration=None):
+        if duration is not None:
+            self.duration = duration
+        elif file_metadata:
+            try:
+                self.duration = file_metadata.getValues('duration')[0].seconds
+            except:
+                log.exception('Could not extract duration from file metadata.')
+
+
+class Image(Dimmensional):
+
+    __slots__ = ()
+
+
+class Audio(Playable):
+
+    __slots__ = ()
+
+
+class Video(Dimmensional, Playable):
+
+    __slots__ = ()
+
+    def update(self, file_metadata=None, height=None, width=None, duration=None):
+        Dimmensional.update(self, file_metadata, height, width)
+        Playable.update(self, file_metadata, duration)
+
+
+class Source(Metadata):
+
+    __slots__ = ()
+
+    def update(self, file_path=None):
+        if file_path is not None:
+            self.name = os.path.basename(file_path)
+            self.media_type, stream_type = guess_media_type(file_path)
+            if not os.path.isfile(file_path):
+                raise MissingPublishedFileError(file_path)
+            self.size = os.path.getsize(file_path)
+            if self.size == 0:
+                raise EmptyPublishedFileError(file_path)
+            self.file_hash_bytes = calculate_sha384_file_hash(file_path)
+            return stream_type
+
+    @property
+    def name(self) -> str:
+        return self.message.name
+
+    @name.setter
+    def name(self, name: str):
+        self.message.name = name
+
+    @property
+    def size(self) -> int:
+        return self.message.size
+
+    @size.setter
+    def size(self, size: int):
+        self.message.size = size
+
+    @property
+    def media_type(self) -> str:
+        return self.message.media_type
+
+    @media_type.setter
+    def media_type(self, media_type: str):
+        self.message.media_type = media_type
+
+    @property
+    def file_hash(self) -> str:
+        return hexlify(self.message.hash).decode()
+
+    @file_hash.setter
+    def file_hash(self, file_hash: str):
+        self.message.hash = unhexlify(file_hash.encode())
+
+    @property
+    def file_hash_bytes(self) -> bytes:
+        return self.message.hash
+
+    @file_hash_bytes.setter
+    def file_hash_bytes(self, file_hash_bytes: bytes):
+        self.message.hash = file_hash_bytes
+
+    @property
+    def sd_hash(self) -> str:
+        return hexlify(self.message.sd_hash).decode()
+
+    @sd_hash.setter
+    def sd_hash(self, sd_hash: str):
+        self.message.sd_hash = unhexlify(sd_hash.encode())
+
+    @property
+    def sd_hash_bytes(self) -> bytes:
+        return self.message.sd_hash
+
+    @sd_hash_bytes.setter
+    def sd_hash_bytes(self, sd_hash: bytes):
+        self.message.sd_hash = sd_hash
+
+    @property
+    def bt_infohash(self) -> str:
+        return hexlify(self.message.bt_infohash).decode()
+
+    @bt_infohash.setter
+    def bt_infohash(self, bt_infohash: str):
+        self.message.bt_infohash = unhexlify(bt_infohash.encode())
+
+    @property
+    def bt_infohash_bytes(self) -> bytes:
+        return self.message.bt_infohash.decode()
+
+    @bt_infohash_bytes.setter
+    def bt_infohash_bytes(self, bt_infohash: bytes):
+        self.message.bt_infohash = bt_infohash
+
+    @property
+    def url(self) -> str:
+        return self.message.url
+
+    @url.setter
+    def url(self, url: str):
+        self.message.url = url
+
+
+class Fee(Metadata):
+
+    __slots__ = ()
+
+    def update(self, address: str = None, currency: str = None, amount=None):
+        if amount:
+            currency = (currency or self.currency or '').lower()
+            if not currency:
+                raise Exception('In order to set a fee amount, please specify a fee currency.')
+            if currency not in ('lbc', 'btc', 'usd'):
+                raise Exception(f'Missing or unknown currency provided: {currency}')
+            setattr(self, currency, Decimal(amount))
+        elif currency:
+            raise Exception('In order to set a fee currency, please specify a fee amount.')
+        if address:
+            if not self.currency:
+                raise Exception('In order to set a fee address, please specify a fee amount and currency.')
+            self.address = address
+
+    @property
+    def currency(self) -> str:
+        if self.message.currency:
+            return FeeMessage.Currency.Name(self.message.currency)
+
+    @property
+    def address(self) -> str:
+        if self.address_bytes:
+            return Base58.encode(self.address_bytes)
+
+    @address.setter
+    def address(self, address: str):
+        self.address_bytes = Base58.decode(address)
+
+    @property
+    def address_bytes(self) -> bytes:
+        return self.message.address
+
+    @address_bytes.setter
+    def address_bytes(self, address: bytes):
+        self.message.address = address
+
+    @property
+    def amount(self) -> Decimal:
+        if self.currency == 'LBC':
+            return self.lbc
+        if self.currency == 'BTC':
+            return self.btc
+        if self.currency == 'USD':
+            return self.usd
+
+    DEWIES = Decimal(COIN)
+
+    @property
+    def lbc(self) -> Decimal:
+        if self.message.currency != FeeMessage.LBC:
+            raise ValueError('LBC can only be returned for LBC fees.')
+        return Decimal(self.message.amount / self.DEWIES)
+
+    @lbc.setter
+    def lbc(self, amount: Decimal):
+        self.dewies = int(amount * self.DEWIES)
+
+    @property
+    def dewies(self) -> int:
+        if self.message.currency != FeeMessage.LBC:
+            raise ValueError('Dewies can only be returned for LBC fees.')
+        return self.message.amount
+
+    @dewies.setter
+    def dewies(self, amount: int):
+        self.message.amount = amount
+        self.message.currency = FeeMessage.LBC
+
+    SATOSHIES = Decimal(COIN)
+
+    @property
+    def btc(self) -> Decimal:
+        if self.message.currency != FeeMessage.BTC:
+            raise ValueError('BTC can only be returned for BTC fees.')
+        return Decimal(self.message.amount / self.SATOSHIES)
+
+    @btc.setter
+    def btc(self, amount: Decimal):
+        self.satoshis = int(amount * self.SATOSHIES)
+
+    @property
+    def satoshis(self) -> int:
+        if self.message.currency != FeeMessage.BTC:
+            raise ValueError('Satoshies can only be returned for BTC fees.')
+        return self.message.amount
+
+    @satoshis.setter
+    def satoshis(self, amount: int):
+        self.message.amount = amount
+        self.message.currency = FeeMessage.BTC
+
+    PENNIES = Decimal('100.0')
+    PENNY = Decimal('0.01')
+
+    @property
+    def usd(self) -> Decimal:
+        if self.message.currency != FeeMessage.USD:
+            raise ValueError('USD can only be returned for USD fees.')
+        return Decimal(self.message.amount / self.PENNIES)
+
+    @usd.setter
+    def usd(self, amount: Decimal):
+        self.pennies = int(amount.quantize(self.PENNY, ROUND_UP) * self.PENNIES)
+
+    @property
+    def pennies(self) -> int:
+        if self.message.currency != FeeMessage.USD:
+            raise ValueError('Pennies can only be returned for USD fees.')
+        return self.message.amount
+
+    @pennies.setter
+    def pennies(self, amount: int):
+        self.message.amount = amount
+        self.message.currency = FeeMessage.USD
+
+
+class ClaimReference(Metadata):
+
+    __slots__ = ()
+
+    @property
+    def claim_id(self) -> str:
+        return hexlify(self.claim_hash[::-1]).decode()
+
+    @claim_id.setter
+    def claim_id(self, claim_id: str):
+        self.claim_hash = unhexlify(claim_id)[::-1]
+
+    @property
+    def claim_hash(self) -> bytes:
+        return self.message.claim_hash
+
+    @claim_hash.setter
+    def claim_hash(self, claim_hash: bytes):
+        self.message.claim_hash = claim_hash
+
+
+class ClaimList(BaseMessageList[ClaimReference]):
+
+    __slots__ = ()
+    item_class = ClaimReference
+
+    @property
+    def _message(self):
+        return self.message.claim_references
+
+    def append(self, value):
+        self.add().claim_id = value
+
+    @property
+    def ids(self) -> List[str]:
+        return [c.claim_id for c in self]
+
+
+class Language(Metadata):
+
+    __slots__ = ()
+
+    @property
+    def langtag(self) -> str:
+        langtag = []
+        if self.language:
+            langtag.append(self.language)
+        if self.script:
+            langtag.append(self.script)
+        if self.region:
+            langtag.append(self.region)
+        return '-'.join(langtag)
+
+    @langtag.setter
+    def langtag(self, langtag: str):
+        parts = langtag.split('-')
+        self.language = parts.pop(0)
+        if parts and len(parts[0]) == 4:
+            self.script = parts.pop(0)
+        if parts and len(parts[0]) == 2 and parts[0].isalpha():
+            self.region = parts.pop(0)
+        if parts and len(parts[0]) == 3 and parts[0].isdigit():
+            self.region = parts.pop(0)
+        assert not parts, f"Failed to parse language tag: {langtag}"
+
+    @property
+    def language(self) -> str:
+        if self.message.language:
+            return LanguageMessage.Language.Name(self.message.language)
+
+    @language.setter
+    def language(self, language: str):
+        self.message.language = LanguageMessage.Language.Value(language)
+
+    @property
+    def script(self) -> str:
+        if self.message.script:
+            return LanguageMessage.Script.Name(self.message.script)
+
+    @script.setter
+    def script(self, script: str):
+        self.message.script = LanguageMessage.Script.Value(script)
+
+    @property
+    def region(self) -> str:
+        if self.message.region:
+            return country_int_to_str(self.message.region)
+
+    @region.setter
+    def region(self, region: str):
+        self.message.region = country_str_to_int(region)
+
+
+class LanguageList(BaseMessageList[Language]):
+    __slots__ = ()
+    item_class = Language
+
+    def append(self, value: str):
+        self.add().langtag = value
+
+
+class Location(Metadata):
+
+    __slots__ = ()
+
+    def from_value(self, value):
+        if isinstance(value, str) and value.startswith('{'):
+            value = json.loads(value)
+
+        if isinstance(value, dict):
+            for key, val in value.items():
+                setattr(self, key, val)
+
+        elif isinstance(value, str):
+            parts = value.split(':')
+            if len(parts) > 2 or (parts[0] and parts[0][0] in ascii_letters):
+                country = parts and parts.pop(0)
+                if country:
+                    self.country = country
+                state = parts and parts.pop(0)
+                if state:
+                    self.state = state
+                city = parts and parts.pop(0)
+                if city:
+                    self.city = city
+                code = parts and parts.pop(0)
+                if code:
+                    self.code = code
+            latitude = parts and parts.pop(0)
+            if latitude:
+                self.latitude = latitude
+            longitude = parts and parts.pop(0)
+            if longitude:
+                self.longitude = longitude
+
+        else:
+            raise ValueError(f'Could not parse country value: {value}')
+
+    def to_dict(self):
+        d = MessageToDict(self.message)
+        if self.message.longitude:
+            d['longitude'] = self.longitude
+        if self.message.latitude:
+            d['latitude'] = self.latitude
+        return d
+
+    @property
+    def country(self) -> str:
+        if self.message.country:
+            return LocationMessage.Country.Name(self.message.country)
+
+    @country.setter
+    def country(self, country: str):
+        self.message.country = LocationMessage.Country.Value(country)
+
+    @property
+    def state(self) -> str:
+        return self.message.state
+
+    @state.setter
+    def state(self, state: str):
+        self.message.state = state
+
+    @property
+    def city(self) -> str:
+        return self.message.city
+
+    @city.setter
+    def city(self, city: str):
+        self.message.city = city
+
+    @property
+    def code(self) -> str:
+        return self.message.code
+
+    @code.setter
+    def code(self, code: str):
+        self.message.code = code
+
+    GPS_PRECISION = Decimal('10000000')
+
+    @property
+    def latitude(self) -> str:
+        if self.message.latitude:
+            return str(Decimal(self.message.latitude) / self.GPS_PRECISION)
+
+    @latitude.setter
+    def latitude(self, latitude: str):
+        latitude = Decimal(latitude)
+        assert -90 <= latitude <= 90, "Latitude must be between -90 and 90 degrees."
+        self.message.latitude = int(latitude * self.GPS_PRECISION)
+
+    @property
+    def longitude(self) -> str:
+        if self.message.longitude:
+            return str(Decimal(self.message.longitude) / self.GPS_PRECISION)
+
+    @longitude.setter
+    def longitude(self, longitude: str):
+        longitude = Decimal(longitude)
+        assert -180 <= longitude <= 180, "Longitude must be between -180 and 180 degrees."
+        self.message.longitude = int(longitude * self.GPS_PRECISION)
+
+
+class LocationList(BaseMessageList[Location]):
+    __slots__ = ()
+    item_class = Location
+
+    def append(self, value):
+        self.add().from_value(value)
+
+
+class TagList(BaseMessageList[str]):
+    __slots__ = ()
+    item_class = str
+
+    def append(self, tag: str):
+        tag = normalize_tag(tag)
+        if tag and tag not in self.message:
+            self.message.append(tag)

lbry/schema/base.py

@@ -0,0 +1,124 @@
+from binascii import hexlify, unhexlify
+from typing import List, Iterator, TypeVar, Generic
+
+from google.protobuf.message import DecodeError
+from google.protobuf.json_format import MessageToDict
+
+
+class Signable:
+
+    __slots__ = (
+        'message', 'version', 'signature',
+        'signature_type', 'unsigned_payload', 'signing_channel_hash'
+    )
+
+    message_class = None
+
+    def __init__(self, message=None):
+        self.message = message or self.message_class()
+        self.version = 2
+        self.signature = None
+        self.signature_type = 'SECP256k1'
+        self.unsigned_payload = None
+        self.signing_channel_hash = None
+
+    def clear_signature(self):
+        self.signature = None
+        self.unsigned_payload = None
+        self.signing_channel_hash = None
+
+    @property
+    def signing_channel_id(self):
+        return hexlify(self.signing_channel_hash[::-1]).decode() if self.signing_channel_hash else None
+
+    @signing_channel_id.setter
+    def signing_channel_id(self, channel_id: str):
+        self.signing_channel_hash = unhexlify(channel_id)[::-1]
+
+    @property
+    def is_signed(self):
+        return self.signature is not None
+
+    def to_dict(self):
+        return MessageToDict(self.message)
+
+    def to_message_bytes(self) -> bytes:
+        return self.message.SerializeToString()
+
+    def to_bytes(self) -> bytes:
+        pieces = bytearray()
+        if self.is_signed:
+            pieces.append(1)
+            pieces.extend(self.signing_channel_hash)
+            pieces.extend(self.signature)
+        else:
+            pieces.append(0)
+        pieces.extend(self.to_message_bytes())
+        return bytes(pieces)
+
+    @classmethod
+    def from_bytes(cls, data: bytes):
+        signable = cls()
+        if data[0] == 0:
+            signable.message.ParseFromString(data[1:])
+        elif data[0] == 1:
+            signable.signing_channel_hash = data[1:21]
+            signable.signature = data[21:85]
+            signable.message.ParseFromString(data[85:])
+        else:
+            raise DecodeError('Could not determine message format version.')
+        return signable
+
+    def __len__(self):
+        return len(self.to_bytes())
+
+    def __bytes__(self):
+        return self.to_bytes()
+
+
+class Metadata:
+
+    __slots__ = 'message',
+
+    def __init__(self, message):
+        self.message = message
+
+
+I = TypeVar('I')
+
+
+class BaseMessageList(Metadata, Generic[I]):
+
+    __slots__ = ()
+
+    item_class = None
+
+    @property
+    def _message(self):
+        return self.message
+
+    def add(self) -> I:
+        return self.item_class(self._message.add())
+
+    def extend(self, values: List[str]):
+        for value in values:
+            self.append(value)
+
+    def append(self, value: str):
+        raise NotImplemented
+
+    def __len__(self):
+        return len(self._message)
+
+    def __iter__(self) -> Iterator[I]:
+        for item in self._message:
+            yield self.item_class(item)
+
+    def __getitem__(self, item) -> I:
+        return self.item_class(self._message[item])
+
+    def __delitem__(self, key):
+        del self._message[key]
+
+    def __eq__(self, other) -> bool:
+        return self._message == other

lbry/schema/claim.py

@@ -0,0 +1,428 @@
+import logging
+from typing import List
+from binascii import hexlify, unhexlify
+
+from asn1crypto.keys import PublicKeyInfo
+from coincurve import PublicKey as cPublicKey
+
+from google.protobuf.json_format import MessageToDict
+from google.protobuf.message import DecodeError
+from hachoir.core.log import log as hachoir_log
+from hachoir.parser import createParser as binary_file_parser
+from hachoir.metadata import extractMetadata as binary_file_metadata
+
+from lbry.schema import compat
+from lbry.schema.base import Signable
+from lbry.schema.mime_types import guess_media_type, guess_stream_type
+from lbry.schema.attrs import (
+    Source, Playable, Dimmensional, Fee, Image, Video, Audio,
+    LanguageList, LocationList, ClaimList, ClaimReference, TagList
+)
+from lbry.schema.types.v2.claim_pb2 import Claim as ClaimMessage
+from lbry.error import InputValueIsNoneError
+
+
+hachoir_log.use_print = False
+log = logging.getLogger(__name__)
+
+
+class Claim(Signable):
+
+    STREAM = 'stream'
+    CHANNEL = 'channel'
+    COLLECTION = 'collection'
+    REPOST = 'repost'
+
+    __slots__ = ()
+
+    message_class = ClaimMessage
+
+    @property
+    def claim_type(self) -> str:
+        return self.message.WhichOneof('type')
+
+    def get_message(self, type_name):
+        message = getattr(self.message, type_name)
+        if self.claim_type is None:
+            message.SetInParent()
+        if self.claim_type != type_name:
+            raise ValueError(f'Claim is not a {type_name}.')
+        return message
+
+    @property
+    def is_stream(self):
+        return self.claim_type == self.STREAM
+
+    @property
+    def stream(self) -> 'Stream':
+        return Stream(self)
+
+    @property
+    def is_channel(self):
+        return self.claim_type == self.CHANNEL
+
+    @property
+    def channel(self) -> 'Channel':
+        return Channel(self)
+
+    @property
+    def is_repost(self):
+        return self.claim_type == self.REPOST
+
+    @property
+    def repost(self) -> 'Repost':
+        return Repost(self)
+
+    @property
+    def is_collection(self):
+        return self.claim_type == self.COLLECTION
+
+    @property
+    def collection(self) -> 'Collection':
+        return Collection(self)
+
+    @classmethod
+    def from_bytes(cls, data: bytes) -> 'Claim':
+        try:
+            return super().from_bytes(data)
+        except DecodeError:
+            claim = cls()
+            if data[0] == ord('{'):
+                claim.version = 0
+                compat.from_old_json_schema(claim, data)
+            elif data[0] not in (0, 1):
+                claim.version = 1
+                compat.from_types_v1(claim, data)
+            else:
+                raise
+            return claim
+
+
+class BaseClaim:
+
+    __slots__ = 'claim', 'message'
+
+    claim_type = None
+    object_fields = 'thumbnail',
+    repeat_fields = 'tags', 'languages', 'locations'
+
+    def __init__(self, claim: Claim = None):
+        self.claim = claim or Claim()
+        self.message = self.claim.get_message(self.claim_type)
+
+    def to_dict(self):
+        claim = MessageToDict(self.claim.message, preserving_proto_field_name=True)
+        claim.update(claim.pop(self.claim_type))
+        if 'languages' in claim:
+            claim['languages'] = self.langtags
+        if 'locations' in claim:
+            claim['locations'] = [l.to_dict() for l in self.locations]
+        return claim
+
+    def none_check(self, kwargs):
+        for key, value in kwargs.items():
+            if value is None:
+                raise InputValueIsNoneError(key)
+
+    def update(self, **kwargs):
+        self.none_check(kwargs)
+
+        for key in list(kwargs):
+            for field in self.object_fields:
+                if key.startswith(f'{field}_'):
+                    attr = getattr(self, field)
+                    setattr(attr, key[len(f'{field}_'):], kwargs.pop(key))
+                    continue
+
+        for l in self.repeat_fields:
+            field = getattr(self, l)
+            if kwargs.pop(f'clear_{l}', False):
+                del field[:]
+            items = kwargs.pop(l, None)
+            if items is not None:
+                if isinstance(items, str):
+                    field.append(items)
+                elif isinstance(items, list):
+                    field.extend(items)
+                else:
+                    raise ValueError(f"Unknown {l} value: {items}")
+
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+
+    @property
+    def title(self) -> str:
+        return self.claim.message.title
+
+    @title.setter
+    def title(self, title: str):
+        self.claim.message.title = title
+
+    @property
+    def description(self) -> str:
+        return self.claim.message.description
+
+    @description.setter
+    def description(self, description: str):
+        self.claim.message.description = description
+
+    @property
+    def thumbnail(self) -> Source:
+        return Source(self.claim.message.thumbnail)
+
+    @property
+    def tags(self) -> List[str]:
+        return TagList(self.claim.message.tags)
+
+    @property
+    def languages(self) -> LanguageList:
+        return LanguageList(self.claim.message.languages)
+
+    @property
+    def langtags(self) -> List[str]:
+        return [l.langtag for l in self.languages]
+
+    @property
+    def locations(self) -> LocationList:
+        return LocationList(self.claim.message.locations)
+
+
+class Stream(BaseClaim):
+
+    __slots__ = ()
+
+    claim_type = Claim.STREAM
+
+    object_fields = BaseClaim.object_fields + ('source',)
+
+    def to_dict(self):
+        claim = super().to_dict()
+        if 'source' in claim:
+            if 'hash' in claim['source']:
+                claim['source']['hash'] = self.source.file_hash
+            if 'sd_hash' in claim['source']:
+                claim['source']['sd_hash'] = self.source.sd_hash
+            elif 'bt_infohash' in claim['source']:
+                claim['source']['bt_infohash'] = self.source.bt_infohash
+            if 'media_type' in claim['source']:
+                claim['stream_type'] = guess_stream_type(claim['source']['media_type'])
+        fee = claim.get('fee', {})
+        if 'address' in fee:
+            fee['address'] = self.fee.address
+        if 'amount' in fee:
+            fee['amount'] = str(self.fee.amount)
+        return claim
+
+    def update(self, file_path=None, height=None, width=None, duration=None, **kwargs):
+
+        if kwargs.pop('clear_fee', False):
+            self.message.ClearField('fee')
+        else:
+            self.fee.update(
+                kwargs.pop('fee_address', None),
+                kwargs.pop('fee_currency', None),
+                kwargs.pop('fee_amount', None)
+            )
+
+        self.none_check(kwargs)
+
+        if 'sd_hash' in kwargs:
+            self.source.sd_hash = kwargs.pop('sd_hash')
+        elif 'bt_infohash' in kwargs:
+            self.source.bt_infohash = kwargs.pop('bt_infohash')
+        if 'file_name' in kwargs:
+            self.source.name = kwargs.pop('file_name')
+        if 'file_hash' in kwargs:
+            self.source.file_hash = kwargs.pop('file_hash')
+
+        stream_type = None
+        if file_path is not None:
+            stream_type = self.source.update(file_path=file_path)
+        elif self.source.name:
+            self.source.media_type, stream_type = guess_media_type(self.source.name)
+        elif self.source.media_type:
+            stream_type = guess_stream_type(self.source.media_type)
+
+        if 'file_size' in kwargs:
+            self.source.size = kwargs.pop('file_size')
+
+        if self.stream_type is not None and self.stream_type != stream_type:
+            self.message.ClearField(self.stream_type)
+
+        if stream_type in ('image', 'video', 'audio'):
+            media = getattr(self, stream_type)
+            media_args = {'file_metadata': None}
+            if file_path is not None and not all((duration, width, height)):
+                try:
+                    media_args['file_metadata'] = binary_file_metadata(binary_file_parser(file_path))
+                except:
+                    log.exception('Could not read file metadata.')
+            if isinstance(media, Playable):
+                media_args['duration'] = duration
+            if isinstance(media, Dimmensional):
+                media_args['height'] = height
+                media_args['width'] = width
+            media.update(**media_args)
+
+        super().update(**kwargs)
+
+    @property
+    def author(self) -> str:
+        return self.message.author
+
+    @author.setter
+    def author(self, author: str):
+        self.message.author = author
+
+    @property
+    def license(self) -> str:
+        return self.message.license
+
+    @license.setter
+    def license(self, license: str):
+        self.message.license = license
+
+    @property
+    def license_url(self) -> str:
+        return self.message.license_url
+
+    @license_url.setter
+    def license_url(self, license_url: str):
+        self.message.license_url = license_url
+
+    @property
+    def release_time(self) -> int:
+        return self.message.release_time
+
+    @release_time.setter
+    def release_time(self, release_time: int):
+        self.message.release_time = release_time
+
+    @property
+    def fee(self) -> Fee:
+        return Fee(self.message.fee)
+
+    @property
+    def has_fee(self) -> bool:
+        return self.message.HasField('fee')
+
+    @property
+    def has_source(self) -> bool:
+        return self.message.HasField('source')
+
+    @property
+    def source(self) -> Source:
+        return Source(self.message.source)
+
+    @property
+    def stream_type(self) -> str:
+        return self.message.WhichOneof('type')
+
+    @property
+    def image(self) -> Image:
+        return Image(self.message.image)
+
+    @property
+    def video(self) -> Video:
+        return Video(self.message.video)
+
+    @property
+    def audio(self) -> Audio:
+        return Audio(self.message.audio)
+
+
+class Channel(BaseClaim):
+
+    __slots__ = ()
+
+    claim_type = Claim.CHANNEL
+
+    object_fields = BaseClaim.object_fields + ('cover',)
+    repeat_fields = BaseClaim.repeat_fields + ('featured',)
+
+    def to_dict(self):
+        claim = super().to_dict()
+        claim['public_key'] = self.public_key
+        if 'featured' in claim:
+            claim['featured'] = self.featured.ids
+        return claim
+
+    @property
+    def public_key(self) -> str:
+        return hexlify(self.public_key_bytes).decode()
+
+    @public_key.setter
+    def public_key(self, sd_public_key: str):
+        self.message.public_key = unhexlify(sd_public_key.encode())
+
+    @property
+    def public_key_bytes(self) -> bytes:
+        if len(self.message.public_key) == 33:
+            return self.message.public_key
+        public_key_info = PublicKeyInfo.load(self.message.public_key)
+        public_key = cPublicKey(public_key_info.native['public_key'])
+        return public_key.format(compressed=True)
+
+    @public_key_bytes.setter
+    def public_key_bytes(self, public_key: bytes):
+        self.message.public_key = public_key
+
+    @property
+    def email(self) -> str:
+        return self.message.email
+
+    @email.setter
+    def email(self, email: str):
+        self.message.email = email
+
+    @property
+    def website_url(self) -> str:
+        return self.message.website_url
+
+    @website_url.setter
+    def website_url(self, website_url: str):
+        self.message.website_url = website_url
+
+    @property
+    def cover(self) -> Source:
+        return Source(self.message.cover)
+
+    @property
+    def featured(self) -> ClaimList:
+        return ClaimList(self.message.featured)
+
+
+class Repost(BaseClaim):
+
+    __slots__ = ()
+
+    claim_type = Claim.REPOST
+
+    def to_dict(self):
+        claim = super().to_dict()
+        if claim.pop('claim_hash', None):
+            claim['claim_id'] = self.reference.claim_id
+        return claim
+
+    @property
+    def reference(self) -> ClaimReference:
+        return ClaimReference(self.message)
+
+
+class Collection(BaseClaim):
+
+    __slots__ = ()
+
+    claim_type = Claim.COLLECTION
+
+    repeat_fields = BaseClaim.repeat_fields + ('claims',)
+
+    def to_dict(self):
+        claim = super().to_dict()
+        if claim.pop('claim_references', None):
+            claim['claims'] = self.claims.ids
+        return claim
+
+    @property
+    def claims(self) -> ClaimList:
+        return ClaimList(self.message)

lbry/schema/compat.py

@@ -0,0 +1,93 @@
+import json
+from decimal import Decimal
+
+from google.protobuf.message import DecodeError
+
+from lbry.schema.types.v1.legacy_claim_pb2 import Claim as OldClaimMessage
+from lbry.schema.types.v1.certificate_pb2 import KeyType
+from lbry.schema.types.v1.fee_pb2 import Fee as FeeMessage
+
+
+def from_old_json_schema(claim, payload: bytes):
+    try:
+        value = json.loads(payload)
+    except:
+        raise DecodeError('Could not parse JSON.')
+    stream = claim.stream
+    stream.source.sd_hash = value['sources']['lbry_sd_hash']
+    stream.source.media_type = (
+            value.get('content_type', value.get('content-type')) or
+            'application/octet-stream'
+    )
+    stream.title = value.get('title', '')
+    stream.description = value.get('description', '')
+    if value.get('thumbnail', ''):
+        stream.thumbnail.url = value.get('thumbnail', '')
+    stream.author = value.get('author', '')
+    stream.license = value.get('license', '')
+    stream.license_url = value.get('license_url', '')
+    language = value.get('language', '')
+    if language:
+        if language.lower() == 'english':
+            language = 'en'
+        try:
+            stream.languages.append(language)
+        except:
+            pass
+    if value.get('nsfw', False):
+        stream.tags.append('mature')
+    if "fee" in value and isinstance(value['fee'], dict):
+        fee = value["fee"]
+        currency = list(fee.keys())[0]
+        if currency == 'LBC':
+            stream.fee.lbc = Decimal(fee[currency]['amount'])
+        elif currency == 'USD':
+            stream.fee.usd = Decimal(fee[currency]['amount'])
+        elif currency == 'BTC':
+            stream.fee.btc = Decimal(fee[currency]['amount'])
+        else:
+            raise DecodeError(f'Unknown currency: {currency}')
+        stream.fee.address = fee[currency]['address']
+    return claim
+
+
+def from_types_v1(claim, payload: bytes):
+    old = OldClaimMessage()
+    old.ParseFromString(payload)
+    if old.claimType == 2:
+        channel = claim.channel
+        channel.public_key_bytes = old.certificate.publicKey
+    else:
+        stream = claim.stream
+        stream.title = old.stream.metadata.title
+        stream.description = old.stream.metadata.description
+        stream.author = old.stream.metadata.author
+        stream.license = old.stream.metadata.license
+        stream.license_url = old.stream.metadata.licenseUrl
+        stream.thumbnail.url = old.stream.metadata.thumbnail
+        if old.stream.metadata.HasField('language'):
+            stream.languages.add().message.language = old.stream.metadata.language
+        stream.source.media_type = old.stream.source.contentType
+        stream.source.sd_hash_bytes = old.stream.source.source
+        if old.stream.metadata.nsfw:
+            stream.tags.append('mature')
+        if old.stream.metadata.HasField('fee'):
+            fee = old.stream.metadata.fee
+            stream.fee.address_bytes = fee.address
+            currency = FeeMessage.Currency.Name(fee.currency)
+            if currency == 'LBC':
+                stream.fee.lbc = Decimal(fee.amount)
+            elif currency == 'USD':
+                stream.fee.usd = Decimal(fee.amount)
+            elif currency == 'BTC':
+                stream.fee.btc = Decimal(fee.amount)
+            else:
+                raise DecodeError(f'Unsupported currency: {currency}')
+        if old.HasField('publisherSignature'):
+            sig = old.publisherSignature
+            claim.signature = sig.signature
+            claim.signature_type = KeyType.Name(sig.signatureType)
+            claim.signing_channel_hash = sig.certificateId[::-1]
+            old.ClearField("publisherSignature")
+            claim.unsigned_payload = old.SerializeToString()
+    return claim

lbry/schema/mime_types.py

@@ -0,0 +1,214 @@
+import os
+import filetype
+import logging
+
+types_map = {
+    # http://www.iana.org/assignments/media-types
+    # Type mapping for automated metadata extraction (video, audio, image, document, binary, model)
+    '.a': ('application/octet-stream', 'binary'),
+    '.ai': ('application/postscript', 'image'),
+    '.aif': ('audio/x-aiff', 'audio'),
+    '.aifc': ('audio/x-aiff', 'audio'),
+    '.aiff': ('audio/x-aiff', 'audio'),
+    '.au': ('audio/basic', 'audio'),
+    '.avi': ('video/x-msvideo', 'video'),
+    '.bat': ('text/plain', 'document'),
+    '.bcpio': ('application/x-bcpio', 'binary'),
+    '.bin': ('application/octet-stream', 'binary'),
+    '.bmp': ('image/bmp', 'image'),
+    '.c': ('text/plain', 'document'),
+    '.cdf': ('application/x-netcdf', 'binary'),
+    '.cpio': ('application/x-cpio', 'binary'),
+    '.csh': ('application/x-csh', 'binary'),
+    '.css': ('text/css', 'document'),
+    '.csv': ('text/csv', 'document'),
+    '.dll': ('application/octet-stream', 'binary'),
+    '.doc': ('application/msword', 'document'),
+    '.dot': ('application/msword', 'document'),
+    '.dvi': ('application/x-dvi', 'binary'),
+    '.eml': ('message/rfc822', 'document'),
+    '.eps': ('application/postscript', 'document'),
+    '.epub': ('application/epub+zip', 'document'),
+    '.etx': ('text/x-setext', 'document'),
+    '.exe': ('application/octet-stream', 'binary'),
+    '.gif': ('image/gif', 'image'),
+    '.gtar': ('application/x-gtar', 'binary'),
+    '.h': ('text/plain', 'document'),
+    '.hdf': ('application/x-hdf', 'binary'),
+    '.htm': ('text/html', 'document'),
+    '.html': ('text/html', 'document'),
+    '.ico': ('image/vnd.microsoft.icon', 'image'),
+    '.ief': ('image/ief', 'image'),
+    '.iges': ('model/iges', 'model'),
+    '.jpe': ('image/jpeg', 'image'),
+    '.jpeg': ('image/jpeg', 'image'),
+    '.jpg': ('image/jpeg', 'image'),
+    '.js': ('application/javascript', 'document'),
+    '.json': ('application/json', 'document'),
+    '.ksh': ('text/plain', 'document'),
+    '.latex': ('application/x-latex', 'binary'),
+    '.m1v': ('video/mpeg', 'video'),
+    '.m3u': ('application/x-mpegurl', 'audio'),
+    '.m3u8': ('application/x-mpegurl', 'video'),
+    '.man': ('application/x-troff-man', 'document'),
+    '.markdown': ('text/markdown', 'document'),
+    '.md': ('text/markdown', 'document'),
+    '.me': ('application/x-troff-me', 'binary'),
+    '.mht': ('message/rfc822', 'document'),
+    '.mhtml': ('message/rfc822', 'document'),
+    '.mif': ('application/x-mif', 'binary'),
+    '.mov': ('video/quicktime', 'video'),
+    '.movie': ('video/x-sgi-movie', 'video'),
+    '.mp2': ('audio/mpeg', 'audio'),
+    '.mp3': ('audio/mpeg', 'audio'),
+    '.mp4': ('video/mp4', 'video'),
+    '.mpa': ('video/mpeg', 'video'),
+    '.mpd': ('application/dash+xml', 'video'),
+    '.mpe': ('video/mpeg', 'video'),
+    '.mpeg': ('video/mpeg', 'video'),
+    '.mpg': ('video/mpeg', 'video'),
+    '.ms': ('application/x-troff-ms', 'binary'),
+    '.m4s': ('video/iso.segment', 'binary'),
+    '.nc': ('application/x-netcdf', 'binary'),
+    '.nws': ('message/rfc822', 'document'),
+    '.o': ('application/octet-stream', 'binary'),
+    '.obj': ('application/octet-stream', 'model'),
+    '.oda': ('application/oda', 'binary'),
+    '.p12': ('application/x-pkcs12', 'binary'),
+    '.p7c': ('application/pkcs7-mime', 'binary'),
+    '.pbm': ('image/x-portable-bitmap', 'image'),
+    '.pdf': ('application/pdf', 'document'),
+    '.pfx': ('application/x-pkcs12', 'binary'),
+    '.pgm': ('image/x-portable-graymap', 'image'),
+    '.pl': ('text/plain', 'document'),
+    '.png': ('image/png', 'image'),
+    '.pnm': ('image/x-portable-anymap', 'image'),
+    '.pot': ('application/vnd.ms-powerpoint', 'document'),
+    '.ppa': ('application/vnd.ms-powerpoint', 'document'),
+    '.ppm': ('image/x-portable-pixmap', 'image'),
+    '.pps': ('application/vnd.ms-powerpoint', 'document'),
+    '.ppt': ('application/vnd.ms-powerpoint', 'document'),
+    '.ps': ('application/postscript', 'document'),
+    '.pwz': ('application/vnd.ms-powerpoint', 'document'),
+    '.py': ('text/x-python', 'document'),
+    '.pyc': ('application/x-python-code', 'binary'),
+    '.pyo': ('application/x-python-code', 'binary'),
+    '.qt': ('video/quicktime', 'video'),
+    '.ra': ('audio/x-pn-realaudio', 'audio'),
+    '.ram': ('application/x-pn-realaudio', 'audio'),
+    '.ras': ('image/x-cmu-raster', 'image'),
+    '.rdf': ('application/xml', 'binary'),
+    '.rgb': ('image/x-rgb', 'image'),
+    '.roff': ('application/x-troff', 'binary'),
+    '.rtx': ('text/richtext', 'document'),
+    '.sgm': ('text/x-sgml', 'document'),
+    '.sgml': ('text/x-sgml', 'document'),
+    '.sh': ('application/x-sh', 'document'),
+    '.shar': ('application/x-shar', 'binary'),
+    '.snd': ('audio/basic', 'audio'),
+    '.so': ('application/octet-stream', 'binary'),
+    '.src': ('application/x-wais-source', 'binary'),
+    '.stl': ('model/stl', 'model'),
+    '.sv4cpio': ('application/x-sv4cpio', 'binary'),
+    '.sv4crc': ('application/x-sv4crc', 'binary'),
+    '.svg': ('image/svg+xml', 'image'),
+    '.swf': ('application/x-shockwave-flash', 'binary'),
+    '.t': ('application/x-troff', 'binary'),
+    '.tar': ('application/x-tar', 'binary'),
+    '.tcl': ('application/x-tcl', 'binary'),
+    '.tex': ('application/x-tex', 'binary'),
+    '.texi': ('application/x-texinfo', 'binary'),
+    '.texinfo': ('application/x-texinfo', 'binary'),
+    '.tif': ('image/tiff', 'image'),
+    '.tiff': ('image/tiff', 'image'),
+    '.tr': ('application/x-troff', 'binary'),
+    '.ts': ('video/mp2t', 'video'),
+    '.tsv': ('text/tab-separated-values', 'document'),
+    '.txt': ('text/plain', 'document'),
+    '.ustar': ('application/x-ustar', 'binary'),
+    '.vcf': ('text/x-vcard', 'document'),
+    '.vtt': ('text/vtt', 'document'),
+    '.wav': ('audio/x-wav', 'audio'),
+    '.webm': ('video/webm', 'video'),
+    '.wiz': ('application/msword', 'document'),
+    '.wsdl': ('application/xml', 'document'),
+    '.xbm': ('image/x-xbitmap', 'image'),
+    '.xlb': ('application/vnd.ms-excel', 'document'),
+    '.xls': ('application/vnd.ms-excel', 'document'),
+    '.xml': ('text/xml', 'document'),
+    '.xpdl': ('application/xml', 'document'),
+    '.xpm': ('image/x-xpixmap', 'image'),
+    '.xsl': ('application/xml', 'document'),
+    '.xwd': ('image/x-xwindowdump', 'image'),
+    '.zip': ('application/zip', 'binary'),
+
+    # These are non-standard types, commonly found in the wild.
+    '.cbr': ('application/vnd.comicbook-rar', 'document'),
+    '.cbz': ('application/vnd.comicbook+zip', 'document'),
+    '.flac': ('audio/flac', 'audio'),
+    '.lbry': ('application/x-ext-lbry', 'document'),
+    '.m4a': ('audio/mp4', 'audio'),
+    '.m4v': ('video/m4v', 'video'),
+    '.mid': ('audio/midi', 'audio'),
+    '.midi': ('audio/midi', 'audio'),
+    '.mkv': ('video/x-matroska', 'video'),
+    '.mobi': ('application/x-mobipocket-ebook', 'document'),
+    '.oga': ('audio/ogg', 'audio'),
+    '.ogv': ('video/ogg', 'video'),
+    '.ogg': ('video/ogg', 'video'),
+    '.pct': ('image/pict', 'image'),
+    '.pic': ('image/pict', 'image'),
+    '.pict': ('image/pict', 'image'),
+    '.prc': ('application/x-mobipocket-ebook', 'document'),
+    '.rtf': ('application/rtf', 'document'),
+    '.xul': ('text/xul', 'document'),
+    
+    # microsoft is special and has its own 'standard'
+    # https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
+    '.wmv': ('video/x-ms-wmv', 'video')
+}
+
+# maps detected extensions to the possible analogs
+# i.e. .cbz file is actually a .zip
+synonyms_map = {
+    '.zip': ['.cbz'],
+    '.rar': ['.cbr'],
+    '.ar': ['.a']
+}
+
+log = logging.getLogger(__name__)
+
+
+def guess_media_type(path):
+    _, ext = os.path.splitext(path)
+    extension = ext.strip().lower()
+
+    try:
+        kind = filetype.guess(path)
+        if kind:
+            real_extension = f".{kind.extension}"
+
+            if extension != real_extension:
+                if extension:
+                    log.warning(f"file extension does not match it's contents: {path}, identified as {real_extension}")
+                else:
+                    log.debug(f"file {path} does not have extension, identified by it's contents as {real_extension}")
+
+                if extension not in synonyms_map.get(real_extension, []):
+                    extension = real_extension
+
+    except OSError as error:
+        pass
+
+    if extension[1:]:
+        if extension in types_map:
+            return types_map[extension]
+        return f'application/x-ext-{extension[1:]}', 'binary'
+    return 'application/octet-stream', 'binary'
+
+
+def guess_stream_type(media_type):
+    for media, stream in types_map.values():
+        if media == media_type:
+            return stream
+    return 'binary'

lbry/schema/purchase.py

@@ -0,0 +1,47 @@
+from google.protobuf.message import DecodeError
+from google.protobuf.json_format import MessageToDict
+from lbry.schema.types.v2.purchase_pb2 import Purchase as PurchaseMessage
+from .attrs import ClaimReference
+
+
+class Purchase(ClaimReference):
+
+    START_BYTE = ord('P')
+
+    __slots__ = ()
+
+    def __init__(self, claim_id=None):
+        super().__init__(PurchaseMessage())
+        if claim_id is not None:
+            self.claim_id = claim_id
+
+    def to_dict(self):
+        return MessageToDict(self.message)
+
+    def to_message_bytes(self) -> bytes:
+        return self.message.SerializeToString()
+
+    def to_bytes(self) -> bytes:
+        pieces = bytearray()
+        pieces.append(self.START_BYTE)
+        pieces.extend(self.to_message_bytes())
+        return bytes(pieces)
+
+    @classmethod
+    def has_start_byte(cls, data: bytes):
+        return data and data[0] == cls.START_BYTE
+
+    @classmethod
+    def from_bytes(cls, data: bytes):
+        purchase = cls()
+        if purchase.has_start_byte(data):
+            purchase.message.ParseFromString(data[1:])
+        else:
+            raise DecodeError('Message does not start with correct byte.')
+        return purchase
+
+    def __len__(self):
+        return len(self.to_bytes())
+
+    def __bytes__(self):
+        return self.to_bytes()

lbry/schema/result.py

@@ -0,0 +1,242 @@
+import base64
+from typing import List, Union, Optional, NamedTuple
+from binascii import hexlify
+from itertools import chain
+
+from lbry.error import ResolveCensoredError
+from lbry.schema.types.v2.result_pb2 import Outputs as OutputsMessage
+from lbry.schema.types.v2.result_pb2 import Error as ErrorMessage
+
+INVALID = ErrorMessage.Code.Name(ErrorMessage.INVALID)
+NOT_FOUND = ErrorMessage.Code.Name(ErrorMessage.NOT_FOUND)
+BLOCKED = ErrorMessage.Code.Name(ErrorMessage.BLOCKED)
+
+
+def set_reference(reference, claim_hash, rows):
+    if claim_hash:
+        for txo in rows:
+            if claim_hash == txo.claim_hash:
+                reference.tx_hash = txo.tx_hash
+                reference.nout = txo.position
+                reference.height = txo.height
+                return
+
+
+class ResolveResult(NamedTuple):
+    name: str
+    normalized_name: str
+    claim_hash: bytes
+    tx_num: int
+    position: int
+    tx_hash: bytes
+    height: int
+    amount: int
+    short_url: str
+    is_controlling: bool
+    canonical_url: str
+    creation_height: int
+    activation_height: int
+    expiration_height: int
+    effective_amount: int
+    support_amount: int
+    reposted: int
+    last_takeover_height: Optional[int]
+    claims_in_channel: Optional[int]
+    channel_hash: Optional[bytes]
+    reposted_claim_hash: Optional[bytes]
+    signature_valid: Optional[bool]
+
+
+class Censor:
+
+    NOT_CENSORED = 0
+    SEARCH = 1
+    RESOLVE = 2
+
+    __slots__ = 'censor_type', 'censored'
+
+    def __init__(self, censor_type):
+        self.censor_type = censor_type
+        self.censored = {}
+
+    def is_censored(self, row):
+        return (row.get('censor_type') or self.NOT_CENSORED) >= self.censor_type
+
+    def apply(self, rows):
+        return [row for row in rows if not self.censor(row)]
+
+    def censor(self, row) -> Optional[bytes]:
+        if self.is_censored(row):
+            censoring_channel_hash = bytes.fromhex(row['censoring_channel_id'])[::-1]
+            self.censored.setdefault(censoring_channel_hash, set())
+            self.censored[censoring_channel_hash].add(row['tx_hash'])
+            return censoring_channel_hash
+        return None
+
+    def to_message(self, outputs: OutputsMessage, extra_txo_rows: dict):
+        for censoring_channel_hash, count in self.censored.items():
+            blocked = outputs.blocked.add()
+            blocked.count = len(count)
+            set_reference(blocked.channel, censoring_channel_hash, extra_txo_rows)
+            outputs.blocked_total += len(count)
+
+
+class Outputs:
+
+    __slots__ = 'txos', 'extra_txos', 'txs', 'offset', 'total', 'blocked', 'blocked_total'
+
+    def __init__(self, txos: List, extra_txos: List, txs: set,
+                 offset: int, total: int, blocked: List, blocked_total: int):
+        self.txos = txos
+        self.txs = txs
+        self.extra_txos = extra_txos
+        self.offset = offset
+        self.total = total
+        self.blocked = blocked
+        self.blocked_total = blocked_total
+
+    def inflate(self, txs):
+        tx_map = {tx.hash: tx for tx in txs}
+        for txo_message in self.extra_txos:
+            self.message_to_txo(txo_message, tx_map)
+        txos = [self.message_to_txo(txo_message, tx_map) for txo_message in self.txos]
+        return txos, self.inflate_blocked(tx_map)
+
+    def inflate_blocked(self, tx_map):
+        return {
+            "total": self.blocked_total,
+            "channels": [{
+                'channel': self.message_to_txo(blocked.channel, tx_map),
+                'blocked': blocked.count
+            } for blocked in self.blocked]
+        }
+
+    def message_to_txo(self, txo_message, tx_map):
+        if txo_message.WhichOneof('meta') == 'error':
+            error = {
+                'error': {
+                    'name': txo_message.error.Code.Name(txo_message.error.code),
+                    'text': txo_message.error.text,
+                }
+            }
+            if error['error']['name'] == BLOCKED:
+                error['error']['censor'] = self.message_to_txo(
+                    txo_message.error.blocked.channel, tx_map
+                )
+            return error
+
+        tx = tx_map.get(txo_message.tx_hash)
+        if not tx:
+            return
+        txo = tx.outputs[txo_message.nout]
+        if txo_message.WhichOneof('meta') == 'claim':
+            claim = txo_message.claim
+            txo.meta = {
+                'short_url': f'lbry://{claim.short_url}',
+                'canonical_url': f'lbry://{claim.canonical_url or claim.short_url}',
+                'reposted': claim.reposted,
+                'is_controlling': claim.is_controlling,
+                'take_over_height': claim.take_over_height,
+                'creation_height': claim.creation_height,
+                'activation_height': claim.activation_height,
+                'expiration_height': claim.expiration_height,
+                'effective_amount': claim.effective_amount,
+                'support_amount': claim.support_amount,
+                # 'trending_group': claim.trending_group,
+                # 'trending_mixed': claim.trending_mixed,
+                # 'trending_local': claim.trending_local,
+                # 'trending_global': claim.trending_global,
+            }
+            if claim.HasField('channel'):
+                txo.channel = tx_map[claim.channel.tx_hash].outputs[claim.channel.nout]
+            if claim.HasField('repost'):
+                txo.reposted_claim = tx_map[claim.repost.tx_hash].outputs[claim.repost.nout]
+            try:
+                if txo.claim.is_channel:
+                    txo.meta['claims_in_channel'] = claim.claims_in_channel
+            except:
+                pass
+        return txo
+
+    @classmethod
+    def from_base64(cls, data: str) -> 'Outputs':
+        return cls.from_bytes(base64.b64decode(data))
+
+    @classmethod
+    def from_bytes(cls, data: bytes) -> 'Outputs':
+        outputs = OutputsMessage()
+        outputs.ParseFromString(data)
+        txs = set()
+        for txo_message in chain(outputs.txos, outputs.extra_txos):
+            if txo_message.WhichOneof('meta') == 'error':
+                continue
+            txs.add((hexlify(txo_message.tx_hash[::-1]).decode(), txo_message.height))
+        return cls(
+            outputs.txos, outputs.extra_txos, txs,
+            outputs.offset, outputs.total,
+            outputs.blocked, outputs.blocked_total
+        )
+
+    @classmethod
+    def to_base64(cls, txo_rows, extra_txo_rows, offset=0, total=None, blocked=None) -> str:
+        return base64.b64encode(cls.to_bytes(txo_rows, extra_txo_rows, offset, total, blocked)).decode()
+
+    @classmethod
+    def to_bytes(cls, txo_rows, extra_txo_rows, offset=0, total=None, blocked: Censor = None) -> bytes:
+        page = OutputsMessage()
+        page.offset = offset
+        if total is not None:
+            page.total = total
+        if blocked is not None:
+            blocked.to_message(page, extra_txo_rows)
+        for row in extra_txo_rows:
+            txo_message: 'OutputsMessage' = page.extra_txos.add()
+            if not isinstance(row, Exception):
+                if row.channel_hash:
+                    set_reference(txo_message.claim.channel, row.channel_hash, extra_txo_rows)
+                if row.reposted_claim_hash:
+                    set_reference(txo_message.claim.repost, row.reposted_claim_hash, extra_txo_rows)
+            cls.encode_txo(txo_message, row)
+
+        for row in txo_rows:
+            # cls.row_to_message(row, page.txos.add(), extra_txo_rows)
+            txo_message: 'OutputsMessage' = page.txos.add()
+            cls.encode_txo(txo_message, row)
+            if not isinstance(row, Exception):
+                if row.channel_hash:
+                    set_reference(txo_message.claim.channel, row.channel_hash, extra_txo_rows)
+                if row.reposted_claim_hash:
+                    set_reference(txo_message.claim.repost, row.reposted_claim_hash, extra_txo_rows)
+            elif isinstance(row, ResolveCensoredError):
+                set_reference(txo_message.error.blocked.channel, row.censor_id, extra_txo_rows)
+        return page.SerializeToString()
+
+    @classmethod
+    def encode_txo(cls, txo_message, resolve_result: Union['ResolveResult', Exception]):
+        if isinstance(resolve_result, Exception):
+            txo_message.error.text = resolve_result.args[0]
+            if isinstance(resolve_result, ValueError):
+                txo_message.error.code = ErrorMessage.INVALID
+            elif isinstance(resolve_result, LookupError):
+                txo_message.error.code = ErrorMessage.NOT_FOUND
+            elif isinstance(resolve_result, ResolveCensoredError):
+                txo_message.error.code = ErrorMessage.BLOCKED
+            return
+        txo_message.tx_hash = resolve_result.tx_hash
+        txo_message.nout = resolve_result.position
+        txo_message.height = resolve_result.height
+        txo_message.claim.short_url = resolve_result.short_url
+        txo_message.claim.reposted = resolve_result.reposted
+        txo_message.claim.is_controlling = resolve_result.is_controlling
+        txo_message.claim.creation_height = resolve_result.creation_height
+        txo_message.claim.activation_height = resolve_result.activation_height
+        txo_message.claim.expiration_height = resolve_result.expiration_height
+        txo_message.claim.effective_amount = resolve_result.effective_amount
+        txo_message.claim.support_amount = resolve_result.support_amount
+
+        if resolve_result.canonical_url is not None:
+            txo_message.claim.canonical_url = resolve_result.canonical_url
+        if resolve_result.last_takeover_height is not None:
+            txo_message.claim.take_over_height = resolve_result.last_takeover_height
+        if resolve_result.claims_in_channel is not None:
+            txo_message.claim.claims_in_channel = resolve_result.claims_in_channel

lbry/schema/support.py

@@ -0,0 +1,23 @@
+from lbry.schema.base import Signable
+from lbry.schema.types.v2.support_pb2 import Support as SupportMessage
+
+
+class Support(Signable):
+    __slots__ = ()
+    message_class = SupportMessage
+
+    @property
+    def emoji(self) -> str:
+        return self.message.emoji
+
+    @emoji.setter
+    def emoji(self, emoji: str):
+        self.message.emoji = emoji
+
+    @property
+    def comment(self) -> str:
+        return self.message.comment
+
+    @comment.setter
+    def comment(self, comment: str):
+        self.message.comment = comment

lbry/schema/tags.py

@@ -0,0 +1,13 @@
+from typing import List
+import re
+
+MULTI_SPACE_RE = re.compile(r"\s{2,}")
+WEIRD_CHARS_RE = re.compile(r"[#!~]")
+
+
+def normalize_tag(tag: str):
+    return MULTI_SPACE_RE.sub(' ', WEIRD_CHARS_RE.sub(' ', tag.lower().replace("'", ""))).strip()
+
+
+def clean_tags(tags: List[str]):
+    return [tag for tag in {normalize_tag(tag) for tag in tags} if tag]

lbry/schema/types/v1/certificate_pb2.py

@@ -0,0 +1,146 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: certificate.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf.internal import enum_type_wrapper
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='certificate.proto',
+  package='legacy_pb',
+  syntax='proto2',
+  serialized_options=None,
+  serialized_pb=_b('\n\x11\x63\x65rtificate.proto\x12\tlegacy_pb\"\xa2\x01\n\x0b\x43\x65rtificate\x12/\n\x07version\x18\x01 \x02(\x0e\x32\x1e.legacy_pb.Certificate.Version\x12#\n\x07keyType\x18\x02 \x02(\x0e\x32\x12.legacy_pb.KeyType\x12\x11\n\tpublicKey\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01*Q\n\x07KeyType\x12\x1b\n\x17UNKNOWN_PUBLIC_KEY_TYPE\x10\x00\x12\x0c\n\x08NIST256p\x10\x01\x12\x0c\n\x08NIST384p\x10\x02\x12\r\n\tSECP256k1\x10\x03')
+)
+
+_KEYTYPE = _descriptor.EnumDescriptor(
+  name='KeyType',
+  full_name='legacy_pb.KeyType',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='UNKNOWN_PUBLIC_KEY_TYPE', index=0, number=0,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='NIST256p', index=1, number=1,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='NIST384p', index=2, number=2,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='SECP256k1', index=3, number=3,
+      serialized_options=None,
+      type=None),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=197,
+  serialized_end=278,
+)
+_sym_db.RegisterEnumDescriptor(_KEYTYPE)
+
+KeyType = enum_type_wrapper.EnumTypeWrapper(_KEYTYPE)
+UNKNOWN_PUBLIC_KEY_TYPE = 0
+NIST256p = 1
+NIST384p = 2
+SECP256k1 = 3
+
+
+_CERTIFICATE_VERSION = _descriptor.EnumDescriptor(
+  name='Version',
+  full_name='legacy_pb.Certificate.Version',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='UNKNOWN_VERSION', index=0, number=0,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='_0_0_1', index=1, number=1,
+      serialized_options=None,
+      type=None),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=153,
+  serialized_end=195,
+)
+_sym_db.RegisterEnumDescriptor(_CERTIFICATE_VERSION)
+
+
+_CERTIFICATE = _descriptor.Descriptor(
+  name='Certificate',
+  full_name='legacy_pb.Certificate',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='version', full_name='legacy_pb.Certificate.version', index=0,
+      number=1, type=14, cpp_type=8, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='keyType', full_name='legacy_pb.Certificate.keyType', index=1,
+      number=2, type=14, cpp_type=8, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='publicKey', full_name='legacy_pb.Certificate.publicKey', index=2,
+      number=4, type=12, cpp_type=9, label=2,
+      has_default_value=False, default_value=_b(""),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+    _CERTIFICATE_VERSION,
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=33,
+  serialized_end=195,
+)
+
+_CERTIFICATE.fields_by_name['version'].enum_type = _CERTIFICATE_VERSION
+_CERTIFICATE.fields_by_name['keyType'].enum_type = _KEYTYPE
+_CERTIFICATE_VERSION.containing_type = _CERTIFICATE
+DESCRIPTOR.message_types_by_name['Certificate'] = _CERTIFICATE
+DESCRIPTOR.enum_types_by_name['KeyType'] = _KEYTYPE
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Certificate = _reflection.GeneratedProtocolMessageType('Certificate', (_message.Message,), dict(
+  DESCRIPTOR = _CERTIFICATE,
+  __module__ = 'certificate_pb2'
+  # @@protoc_insertion_point(class_scope:legacy_pb.Certificate)
+  ))
+_sym_db.RegisterMessage(Certificate)
+
+
+# @@protoc_insertion_point(module_scope)

lbry/schema/types/v1/fee_pb2.py

@@ -0,0 +1,148 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: fee.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='fee.proto',
+  package='legacy_pb',
+  syntax='proto2',
+  serialized_options=None,
+  serialized_pb=_b('\n\tfee.proto\x12\tlegacy_pb\"\xe3\x01\n\x03\x46\x65\x65\x12\'\n\x07version\x18\x01 \x02(\x0e\x32\x16.legacy_pb.Fee.Version\x12)\n\x08\x63urrency\x18\x02 \x02(\x0e\x32\x17.legacy_pb.Fee.Currency\x12\x0f\n\x07\x61\x64\x64ress\x18\x03 \x02(\x0c\x12\x0e\n\x06\x61mount\x18\x04 \x02(\x02\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\";\n\x08\x43urrency\x12\x14\n\x10UNKNOWN_CURRENCY\x10\x00\x12\x07\n\x03LBC\x10\x01\x12\x07\n\x03\x42TC\x10\x02\x12\x07\n\x03USD\x10\x03')
+)
+
+
+
+_FEE_VERSION = _descriptor.EnumDescriptor(
+  name='Version',
+  full_name='legacy_pb.Fee.Version',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='UNKNOWN_VERSION', index=0, number=0,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='_0_0_1', index=1, number=1,
+      serialized_options=None,
+      type=None),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=149,
+  serialized_end=191,
+)
+_sym_db.RegisterEnumDescriptor(_FEE_VERSION)
+
+_FEE_CURRENCY = _descriptor.EnumDescriptor(
+  name='Currency',
+  full_name='legacy_pb.Fee.Currency',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='UNKNOWN_CURRENCY', index=0, number=0,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='LBC', index=1, number=1,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='BTC', index=2, number=2,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='USD', index=3, number=3,
+      serialized_options=None,
+      type=None),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=193,
+  serialized_end=252,
+)
+_sym_db.RegisterEnumDescriptor(_FEE_CURRENCY)
+
+
+_FEE = _descriptor.Descriptor(
+  name='Fee',
+  full_name='legacy_pb.Fee',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='version', full_name='legacy_pb.Fee.version', index=0,
+      number=1, type=14, cpp_type=8, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='currency', full_name='legacy_pb.Fee.currency', index=1,
+      number=2, type=14, cpp_type=8, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='address', full_name='legacy_pb.Fee.address', index=2,
+      number=3, type=12, cpp_type=9, label=2,
+      has_default_value=False, default_value=_b(""),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='amount', full_name='legacy_pb.Fee.amount', index=3,
+      number=4, type=2, cpp_type=6, label=2,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+    _FEE_VERSION,
+    _FEE_CURRENCY,
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=25,
+  serialized_end=252,
+)
+
+_FEE.fields_by_name['version'].enum_type = _FEE_VERSION
+_FEE.fields_by_name['currency'].enum_type = _FEE_CURRENCY
+_FEE_VERSION.containing_type = _FEE
+_FEE_CURRENCY.containing_type = _FEE
+DESCRIPTOR.message_types_by_name['Fee'] = _FEE
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Fee = _reflection.GeneratedProtocolMessageType('Fee', (_message.Message,), dict(
+  DESCRIPTOR = _FEE,
+  __module__ = 'fee_pb2'
+  # @@protoc_insertion_point(class_scope:legacy_pb.Fee)
+  ))
+_sym_db.RegisterMessage(Fee)
+
+
+# @@protoc_insertion_point(module_scope)

lbry/schema/types/v1/legacy_claim_pb2.py

@@ -0,0 +1,158 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: legacy_claim.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from . import stream_pb2 as stream__pb2
+from . import certificate_pb2 as certificate__pb2
+from . import signature_pb2 as signature__pb2
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='legacy_claim.proto',
+  package='legacy_pb',
+  syntax='proto2',
+  serialized_options=None,
+  serialized_pb=_b('\n\x12legacy_claim.proto\x12\tlegacy_pb\x1a\x0cstream.proto\x1a\x11\x63\x65rtificate.proto\x1a\x0fsignature.proto\"\xd9\x02\n\x05\x43laim\x12)\n\x07version\x18\x01 \x02(\x0e\x32\x18.legacy_pb.Claim.Version\x12-\n\tclaimType\x18\x02 \x02(\x0e\x32\x1a.legacy_pb.Claim.ClaimType\x12!\n\x06stream\x18\x03 \x01(\x0b\x32\x11.legacy_pb.Stream\x12+\n\x0b\x63\x65rtificate\x18\x04 \x01(\x0b\x32\x16.legacy_pb.Certificate\x12\x30\n\x12publisherSignature\x18\x05 \x01(\x0b\x32\x14.legacy_pb.Signature\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"H\n\tClaimType\x12\x16\n\x12UNKNOWN_CLAIM_TYPE\x10\x00\x12\x0e\n\nstreamType\x10\x01\x12\x13\n\x0f\x63\x65rtificateType\x10\x02')
+  ,
+  dependencies=[stream__pb2.DESCRIPTOR,certificate__pb2.DESCRIPTOR,signature__pb2.DESCRIPTOR,])
+
+
+
+_CLAIM_VERSION = _descriptor.EnumDescriptor(
+  name='Version',
+  full_name='legacy_pb.Claim.Version',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='UNKNOWN_VERSION', index=0, number=0,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='_0_0_1', index=1, number=1,
+      serialized_options=None,
+      type=None),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=313,
+  serialized_end=355,
+)
+_sym_db.RegisterEnumDescriptor(_CLAIM_VERSION)
+
+_CLAIM_CLAIMTYPE = _descriptor.EnumDescriptor(
+  name='ClaimType',
+  full_name='legacy_pb.Claim.ClaimType',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='UNKNOWN_CLAIM_TYPE', index=0, number=0,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='streamType', index=1, number=1,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='certificateType', index=2, number=2,
+      serialized_options=None,
+      type=None),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=357,
+  serialized_end=429,
+)
+_sym_db.RegisterEnumDescriptor(_CLAIM_CLAIMTYPE)
+
+
+_CLAIM = _descriptor.Descriptor(
+  name='Claim',
+  full_name='legacy_pb.Claim',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='version', full_name='legacy_pb.Claim.version', index=0,
+      number=1, type=14, cpp_type=8, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='claimType', full_name='legacy_pb.Claim.claimType', index=1,
+      number=2, type=14, cpp_type=8, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='stream', full_name='legacy_pb.Claim.stream', index=2,
+      number=3, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='certificate', full_name='legacy_pb.Claim.certificate', index=3,
+      number=4, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='publisherSignature', full_name='legacy_pb.Claim.publisherSignature', index=4,
+      number=5, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+    _CLAIM_VERSION,
+    _CLAIM_CLAIMTYPE,
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=84,
+  serialized_end=429,
+)
+
+_CLAIM.fields_by_name['version'].enum_type = _CLAIM_VERSION
+_CLAIM.fields_by_name['claimType'].enum_type = _CLAIM_CLAIMTYPE
+_CLAIM.fields_by_name['stream'].message_type = stream__pb2._STREAM
+_CLAIM.fields_by_name['certificate'].message_type = certificate__pb2._CERTIFICATE
+_CLAIM.fields_by_name['publisherSignature'].message_type = signature__pb2._SIGNATURE
+_CLAIM_VERSION.containing_type = _CLAIM
+_CLAIM_CLAIMTYPE.containing_type = _CLAIM
+DESCRIPTOR.message_types_by_name['Claim'] = _CLAIM
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Claim = _reflection.GeneratedProtocolMessageType('Claim', (_message.Message,), dict(
+  DESCRIPTOR = _CLAIM,
+  __module__ = 'legacy_claim_pb2'
+  # @@protoc_insertion_point(class_scope:legacy_pb.Claim)
+  ))
+_sym_db.RegisterMessage(Claim)
+
+
+# @@protoc_insertion_point(module_scope)

lbry/schema/types/v1/signature_pb2.py

@@ -0,0 +1,118 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: signature.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from . import certificate_pb2 as certificate__pb2
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='signature.proto',
+  package='legacy_pb',
+  syntax='proto2',
+  serialized_options=None,
+  serialized_pb=_b('\n\x0fsignature.proto\x12\tlegacy_pb\x1a\x11\x63\x65rtificate.proto\"\xbb\x01\n\tSignature\x12-\n\x07version\x18\x01 \x02(\x0e\x32\x1c.legacy_pb.Signature.Version\x12)\n\rsignatureType\x18\x02 \x02(\x0e\x32\x12.legacy_pb.KeyType\x12\x11\n\tsignature\x18\x03 \x02(\x0c\x12\x15\n\rcertificateId\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
+  ,
+  dependencies=[certificate__pb2.DESCRIPTOR,])
+
+
+
+_SIGNATURE_VERSION = _descriptor.EnumDescriptor(
+  name='Version',
+  full_name='legacy_pb.Signature.Version',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='UNKNOWN_VERSION', index=0, number=0,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='_0_0_1', index=1, number=1,
+      serialized_options=None,
+      type=None),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=195,
+  serialized_end=237,
+)
+_sym_db.RegisterEnumDescriptor(_SIGNATURE_VERSION)
+
+
+_SIGNATURE = _descriptor.Descriptor(
+  name='Signature',
+  full_name='legacy_pb.Signature',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='version', full_name='legacy_pb.Signature.version', index=0,
+      number=1, type=14, cpp_type=8, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='signatureType', full_name='legacy_pb.Signature.signatureType', index=1,
+      number=2, type=14, cpp_type=8, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='signature', full_name='legacy_pb.Signature.signature', index=2,
+      number=3, type=12, cpp_type=9, label=2,
+      has_default_value=False, default_value=_b(""),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='certificateId', full_name='legacy_pb.Signature.certificateId', index=3,
+      number=4, type=12, cpp_type=9, label=2,
+      has_default_value=False, default_value=_b(""),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+    _SIGNATURE_VERSION,
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=50,
+  serialized_end=237,
+)
+
+_SIGNATURE.fields_by_name['version'].enum_type = _SIGNATURE_VERSION
+_SIGNATURE.fields_by_name['signatureType'].enum_type = certificate__pb2._KEYTYPE
+_SIGNATURE_VERSION.containing_type = _SIGNATURE
+DESCRIPTOR.message_types_by_name['Signature'] = _SIGNATURE
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Signature = _reflection.GeneratedProtocolMessageType('Signature', (_message.Message,), dict(
+  DESCRIPTOR = _SIGNATURE,
+  __module__ = 'signature_pb2'
+  # @@protoc_insertion_point(class_scope:legacy_pb.Signature)
+  ))
+_sym_db.RegisterMessage(Signature)
+
+
+# @@protoc_insertion_point(module_scope)

lbry/schema/types/v1/source_pb2.py

@@ -0,0 +1,140 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: source.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='source.proto',
+  package='legacy_pb',
+  syntax='proto2',
+  serialized_options=None,
+  serialized_pb=_b('\n\x0csource.proto\x12\tlegacy_pb\"\xf2\x01\n\x06Source\x12*\n\x07version\x18\x01 \x02(\x0e\x32\x19.legacy_pb.Source.Version\x12\x31\n\nsourceType\x18\x02 \x02(\x0e\x32\x1d.legacy_pb.Source.SourceTypes\x12\x0e\n\x06source\x18\x03 \x02(\x0c\x12\x13\n\x0b\x63ontentType\x18\x04 \x02(\t\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"8\n\x0bSourceTypes\x12\x17\n\x13UNKNOWN_SOURCE_TYPE\x10\x00\x12\x10\n\x0clbry_sd_hash\x10\x01')
+)
+
+
+
+_SOURCE_VERSION = _descriptor.EnumDescriptor(
+  name='Version',
+  full_name='legacy_pb.Source.Version',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='UNKNOWN_VERSION', index=0, number=0,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='_0_0_1', index=1, number=1,
+      serialized_options=None,
+      type=None),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=170,
+  serialized_end=212,
+)
+_sym_db.RegisterEnumDescriptor(_SOURCE_VERSION)
+
+_SOURCE_SOURCETYPES = _descriptor.EnumDescriptor(
+  name='SourceTypes',
+  full_name='legacy_pb.Source.SourceTypes',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='UNKNOWN_SOURCE_TYPE', index=0, number=0,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='lbry_sd_hash', index=1, number=1,
+      serialized_options=None,
+      type=None),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=214,
+  serialized_end=270,
+)
+_sym_db.RegisterEnumDescriptor(_SOURCE_SOURCETYPES)
+
+
+_SOURCE = _descriptor.Descriptor(
+  name='Source',
+  full_name='legacy_pb.Source',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='version', full_name='legacy_pb.Source.version', index=0,
+      number=1, type=14, cpp_type=8, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='sourceType', full_name='legacy_pb.Source.sourceType', index=1,
+      number=2, type=14, cpp_type=8, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='source', full_name='legacy_pb.Source.source', index=2,
+      number=3, type=12, cpp_type=9, label=2,
+      has_default_value=False, default_value=_b(""),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='contentType', full_name='legacy_pb.Source.contentType', index=3,
+      number=4, type=9, cpp_type=9, label=2,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+    _SOURCE_VERSION,
+    _SOURCE_SOURCETYPES,
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=28,
+  serialized_end=270,
+)
+
+_SOURCE.fields_by_name['version'].enum_type = _SOURCE_VERSION
+_SOURCE.fields_by_name['sourceType'].enum_type = _SOURCE_SOURCETYPES
+_SOURCE_VERSION.containing_type = _SOURCE
+_SOURCE_SOURCETYPES.containing_type = _SOURCE
+DESCRIPTOR.message_types_by_name['Source'] = _SOURCE
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Source = _reflection.GeneratedProtocolMessageType('Source', (_message.Message,), dict(
+  DESCRIPTOR = _SOURCE,
+  __module__ = 'source_pb2'
+  # @@protoc_insertion_point(class_scope:legacy_pb.Source)
+  ))
+_sym_db.RegisterMessage(Source)
+
+
+# @@protoc_insertion_point(module_scope)

lbry/schema/types/v1/stream_pb2.py

@@ -0,0 +1,113 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: stream.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from . import metadata_pb2 as metadata__pb2
+from . import source_pb2 as source__pb2
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='stream.proto',
+  package='legacy_pb',
+  syntax='proto2',
+  serialized_options=None,
+  serialized_pb=_b('\n\x0cstream.proto\x12\tlegacy_pb\x1a\x0emetadata.proto\x1a\x0csource.proto\"\xaa\x01\n\x06Stream\x12*\n\x07version\x18\x01 \x02(\x0e\x32\x19.legacy_pb.Stream.Version\x12%\n\x08metadata\x18\x02 \x02(\x0b\x32\x13.legacy_pb.Metadata\x12!\n\x06source\x18\x03 \x02(\x0b\x32\x11.legacy_pb.Source\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
+  ,
+  dependencies=[metadata__pb2.DESCRIPTOR,source__pb2.DESCRIPTOR,])
+
+
+
+_STREAM_VERSION = _descriptor.EnumDescriptor(
+  name='Version',
+  full_name='legacy_pb.Stream.Version',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='UNKNOWN_VERSION', index=0, number=0,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='_0_0_1', index=1, number=1,
+      serialized_options=None,
+      type=None),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=186,
+  serialized_end=228,
+)
+_sym_db.RegisterEnumDescriptor(_STREAM_VERSION)
+
+
+_STREAM = _descriptor.Descriptor(
+  name='Stream',
+  full_name='legacy_pb.Stream',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='version', full_name='legacy_pb.Stream.version', index=0,
+      number=1, type=14, cpp_type=8, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='metadata', full_name='legacy_pb.Stream.metadata', index=1,
+      number=2, type=11, cpp_type=10, label=2,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='source', full_name='legacy_pb.Stream.source', index=2,
+      number=3, type=11, cpp_type=10, label=2,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+    _STREAM_VERSION,
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=58,
+  serialized_end=228,
+)
+
+_STREAM.fields_by_name['version'].enum_type = _STREAM_VERSION
+_STREAM.fields_by_name['metadata'].message_type = metadata__pb2._METADATA
+_STREAM.fields_by_name['source'].message_type = source__pb2._SOURCE
+_STREAM_VERSION.containing_type = _STREAM
+DESCRIPTOR.message_types_by_name['Stream'] = _STREAM
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Stream = _reflection.GeneratedProtocolMessageType('Stream', (_message.Message,), dict(
+  DESCRIPTOR = _STREAM,
+  __module__ = 'stream_pb2'
+  # @@protoc_insertion_point(class_scope:legacy_pb.Stream)
+  ))
+_sym_db.RegisterMessage(Stream)
+
+
+# @@protoc_insertion_point(module_scope)

lbry/schema/types/v2/purchase_pb2.py

@@ -0,0 +1,69 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: purchase.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf import descriptor_pb2
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='purchase.proto',
+  package='pb',
+  syntax='proto3',
+  serialized_pb=_b('\n\x0epurchase.proto\x12\x02pb\"\x1e\n\x08Purchase\x12\x12\n\nclaim_hash\x18\x01 \x01(\x0c\x62\x06proto3')
+)
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+
+
+
+_PURCHASE = _descriptor.Descriptor(
+  name='Purchase',
+  full_name='pb.Purchase',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='claim_hash', full_name='pb.Purchase.claim_hash', index=0,
+      number=1, type=12, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b(""),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=22,
+  serialized_end=52,
+)
+
+DESCRIPTOR.message_types_by_name['Purchase'] = _PURCHASE
+
+Purchase = _reflection.GeneratedProtocolMessageType('Purchase', (_message.Message,), dict(
+  DESCRIPTOR = _PURCHASE,
+  __module__ = 'purchase_pb2'
+  # @@protoc_insertion_point(class_scope:pb.Purchase)
+  ))
+_sym_db.RegisterMessage(Purchase)
+
+
+# @@protoc_insertion_point(module_scope)

lbry/schema/types/v2/result_pb2.py

@@ -0,0 +1,464 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: result.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='result.proto',
+  package='pb',
+  syntax='proto3',
+  serialized_options=b'Z$github.com/lbryio/hub/protobuf/go/pb',
+  create_key=_descriptor._internal_create_key,
+  serialized_pb=b'\n\x0cresult.proto\x12\x02pb\"\x97\x01\n\x07Outputs\x12\x18\n\x04txos\x18\x01 \x03(\x0b\x32\n.pb.Output\x12\x1e\n\nextra_txos\x18\x02 \x03(\x0b\x32\n.pb.Output\x12\r\n\x05total\x18\x03 \x01(\r\x12\x0e\n\x06offset\x18\x04 \x01(\r\x12\x1c\n\x07\x62locked\x18\x05 \x03(\x0b\x32\x0b.pb.Blocked\x12\x15\n\rblocked_total\x18\x06 \x01(\r\"{\n\x06Output\x12\x0f\n\x07tx_hash\x18\x01 \x01(\x0c\x12\x0c\n\x04nout\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\x1e\n\x05\x63laim\x18\x07 \x01(\x0b\x32\r.pb.ClaimMetaH\x00\x12\x1a\n\x05\x65rror\x18\x0f \x01(\x0b\x32\t.pb.ErrorH\x00\x42\x06\n\x04meta\"\xe6\x02\n\tClaimMeta\x12\x1b\n\x07\x63hannel\x18\x01 \x01(\x0b\x32\n.pb.Output\x12\x1a\n\x06repost\x18\x02 \x01(\x0b\x32\n.pb.Output\x12\x11\n\tshort_url\x18\x03 \x01(\t\x12\x15\n\rcanonical_url\x18\x04 \x01(\t\x12\x16\n\x0eis_controlling\x18\x05 \x01(\x08\x12\x18\n\x10take_over_height\x18\x06 \x01(\r\x12\x17\n\x0f\x63reation_height\x18\x07 \x01(\r\x12\x19\n\x11\x61\x63tivation_height\x18\x08 \x01(\r\x12\x19\n\x11\x65xpiration_height\x18\t \x01(\r\x12\x19\n\x11\x63laims_in_channel\x18\n \x01(\r\x12\x10\n\x08reposted\x18\x0b \x01(\r\x12\x18\n\x10\x65\x66\x66\x65\x63tive_amount\x18\x14 \x01(\x04\x12\x16\n\x0esupport_amount\x18\x15 \x01(\x04\x12\x16\n\x0etrending_score\x18\x16 \x01(\x01\"\x94\x01\n\x05\x45rror\x12\x1c\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x0e.pb.Error.Code\x12\x0c\n\x04text\x18\x02 \x01(\t\x12\x1c\n\x07\x62locked\x18\x03 \x01(\x0b\x32\x0b.pb.Blocked\"A\n\x04\x43ode\x12\x10\n\x0cUNKNOWN_CODE\x10\x00\x12\r\n\tNOT_FOUND\x10\x01\x12\x0b\n\x07INVALID\x10\x02\x12\x0b\n\x07\x42LOCKED\x10\x03\"5\n\x07\x42locked\x12\r\n\x05\x63ount\x18\x01 \x01(\r\x12\x1b\n\x07\x63hannel\x18\x02 \x01(\x0b\x32\n.pb.OutputB&Z$github.com/lbryio/hub/protobuf/go/pbb\x06proto3'
+)
+
+
+
+_ERROR_CODE = _descriptor.EnumDescriptor(
+  name='Code',
+  full_name='pb.Error.Code',
+  filename=None,
+  file=DESCRIPTOR,
+  create_key=_descriptor._internal_create_key,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='UNKNOWN_CODE', index=0, number=0,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+    _descriptor.EnumValueDescriptor(
+      name='NOT_FOUND', index=1, number=1,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+    _descriptor.EnumValueDescriptor(
+      name='INVALID', index=2, number=2,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+    _descriptor.EnumValueDescriptor(
+      name='BLOCKED', index=3, number=3,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=744,
+  serialized_end=809,
+)
+_sym_db.RegisterEnumDescriptor(_ERROR_CODE)
+
+
+_OUTPUTS = _descriptor.Descriptor(
+  name='Outputs',
+  full_name='pb.Outputs',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='txos', full_name='pb.Outputs.txos', index=0,
+      number=1, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='extra_txos', full_name='pb.Outputs.extra_txos', index=1,
+      number=2, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='total', full_name='pb.Outputs.total', index=2,
+      number=3, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='offset', full_name='pb.Outputs.offset', index=3,
+      number=4, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='blocked', full_name='pb.Outputs.blocked', index=4,
+      number=5, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='blocked_total', full_name='pb.Outputs.blocked_total', index=5,
+      number=6, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=21,
+  serialized_end=172,
+)
+
+
+_OUTPUT = _descriptor.Descriptor(
+  name='Output',
+  full_name='pb.Output',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='tx_hash', full_name='pb.Output.tx_hash', index=0,
+      number=1, type=12, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"",
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='nout', full_name='pb.Output.nout', index=1,
+      number=2, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='height', full_name='pb.Output.height', index=2,
+      number=3, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='claim', full_name='pb.Output.claim', index=3,
+      number=7, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='error', full_name='pb.Output.error', index=4,
+      number=15, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+    _descriptor.OneofDescriptor(
+      name='meta', full_name='pb.Output.meta',
+      index=0, containing_type=None,
+      create_key=_descriptor._internal_create_key,
+    fields=[]),
+  ],
+  serialized_start=174,
+  serialized_end=297,
+)
+
+
+_CLAIMMETA = _descriptor.Descriptor(
+  name='ClaimMeta',
+  full_name='pb.ClaimMeta',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='channel', full_name='pb.ClaimMeta.channel', index=0,
+      number=1, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='repost', full_name='pb.ClaimMeta.repost', index=1,
+      number=2, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='short_url', full_name='pb.ClaimMeta.short_url', index=2,
+      number=3, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='canonical_url', full_name='pb.ClaimMeta.canonical_url', index=3,
+      number=4, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='is_controlling', full_name='pb.ClaimMeta.is_controlling', index=4,
+      number=5, type=8, cpp_type=7, label=1,
+      has_default_value=False, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='take_over_height', full_name='pb.ClaimMeta.take_over_height', index=5,
+      number=6, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='creation_height', full_name='pb.ClaimMeta.creation_height', index=6,
+      number=7, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='activation_height', full_name='pb.ClaimMeta.activation_height', index=7,
+      number=8, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='expiration_height', full_name='pb.ClaimMeta.expiration_height', index=8,
+      number=9, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='claims_in_channel', full_name='pb.ClaimMeta.claims_in_channel', index=9,
+      number=10, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='reposted', full_name='pb.ClaimMeta.reposted', index=10,
+      number=11, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='effective_amount', full_name='pb.ClaimMeta.effective_amount', index=11,
+      number=20, type=4, cpp_type=4, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='support_amount', full_name='pb.ClaimMeta.support_amount', index=12,
+      number=21, type=4, cpp_type=4, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='trending_score', full_name='pb.ClaimMeta.trending_score', index=13,
+      number=22, type=1, cpp_type=5, label=1,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=300,
+  serialized_end=658,
+)
+
+
+_ERROR = _descriptor.Descriptor(
+  name='Error',
+  full_name='pb.Error',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='code', full_name='pb.Error.code', index=0,
+      number=1, type=14, cpp_type=8, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='text', full_name='pb.Error.text', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='blocked', full_name='pb.Error.blocked', index=2,
+      number=3, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+    _ERROR_CODE,
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=661,
+  serialized_end=809,
+)
+
+
+_BLOCKED = _descriptor.Descriptor(
+  name='Blocked',
+  full_name='pb.Blocked',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='count', full_name='pb.Blocked.count', index=0,
+      number=1, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='channel', full_name='pb.Blocked.channel', index=1,
+      number=2, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=811,
+  serialized_end=864,
+)
+
+_OUTPUTS.fields_by_name['txos'].message_type = _OUTPUT
+_OUTPUTS.fields_by_name['extra_txos'].message_type = _OUTPUT
+_OUTPUTS.fields_by_name['blocked'].message_type = _BLOCKED
+_OUTPUT.fields_by_name['claim'].message_type = _CLAIMMETA
+_OUTPUT.fields_by_name['error'].message_type = _ERROR
+_OUTPUT.oneofs_by_name['meta'].fields.append(
+  _OUTPUT.fields_by_name['claim'])
+_OUTPUT.fields_by_name['claim'].containing_oneof = _OUTPUT.oneofs_by_name['meta']
+_OUTPUT.oneofs_by_name['meta'].fields.append(
+  _OUTPUT.fields_by_name['error'])
+_OUTPUT.fields_by_name['error'].containing_oneof = _OUTPUT.oneofs_by_name['meta']
+_CLAIMMETA.fields_by_name['channel'].message_type = _OUTPUT
+_CLAIMMETA.fields_by_name['repost'].message_type = _OUTPUT
+_ERROR.fields_by_name['code'].enum_type = _ERROR_CODE
+_ERROR.fields_by_name['blocked'].message_type = _BLOCKED
+_ERROR_CODE.containing_type = _ERROR
+_BLOCKED.fields_by_name['channel'].message_type = _OUTPUT
+DESCRIPTOR.message_types_by_name['Outputs'] = _OUTPUTS
+DESCRIPTOR.message_types_by_name['Output'] = _OUTPUT
+DESCRIPTOR.message_types_by_name['ClaimMeta'] = _CLAIMMETA
+DESCRIPTOR.message_types_by_name['Error'] = _ERROR
+DESCRIPTOR.message_types_by_name['Blocked'] = _BLOCKED
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Outputs = _reflection.GeneratedProtocolMessageType('Outputs', (_message.Message,), {
+  'DESCRIPTOR' : _OUTPUTS,
+  '__module__' : 'result_pb2'
+  # @@protoc_insertion_point(class_scope:pb.Outputs)
+  })
+_sym_db.RegisterMessage(Outputs)
+
+Output = _reflection.GeneratedProtocolMessageType('Output', (_message.Message,), {
+  'DESCRIPTOR' : _OUTPUT,
+  '__module__' : 'result_pb2'
+  # @@protoc_insertion_point(class_scope:pb.Output)
+  })
+_sym_db.RegisterMessage(Output)
+
+ClaimMeta = _reflection.GeneratedProtocolMessageType('ClaimMeta', (_message.Message,), {
+  'DESCRIPTOR' : _CLAIMMETA,
+  '__module__' : 'result_pb2'
+  # @@protoc_insertion_point(class_scope:pb.ClaimMeta)
+  })
+_sym_db.RegisterMessage(ClaimMeta)
+
+Error = _reflection.GeneratedProtocolMessageType('Error', (_message.Message,), {
+  'DESCRIPTOR' : _ERROR,
+  '__module__' : 'result_pb2'
+  # @@protoc_insertion_point(class_scope:pb.Error)
+  })
+_sym_db.RegisterMessage(Error)
+
+Blocked = _reflection.GeneratedProtocolMessageType('Blocked', (_message.Message,), {
+  'DESCRIPTOR' : _BLOCKED,
+  '__module__' : 'result_pb2'
+  # @@protoc_insertion_point(class_scope:pb.Blocked)
+  })
+_sym_db.RegisterMessage(Blocked)
+
+
+DESCRIPTOR._options = None
+# @@protoc_insertion_point(module_scope)

lbry/schema/types/v2/support_pb2.py

@@ -0,0 +1,76 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: support.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf import descriptor_pb2
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='support.proto',
+  package='pb',
+  syntax='proto3',
+  serialized_pb=_b('\n\rsupport.proto\x12\x02pb\")\n\x07Support\x12\r\n\x05\x65moji\x18\x01 \x01(\t\x12\x0f\n\x07\x63omment\x18\x02 \x01(\tb\x06proto3')
+)
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+
+
+
+_SUPPORT = _descriptor.Descriptor(
+  name='Support',
+  full_name='pb.Support',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='emoji', full_name='pb.Support.emoji', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='comment', full_name='pb.Support.comment', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=21,
+  serialized_end=62,
+)
+
+DESCRIPTOR.message_types_by_name['Support'] = _SUPPORT
+
+Support = _reflection.GeneratedProtocolMessageType('Support', (_message.Message,), dict(
+  DESCRIPTOR = _SUPPORT,
+  __module__ = 'support_pb2'
+  # @@protoc_insertion_point(class_scope:pb.Support)
+  ))
+_sym_db.RegisterMessage(Support)
+
+
+# @@protoc_insertion_point(module_scope)

lbry/schema/types/v2/wallet.json

@@ -0,0 +1,139 @@
+{
+  "title": "Wallet",
+  "description": "An LBC wallet",
+  "type": "object",
+  "required": ["name", "version", "accounts", "preferences"],
+  "additionalProperties": false,
+  "properties": {
+    "name": {
+      "description": "Human readable name for this wallet",
+      "type": "string"
+    },
+    "version": {
+      "description": "Wallet spec version",
+      "type": "integer",
+      "$comment": "Should this be a string? We may need some sort of decimal type if we want exact decimal versions."
+    },
+    "accounts": {
+      "description": "Accounts associated with this wallet",
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["address_generator", "certificates", "encrypted", "ledger", "modified_on", "name", "private_key", "public_key", "seed"],
+        "additionalProperties": false,
+        "properties": {
+          "address_generator": {
+            "description": "Higher level manager of either singular or deterministically generated addresses",
+            "type": "object",
+            "oneOf": [
+              {
+                "required": ["name", "change", "receiving"],
+                "additionalProperties": false,
+                "properties": {
+                  "name": {
+                    "description": "type of address generator: a deterministic chain of addresses",
+                    "enum": ["deterministic-chain"],
+                    "type": "string"
+                  },
+                  "change": {
+                    "$ref": "#/$defs/address_manager",
+                    "description": "Manager for deterministically generated change address (not used for single address)"
+                  },
+                  "receiving": {
+                    "$ref": "#/$defs/address_manager",
+                    "description": "Manager for deterministically generated receiving address (not used for single address)"
+                  }
+                }
+              }, {
+                "required": ["name"],
+                "additionalProperties": false,
+                "properties": {
+                  "name": {
+                    "description": "type of address generator: a single address",
+                    "enum": ["single-address"],
+                    "type": "string"
+                  }
+                }
+              }
+            ]
+          },
+          "certificates": {
+            "type": "object",
+            "description": "Channel keys. Mapping from public key address to pem-formatted private key.",
+            "additionalProperties": {"type": "string"}
+          },
+          "encrypted": {
+            "type": "boolean",
+            "description": "Whether private key and seed are encrypted with a password"
+          },
+          "ledger": {
+            "description": "Which network to use",
+            "type": "string",
+            "examples": [
+              "lbc_mainnet",
+              "lbc_testnet"
+            ]
+          },
+          "modified_on": {
+            "description": "last modified time in Unix Time",
+            "type": "integer"
+          },
+          "name": {
+            "description": "Name for account, possibly human readable",
+            "type": "string"
+          },
+          "private_key": {
+            "description": "Private key for address if `address_generator` is a single address. Root of chain of private keys for addresses if `address_generator` is a deterministic chain of addresses. Encrypted if `encrypted` is true.",
+            "type": "string"
+          },
+          "public_key": {
+            "description": "Public key for address if `address_generator` is a single address. Root of chain of public keys for addresses if `address_generator` is a deterministic chain of addresses.",
+            "type": "string"
+          },
+          "seed": {
+            "description": "Human readable representation of `private_key`. encrypted if `encrypted` is set to `true`",
+            "type": "string"
+          }
+        }
+      }
+    },
+    "preferences": {
+      "description": "Timestamped application-level preferences. Values can be objects or of a primitive type.",
+      "$comment": "enable-sync is seen in example wallet. encrypt-on-disk is seen in example wallet. they both have a boolean `value` field. Do we want them explicitly defined here? local and shared seem to have at least a similar structure (type, value [yes, again], version), value being the free-form part. Should we define those here? Or can there be any key under preferences, and `value` be literally be anything in any form?",
+      "type": "object",
+      "additionalProperties": {
+        "type": "object",
+        "required": ["ts", "value"],
+        "additionalProperties": false,
+        "properties": {
+          "ts": {
+            "type": "number",
+            "description": "When the item was set, in Unix time format.",
+            "$comment": "Do we want a string (decimal)?"
+          },
+          "value": {
+            "$comment": "Sometimes this has been an object, sometimes just a boolean. I don't want to prescribe anything."
+          }
+        }
+      }
+    }
+  },
+  "$defs": {
+    "address_manager": {
+      "description": "Manager for deterministically generated addresses",
+      "type": "object",
+      "required": ["gap", "maximum_uses_per_address"],
+      "additionalProperties": false,
+      "properties": {
+        "gap": {
+          "description": "Maximum allowed consecutive generated addresses with no transactions",
+          "type": "integer"
+        },
+        "maximum_uses_per_address": {
+          "description": "Maximum number of uses for each generated address",
+          "type": "integer"
+        }
+      }
+    }
+  }
+}

lbry/schema/url.py

@@ -0,0 +1,130 @@
+import re
+import unicodedata
+from typing import NamedTuple, Tuple
+
+
+def _create_url_regex():
+    # see https://spec.lbry.com/ and test_url.py
+    invalid_names_regex = \
+        r"[^=&#:$@%?;\"/\\<>%{}|^~`\[\]" \
+        r"\u0000-\u0020\uD800-\uDFFF\uFFFE-\uFFFF]+"
+
+    def _named(name, regex):
+        return "(?P<" + name + ">" + regex + ")"
+
+    def _group(regex):
+        return "(?:" + regex + ")"
+
+    def _oneof(*choices):
+        return _group('|'.join(choices))
+
+    def _claim(name, prefix=""):
+        return _group(
+            _named(name+"_name", prefix + invalid_names_regex) +
+            _oneof(
+                _group('[:#]' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
+                _group(r'\$' + _named(name+"_amount_order", '[1-9][0-9]*'))
+            ) + '?'
+        )
+
+    return (
+        '^' +
+        _named("scheme", "lbry://") + '?' +
+        _oneof(
+            _group(_claim("channel_with_stream", "@") + "/" + _claim("stream_in_channel")),
+            _claim("channel", "@"),
+            _claim("stream")
+        ) +
+        '$'
+    )
+
+
+URL_REGEX = _create_url_regex()
+
+
+def normalize_name(name):
+    return unicodedata.normalize('NFD', name).casefold()
+
+
+class PathSegment(NamedTuple):
+    name: str
+    claim_id: str = None
+    amount_order: int = None
+
+    @property
+    def normalized(self):
+        return normalize_name(self.name)
+
+    @property
+    def is_shortid(self):
+        return self.claim_id is not None and len(self.claim_id) < 40
+
+    @property
+    def is_fullid(self):
+        return self.claim_id is not None and len(self.claim_id) == 40
+
+    def to_dict(self):
+        q = {'name': self.name}
+        if self.claim_id is not None:
+            q['claim_id'] = self.claim_id
+        if self.amount_order is not None:
+            q['amount_order'] = self.amount_order
+        return q
+
+    def __str__(self):
+        if self.claim_id is not None:
+            return f"{self.name}:{self.claim_id}"
+        elif self.amount_order is not None:
+            return f"{self.name}${self.amount_order}"
+        return self.name
+
+
+class URL(NamedTuple):
+    stream: PathSegment
+    channel: PathSegment
+
+    @property
+    def has_channel(self):
+        return self.channel is not None
+
+    @property
+    def has_stream(self):
+        return self.stream is not None
+
+    @property
+    def has_stream_in_channel(self):
+        return self.has_channel and self.has_stream
+
+    @property
+    def parts(self) -> Tuple:
+        if self.has_stream_in_channel:
+            return self.channel, self.stream
+        if self.has_channel:
+            return self.channel,
+        return self.stream,
+
+    def __str__(self):
+        return f"lbry://{'/'.join(str(p) for p in self.parts)}"
+
+    @classmethod
+    def parse(cls, url):
+        match = re.match(URL_REGEX, url)
+
+        if match is None:
+            raise ValueError('Invalid LBRY URL')
+
+        segments = {}
+        parts = match.groupdict()
+        for segment in ('channel', 'stream', 'channel_with_stream', 'stream_in_channel'):
+            if parts[f'{segment}_name'] is not None:
+                segments[segment] = PathSegment(
+                    parts[f'{segment}_name'],
+                    parts[f'{segment}_claim_id'],
+                    parts[f'{segment}_amount_order']
+                )
+
+        if 'channel_with_stream' in segments:
+            segments['channel'] = segments['channel_with_stream']
+            segments['stream'] = segments['stream_in_channel']
+
+        return cls(segments.get('stream', None), segments.get('channel', None))

lbry/stream/background_downloader.py

@@ -23,6 +23,7 @@ class BackgroundDownloader:
         except ValueError:
             return
         except asyncio.CancelledError:
+            log.debug("Cancelled background downloader")
             raise
         except Exception:
             log.error("Unexpected download error on background downloader")

lbry/stream/descriptor.py

@@ -194,12 +194,13 @@ class StreamDescriptor:
             raise InvalidStreamDescriptorError("Stream terminator blob should not have a hash")
         if any(i != blob_info['blob_num'] for i, blob_info in enumerate(decoded['blobs'])):
             raise InvalidStreamDescriptorError("Stream contains out of order or skipped blobs")
+        added_on = time.time()
         descriptor = cls(
             loop, blob_dir,
             binascii.unhexlify(decoded['stream_name']).decode(),
             decoded['key'],
             binascii.unhexlify(decoded['suggested_file_name']).decode(),
-            [BlobInfo(info['blob_num'], info['length'], info['iv'], info.get('blob_hash'))
+            [BlobInfo(info['blob_num'], info['length'], info['iv'], added_on, info.get('blob_hash'))
              for info in decoded['blobs']],
             decoded['stream_hash'],
             blob.blob_hash
@@ -266,7 +267,7 @@ class StreamDescriptor:
             blobs.append(blob_info)
         blobs.append(
             # add the stream terminator
-            BlobInfo(len(blobs), 0, binascii.hexlify(next(iv_generator)).decode(), None, added_on, True)
+            BlobInfo(len(blobs), 0, binascii.hexlify(next(iv_generator)).decode(), added_on, None, True)
         )
         file_name = os.path.basename(file_path)
         suggested_file_name = sanitize_file_name(file_name)

lbry/stream/downloader.py

@@ -8,6 +8,8 @@ from lbry.error import DownloadSDTimeoutError
 from lbry.utils import lru_cache_concurrent
 from lbry.stream.descriptor import StreamDescriptor
 from lbry.blob_exchange.downloader import BlobDownloader
+from lbry.torrent.tracker import enqueue_tracker_search
+
 if typing.TYPE_CHECKING:
     from lbry.conf import Config
     from lbry.dht.node import Node
@@ -25,8 +27,8 @@ class StreamDownloader:
         self.config = config
         self.blob_manager = blob_manager
         self.sd_hash = sd_hash
-        self.search_queue = asyncio.Queue(loop=loop)     # blob hashes to feed into the iterative finder
-        self.peer_queue = asyncio.Queue(loop=loop)       # new peers to try
+        self.search_queue = asyncio.Queue()     # blob hashes to feed into the iterative finder
+        self.peer_queue = asyncio.Queue()       # new peers to try
         self.blob_downloader = BlobDownloader(self.loop, self.config, self.blob_manager, self.peer_queue)
         self.descriptor: typing.Optional[StreamDescriptor] = descriptor
         self.node: typing.Optional['Node'] = None
@@ -70,7 +72,7 @@ class StreamDownloader:
                 now = self.loop.time()
                 sd_blob = await asyncio.wait_for(
                     self.blob_downloader.download_blob(self.sd_hash, connection_id),
-                    self.config.blob_download_timeout, loop=self.loop
+                    self.config.blob_download_timeout
                 )
                 log.info("downloaded sd blob %s", self.sd_hash)
                 self.time_to_descriptor = self.loop.time() - now
@@ -91,6 +93,7 @@ class StreamDownloader:
                 self.accumulate_task.cancel()
             _, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue)
         await self.add_fixed_peers()
+        enqueue_tracker_search(bytes.fromhex(self.sd_hash), self.peer_queue)
         # start searching for peers for the sd hash
         self.search_queue.put_nowait(self.sd_hash)
         log.info("searching for peers for stream %s", self.sd_hash)
@@ -108,7 +111,7 @@ class StreamDownloader:
             raise ValueError(f"blob {blob_info.blob_hash} is not part of stream with sd hash {self.sd_hash}")
         blob = await asyncio.wait_for(
             self.blob_downloader.download_blob(blob_info.blob_hash, blob_info.length, connection_id),
-            self.config.blob_download_timeout * 10, loop=self.loop
+            self.config.blob_download_timeout * 10
         )
         return blob
 

lbry/stream/managed_stream.py

@@ -6,7 +6,7 @@ import logging
 from typing import Optional
 from aiohttp.web import Request, StreamResponse, HTTPRequestRangeNotSatisfiable
 from lbry.error import DownloadSDTimeoutError
-from scribe.schema.mime_types import guess_media_type
+from lbry.schema.mime_types import guess_media_type
 from lbry.stream.downloader import StreamDownloader
 from lbry.stream.descriptor import StreamDescriptor, sanitize_file_name
 from lbry.stream.reflector.client import StreamReflectorClient
@@ -16,10 +16,8 @@ from lbry.file.source import ManagedDownloadSource
 
 if typing.TYPE_CHECKING:
     from lbry.conf import Config
-    from scribe.schema.claim import Claim
     from lbry.blob.blob_manager import BlobManager
     from lbry.blob.blob_info import BlobInfo
-    from lbry.dht.node import Node
     from lbry.extras.daemon.analytics import AnalyticsManager
     from lbry.wallet.transaction import Transaction
 
@@ -62,9 +60,9 @@ class ManagedStream(ManagedDownloadSource):
         self.file_output_task: typing.Optional[asyncio.Task] = None
         self.delayed_stop_task: typing.Optional[asyncio.Task] = None
         self.streaming_responses: typing.List[typing.Tuple[Request, StreamResponse]] = []
-        self.fully_reflected = asyncio.Event(loop=self.loop)
-        self.streaming = asyncio.Event(loop=self.loop)
-        self._running = asyncio.Event(loop=self.loop)
+        self.fully_reflected = asyncio.Event()
+        self.streaming = asyncio.Event()
+        self._running = asyncio.Event()
 
     @property
     def sd_hash(self) -> str:
@@ -84,7 +82,19 @@ class ManagedStream(ManagedDownloadSource):
 
     @property
     def file_name(self) -> Optional[str]:
-        return self._file_name or (self.descriptor.suggested_file_name if self.descriptor else None)
+        return self._file_name or self.suggested_file_name
+
+    @property
+    def suggested_file_name(self) -> Optional[str]:
+        first_option = ((self.descriptor and self.descriptor.suggested_file_name) or '').strip()
+        return sanitize_file_name(first_option or (self.stream_claim_info and self.stream_claim_info.claim and
+                                                   self.stream_claim_info.claim.stream.source.name))
+
+    @property
+    def stream_name(self) -> Optional[str]:
+        first_option = ((self.descriptor and self.descriptor.stream_name) or '').strip()
+        return first_option or (self.stream_claim_info and self.stream_claim_info.claim and
+                                self.stream_claim_info.claim.stream.source.name)
 
     @property
     def written_bytes(self) -> int:
@@ -118,7 +128,7 @@ class ManagedStream(ManagedDownloadSource):
 
     @property
     def mime_type(self):
-        return guess_media_type(os.path.basename(self.descriptor.suggested_file_name))[0]
+        return guess_media_type(os.path.basename(self.suggested_file_name))[0]
 
     @property
     def download_path(self):
@@ -151,7 +161,7 @@ class ManagedStream(ManagedDownloadSource):
         log.info("start downloader for stream (sd hash: %s)", self.sd_hash)
         self._running.set()
         try:
-            await asyncio.wait_for(self.downloader.start(), timeout, loop=self.loop)
+            await asyncio.wait_for(self.downloader.start(), timeout)
         except asyncio.TimeoutError:
             self._running.clear()
             raise DownloadSDTimeoutError(self.sd_hash)
@@ -164,7 +174,7 @@ class ManagedStream(ManagedDownloadSource):
                 if not self._file_name:
                     self._file_name = await get_next_available_file_name(
                         self.loop, self.download_directory,
-                        self._file_name or sanitize_file_name(self.descriptor.suggested_file_name)
+                        self._file_name or sanitize_file_name(self.suggested_file_name)
                     )
                 file_name, download_dir = self._file_name, self.download_directory
             else:
@@ -181,7 +191,7 @@ class ManagedStream(ManagedDownloadSource):
         Stop any running save/stream tasks as well as the downloader and update the status in the database
         """
 
-        self.stop_tasks()
+        await self.stop_tasks()
         if (finished and self.status != self.STATUS_FINISHED) or self.status == self.STATUS_RUNNING:
             await self.update_status(self.STATUS_FINISHED if finished else self.STATUS_STOPPED)
 
@@ -269,7 +279,7 @@ class ManagedStream(ManagedDownloadSource):
             log.info("finished saving file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id,
                      self.sd_hash[:6], self.full_path)
             await self.blob_manager.storage.set_saved_file(self.stream_hash)
-        except Exception as err:
+        except (Exception, asyncio.CancelledError) as err:
             if os.path.isfile(output_path):
                 log.warning("removing incomplete download %s for %s", output_path, self.sd_hash)
                 os.remove(output_path)
@@ -296,14 +306,14 @@ class ManagedStream(ManagedDownloadSource):
         self.download_directory = download_directory or self.download_directory or self.config.download_dir
         if not self.download_directory:
             raise ValueError("no directory to download to")
-        if not (file_name or self._file_name or self.descriptor.suggested_file_name):
+        if not (file_name or self._file_name or self.suggested_file_name):
             raise ValueError("no file name to download to")
         if not os.path.isdir(self.download_directory):
             log.warning("download directory '%s' does not exist, attempting to make it", self.download_directory)
             os.mkdir(self.download_directory)
         self._file_name = await get_next_available_file_name(
             self.loop, self.download_directory,
-            file_name or self._file_name or sanitize_file_name(self.descriptor.suggested_file_name)
+            file_name or self._file_name or sanitize_file_name(self.suggested_file_name)
         )
         await self.blob_manager.storage.change_file_download_dir_and_file_name(
             self.stream_hash, self.download_directory, self.file_name
@@ -311,15 +321,16 @@ class ManagedStream(ManagedDownloadSource):
         await self.update_status(ManagedStream.STATUS_RUNNING)
         self.file_output_task = self.loop.create_task(self._save_file(self.full_path))
         try:
-            await asyncio.wait_for(self.started_writing.wait(), self.config.download_timeout, loop=self.loop)
+            await asyncio.wait_for(self.started_writing.wait(), self.config.download_timeout)
         except asyncio.TimeoutError:
             log.warning("timeout starting to write data for lbry://%s#%s", self.claim_name, self.claim_id)
-            self.stop_tasks()
+            await self.stop_tasks()
             await self.update_status(ManagedStream.STATUS_STOPPED)
 
-    def stop_tasks(self):
+    async def stop_tasks(self):
         if self.file_output_task and not self.file_output_task.done():
             self.file_output_task.cancel()
+            await asyncio.gather(self.file_output_task, return_exceptions=True)
         self.file_output_task = None
         while self.streaming_responses:
             req, response = self.streaming_responses.pop()
@@ -356,7 +367,7 @@ class ManagedStream(ManagedDownloadSource):
             return sent
         except ConnectionError:
             return sent
-        except (OSError, Exception) as err:
+        except (OSError, Exception, asyncio.CancelledError) as err:
             if isinstance(err, asyncio.CancelledError):
                 log.warning("stopped uploading %s#%s to reflector", self.claim_name, self.claim_id)
             elif isinstance(err, OSError):
@@ -391,7 +402,7 @@ class ManagedStream(ManagedDownloadSource):
                          self.sd_hash[:6])
                 await self.stop()
                 return
-            await asyncio.sleep(1, loop=self.loop)
+            await asyncio.sleep(1)
 
     def _prepare_range_response_headers(self, get_range: str) -> typing.Tuple[typing.Dict[str, str], int, int, int]:
         if '=' in get_range:

lbry/stream/reflector/server.py

@@ -21,7 +21,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
         self.loop = asyncio.get_event_loop()
         self.blob_manager = blob_manager
         self.server_task: asyncio.Task = None
-        self.started_listening = asyncio.Event(loop=self.loop)
+        self.started_listening = asyncio.Event()
         self.buf = b''
         self.transport: asyncio.StreamWriter = None
         self.writer: typing.Optional['HashBlobWriter'] = None
@@ -29,9 +29,9 @@ class ReflectorServerProtocol(asyncio.Protocol):
         self.descriptor: typing.Optional['StreamDescriptor'] = None
         self.sd_blob: typing.Optional['BlobFile'] = None
         self.received = []
-        self.incoming = incoming_event or asyncio.Event(loop=self.loop)
-        self.not_incoming = not_incoming_event or asyncio.Event(loop=self.loop)
-        self.stop_event = stop_event or asyncio.Event(loop=self.loop)
+        self.incoming = incoming_event or asyncio.Event()
+        self.not_incoming = not_incoming_event or asyncio.Event()
+        self.stop_event = stop_event or asyncio.Event()
         self.chunk_size = response_chunk_size
         self.wait_for_stop_task: typing.Optional[asyncio.Task] = None
         self.partial_event = partial_event
@@ -94,7 +94,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
                 self.incoming.set()
                 self.send_response({"send_sd_blob": True})
                 try:
-                    await asyncio.wait_for(self.sd_blob.verified.wait(), 30, loop=self.loop)
+                    await asyncio.wait_for(self.sd_blob.verified.wait(), 30)
                     self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
                         self.loop, self.blob_manager.blob_dir, self.sd_blob
                     )
@@ -140,7 +140,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
                 self.incoming.set()
                 self.send_response({"send_blob": True})
                 try:
-                    await asyncio.wait_for(blob.verified.wait(), 30, loop=self.loop)
+                    await asyncio.wait_for(blob.verified.wait(), 30)
                     self.send_response({"received_blob": True})
                 except asyncio.TimeoutError:
                     self.send_response({"received_blob": False})
@@ -162,10 +162,10 @@ class ReflectorServer:
         self.loop = asyncio.get_event_loop()
         self.blob_manager = blob_manager
         self.server_task: typing.Optional[asyncio.Task] = None
-        self.started_listening = asyncio.Event(loop=self.loop)
-        self.stopped_listening = asyncio.Event(loop=self.loop)
-        self.incoming_event = incoming_event or asyncio.Event(loop=self.loop)
-        self.not_incoming_event = not_incoming_event or asyncio.Event(loop=self.loop)
+        self.started_listening = asyncio.Event()
+        self.stopped_listening = asyncio.Event()
+        self.incoming_event = incoming_event or asyncio.Event()
+        self.not_incoming_event = not_incoming_event or asyncio.Event()
         self.response_chunk_size = response_chunk_size
         self.stop_event = stop_event
         self.partial_needs = partial_needs  # for testing cases where it doesn't know what it wants

lbry/stream/stream_manager.py

@@ -54,7 +54,7 @@ class StreamManager(SourceManager):
         self.re_reflect_task: Optional[asyncio.Task] = None
         self.update_stream_finished_futs: typing.List[asyncio.Future] = []
         self.running_reflector_uploads: typing.Dict[str, asyncio.Task] = {}
-        self.started = asyncio.Event(loop=self.loop)
+        self.started = asyncio.Event()
 
     @property
     def streams(self):
@@ -70,6 +70,7 @@ class StreamManager(SourceManager):
 
     async def recover_streams(self, file_infos: typing.List[typing.Dict]):
         to_restore = []
+        to_check = []
 
         async def recover_stream(sd_hash: str, stream_hash: str, stream_name: str,
                                  suggested_file_name: str, key: str,
@@ -82,6 +83,7 @@ class StreamManager(SourceManager):
             if not descriptor:
                 return
             to_restore.append((descriptor, sd_blob, content_fee))
+            to_check.extend([sd_blob.blob_hash] + [blob.blob_hash for blob in descriptor.blobs[:-1]])
 
         await asyncio.gather(*[
             recover_stream(
@@ -93,6 +95,8 @@ class StreamManager(SourceManager):
 
         if to_restore:
             await self.storage.recover_streams(to_restore, self.config.download_dir)
+        if to_check:
+            await self.blob_manager.ensure_completed_blobs_status(to_check)
 
         # if self.blob_manager._save_blobs:
         #     log.info("Recovered %i/%i attempted streams", len(to_restore), len(file_infos))
@@ -146,7 +150,7 @@ class StreamManager(SourceManager):
                 file_info['added_on'], file_info['fully_reflected']
             )))
         if add_stream_tasks:
-            await asyncio.gather(*add_stream_tasks, loop=self.loop)
+            await asyncio.gather(*add_stream_tasks)
         log.info("Started stream manager with %i files", len(self._sources))
         if not self.node:
             log.info("no DHT node given, resuming downloads trusting that we can contact reflector")
@@ -155,14 +159,11 @@ class StreamManager(SourceManager):
             self.resume_saving_task = asyncio.ensure_future(asyncio.gather(
                 *(self._sources[sd_hash].save_file(file_name, download_directory)
                   for (file_name, download_directory, sd_hash) in to_resume_saving),
-                loop=self.loop
             ))
 
     async def reflect_streams(self):
         try:
             return await self._reflect_streams()
-        except asyncio.CancelledError:
-            raise
         except Exception:
             log.exception("reflector task encountered an unexpected error!")
 
@@ -182,21 +183,21 @@ class StreamManager(SourceManager):
                         batch.append(self.reflect_stream(stream))
                     if len(batch) >= self.config.concurrent_reflector_uploads:
                         log.debug("waiting for batch of %s reflecting streams", len(batch))
-                        await asyncio.gather(*batch, loop=self.loop)
+                        await asyncio.gather(*batch)
                         log.debug("done processing %s streams", len(batch))
                         batch = []
                 if batch:
                     log.debug("waiting for batch of %s reflecting streams", len(batch))
-                    await asyncio.gather(*batch, loop=self.loop)
+                    await asyncio.gather(*batch)
                     log.debug("done processing %s streams", len(batch))
-            await asyncio.sleep(300, loop=self.loop)
+            await asyncio.sleep(300)
 
     async def start(self):
         await super().start()
         self.re_reflect_task = self.loop.create_task(self.reflect_streams())
 
-    def stop(self):
-        super().stop()
+    async def stop(self):
+        await super().stop()
         if self.resume_saving_task and not self.resume_saving_task.done():
             self.resume_saving_task.cancel()
         if self.re_reflect_task and not self.re_reflect_task.done():
@@ -223,7 +224,8 @@ class StreamManager(SourceManager):
         )
         return task
 
-    async def _retriable_reflect_stream(self, stream, host, port):
+    @staticmethod
+    async def _retriable_reflect_stream(stream, host, port):
         sent = await stream.upload_to_reflector(host, port)
         while not stream.is_fully_reflected and stream.reflector_progress > 0 and len(sent) > 0:
             stream.reflector_progress = 0
@@ -258,7 +260,7 @@ class StreamManager(SourceManager):
             return
         if source.identifier in self.running_reflector_uploads:
             self.running_reflector_uploads[source.identifier].cancel()
-        source.stop_tasks()
+        await source.stop_tasks()
         if source.identifier in self.streams:
             del self.streams[source.identifier]
         blob_hashes = [source.identifier] + [b.blob_hash for b in source.descriptor.blobs[:-1]]

lbry/testcase.py

@@ -19,8 +19,8 @@ from lbry.conf import Config
 from lbry.wallet.util import satoshis_to_coins
 from lbry.wallet.dewies import lbc_to_dewies
 from lbry.wallet.orchstr8 import Conductor
-from lbry.wallet.orchstr8.node import LBCWalletNode, WalletNode, HubNode
-from scribe.schema.claim import Claim
+from lbry.wallet.orchstr8.node import LBCWalletNode, WalletNode
+from lbry.schema.claim import Claim
 
 from lbry.extras.daemon.daemon import Daemon, jsonrpc_dumps_pretty
 from lbry.extras.daemon.components import Component, WalletComponent
@@ -204,7 +204,13 @@ class AsyncioTestCase(unittest.TestCase):
 
     def add_timeout(self):
         if self.TIMEOUT:
-            self.loop.call_later(self.TIMEOUT, self.cancel)
+            self.loop.call_later(self.TIMEOUT, self.check_timeout, time())
+
+    def check_timeout(self, started):
+        if time() - started >= self.TIMEOUT:
+            self.cancel()
+        else:
+            self.loop.call_later(self.TIMEOUT, self.check_timeout, started)
 
 
 class AdvanceTimeTestCase(AsyncioTestCase):
@@ -231,7 +237,6 @@ class IntegrationTestCase(AsyncioTestCase):
         super().__init__(*args, **kwargs)
         self.conductor: Optional[Conductor] = None
         self.blockchain: Optional[LBCWalletNode] = None
-        self.hub: Optional[HubNode] = None
         self.wallet_node: Optional[WalletNode] = None
         self.manager: Optional[WalletManager] = None
         self.ledger: Optional[Ledger] = None
@@ -248,10 +253,7 @@ class IntegrationTestCase(AsyncioTestCase):
         self.addCleanup(self.conductor.stop_spv)
         await self.conductor.start_wallet()
         self.addCleanup(self.conductor.stop_wallet)
-        await self.conductor.start_hub()
-        self.addCleanup(self.conductor.stop_hub)
         self.blockchain = self.conductor.lbcwallet_node
-        self.hub = self.conductor.hub_node
         self.wallet_node = self.conductor.wallet_node
         self.manager = self.wallet_node.manager
         self.ledger = self.wallet_node.ledger
@@ -299,15 +301,8 @@ class IntegrationTestCase(AsyncioTestCase):
             watcher = (ledger or self.ledger).on_transaction.where(
                 lambda e: ((e.tx.id in txids and txids.remove(e.tx.id)), len(txids) <= 0)[-1]  # multi-statement lambda
             )
-            self.conductor.spv_node.server.synchronized.clear()
-            await self.blockchain.generate(blocks_to_generate)
-            height = self.blockchain.block_expected
+            await self.generate(blocks_to_generate)
             await watcher
-            while True:
-                await self.conductor.spv_node.server.synchronized.wait()
-                self.conductor.spv_node.server.synchronized.clear()
-                if self.conductor.spv_node.server.db.db_height >= height:
-                    break
 
     def on_address_update(self, address):
         return self.ledger.on_transaction.where(
@@ -322,15 +317,18 @@ class IntegrationTestCase(AsyncioTestCase):
     async def generate(self, blocks):
         """ Ask lbrycrd to generate some blocks and wait until ledger has them. """
         prepare = self.ledger.on_header.where(self.blockchain.is_expected_block)
-        height = self.blockchain.block_expected
         self.conductor.spv_node.server.synchronized.clear()
         await self.blockchain.generate(blocks)
+        height = self.blockchain.block_expected
         await prepare  # no guarantee that it didn't happen already, so start waiting from before calling generate
         while True:
             await self.conductor.spv_node.server.synchronized.wait()
             self.conductor.spv_node.server.synchronized.clear()
-            if self.conductor.spv_node.server.db.db_height >= height:
-                break
+            if self.conductor.spv_node.server.db.db_height < height:
+                continue
+            if self.conductor.spv_node.server._es_height < height:
+                continue
+            break
 
 
 class FakeExchangeRateManager(ExchangeRateManager):

lbry/torrent/session.py

@@ -10,47 +10,13 @@ from typing import Optional
 import libtorrent
 
 
-NOTIFICATION_MASKS = [
-    "error",
-    "peer",
-    "port_mapping",
-    "storage",
-    "tracker",
-    "debug",
-    "status",
-    "progress",
-    "ip_block",
-    "dht",
-    "stats",
-    "session_log",
-    "torrent_log",
-    "peer_log",
-    "incoming_request",
-    "dht_log",
-    "dht_operation",
-    "port_mapping_log",
-    "picker_log",
-    "file_progress",
-    "piece_progress",
-    "upload",
-    "block_progress"
-]
 log = logging.getLogger(__name__)
-
-
 DEFAULT_FLAGS = (  # fixme: somehow the logic here is inverted?
         libtorrent.add_torrent_params_flags_t.flag_auto_managed
         | libtorrent.add_torrent_params_flags_t.flag_update_subscribe
 )
 
 
-def get_notification_type(notification) -> str:
-    for i, notification_type in enumerate(NOTIFICATION_MASKS):
-        if (1 << i) & notification:
-            return notification_type
-    raise ValueError("unrecognized notification type")
-
-
 class TorrentHandle:
     def __init__(self, loop, executor, handle):
         self._loop = loop
@@ -121,7 +87,7 @@ class TorrentHandle:
             self._show_status()
             if self.finished.is_set():
                 break
-            await asyncio.sleep(0.1, loop=self._loop)
+            await asyncio.sleep(0.1)
 
     async def pause(self):
         await self._loop.run_in_executor(
@@ -156,10 +122,8 @@ class TorrentSession:
     async def bind(self, interface: str = '0.0.0.0', port: int = 10889):
         settings = {
             'listen_interfaces': f"{interface}:{port}",
-            'enable_outgoing_utp': True,
-            'enable_incoming_utp': True,
-            'enable_outgoing_tcp': False,
-            'enable_incoming_tcp': False
+            'enable_natpmp': False,
+            'enable_upnp': False
         }
         self._session = await self._loop.run_in_executor(
             self._executor, libtorrent.session, settings  # pylint: disable=c-extension-no-member
@@ -186,7 +150,7 @@ class TorrentSession:
             await self._loop.run_in_executor(
                 self._executor, self._pop_alerts
             )
-            await asyncio.sleep(1, loop=self._loop)
+            await asyncio.sleep(1)
 
     async def pause(self):
         await self._loop.run_in_executor(

lbry/torrent/torrent.py

@@ -36,7 +36,7 @@ class Torrent:
     def __init__(self, loop, handle):
         self._loop = loop
         self._handle = handle
-        self.finished = asyncio.Event(loop=loop)
+        self.finished = asyncio.Event()
 
     def _threaded_update_status(self):
         status = self._handle.status()
@@ -58,7 +58,7 @@ class Torrent:
                 log.info("finished downloading torrent!")
                 await self.pause()
                 break
-            await asyncio.sleep(1, loop=self._loop)
+            await asyncio.sleep(1)
 
     async def pause(self):
         log.info("pause torrent")

lbry/torrent/torrent_manager.py

@@ -74,7 +74,7 @@ class TorrentSource(ManagedDownloadSource):
     def bt_infohash(self):
         return self.identifier
 
-    def stop_tasks(self):
+    async def stop_tasks(self):
         pass
 
     @property
@@ -118,8 +118,8 @@ class TorrentManager(SourceManager):
     async def start(self):
         await super().start()
 
-    def stop(self):
-        super().stop()
+    async def stop(self):
+        await super().stop()
         log.info("finished stopping the torrent manager")
 
     async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):

lbry/torrent/tracker.py

@@ -0,0 +1,285 @@
+import random
+import socket
+import string
+import struct
+import asyncio
+import logging
+import time
+import ipaddress
+from collections import namedtuple
+from functools import reduce
+from typing import Optional
+
+from lbry.dht.node import get_kademlia_peers_from_hosts
+from lbry.utils import resolve_host, async_timed_cache, cache_concurrent
+from lbry.wallet.stream import StreamController
+from lbry import version
+
+log = logging.getLogger(__name__)
+CONNECTION_EXPIRES_AFTER_SECONDS = 50
+PREFIX = 'LB'  # todo: PR BEP20 to add ourselves
+DEFAULT_TIMEOUT_SECONDS = 10.0
+DEFAULT_CONCURRENCY_LIMIT = 100
+# see: http://bittorrent.org/beps/bep_0015.html and http://xbtt.sourceforge.net/udp_tracker_protocol.html
+ConnectRequest = namedtuple("ConnectRequest", ["connection_id", "action", "transaction_id"])
+ConnectResponse = namedtuple("ConnectResponse", ["action", "transaction_id", "connection_id"])
+AnnounceRequest = namedtuple("AnnounceRequest",
+                             ["connection_id", "action", "transaction_id", "info_hash", "peer_id", "downloaded", "left",
+                              "uploaded", "event", "ip_addr", "key", "num_want", "port"])
+AnnounceResponse = namedtuple("AnnounceResponse",
+                              ["action", "transaction_id", "interval", "leechers", "seeders", "peers"])
+CompactIPv4Peer = namedtuple("CompactPeer", ["address", "port"])
+ScrapeRequest = namedtuple("ScrapeRequest", ["connection_id", "action", "transaction_id", "infohashes"])
+ScrapeResponse = namedtuple("ScrapeResponse", ["action", "transaction_id", "items"])
+ScrapeResponseItem = namedtuple("ScrapeResponseItem", ["seeders", "completed", "leechers"])
+ErrorResponse = namedtuple("ErrorResponse", ["action", "transaction_id", "message"])
+structs = {
+    ConnectRequest: struct.Struct(">QII"),
+    ConnectResponse: struct.Struct(">IIQ"),
+    AnnounceRequest: struct.Struct(">QII20s20sQQQIIIiH"),
+    AnnounceResponse: struct.Struct(">IIIII"),
+    CompactIPv4Peer: struct.Struct(">IH"),
+    ScrapeRequest: struct.Struct(">QII"),
+    ScrapeResponse: struct.Struct(">II"),
+    ScrapeResponseItem: struct.Struct(">III"),
+    ErrorResponse: struct.Struct(">II")
+}
+
+
+def decode(cls, data, offset=0):
+    decoder = structs[cls]
+    if cls is AnnounceResponse:
+        return AnnounceResponse(*decoder.unpack_from(data, offset),
+                                peers=[decode(CompactIPv4Peer, data, index) for index in range(20, len(data), 6)])
+    elif cls is ScrapeResponse:
+        return ScrapeResponse(*decoder.unpack_from(data, offset),
+                              items=[decode(ScrapeResponseItem, data, index) for index in range(8, len(data), 12)])
+    elif cls is ErrorResponse:
+        return ErrorResponse(*decoder.unpack_from(data, offset), data[decoder.size:])
+    return cls(*decoder.unpack_from(data, offset))
+
+
+def encode(obj):
+    if isinstance(obj, ScrapeRequest):
+        return structs[ScrapeRequest].pack(*obj[:-1]) + b''.join(obj.infohashes)
+    elif isinstance(obj, ErrorResponse):
+        return structs[ErrorResponse].pack(*obj[:-1]) + obj.message
+    elif isinstance(obj, AnnounceResponse):
+        return structs[AnnounceResponse].pack(*obj[:-1]) + b''.join([encode(peer) for peer in obj.peers])
+    return structs[type(obj)].pack(*obj)
+
+
+def make_peer_id(random_part: Optional[str] = None) -> bytes:
+    # see https://wiki.theory.org/BitTorrentSpecification#peer_id and https://www.bittorrent.org/beps/bep_0020.html
+    # not to confuse with node id; peer id identifies uniquely the software, version and instance
+    random_part = random_part or ''.join(random.choice(string.ascii_letters) for _ in range(20))
+    return f"{PREFIX}-{'-'.join(map(str, version))}-{random_part}"[:20].encode()
+
+
+class UDPTrackerClientProtocol(asyncio.DatagramProtocol):
+    def __init__(self, timeout: float = DEFAULT_TIMEOUT_SECONDS):
+        self.transport = None
+        self.data_queue = {}
+        self.timeout = timeout
+        self.semaphore = asyncio.Semaphore(DEFAULT_CONCURRENCY_LIMIT)
+
+    def connection_made(self, transport: asyncio.DatagramTransport) -> None:
+        self.transport = transport
+
+    async def request(self, obj, tracker_ip, tracker_port):
+        self.data_queue[obj.transaction_id] = asyncio.get_running_loop().create_future()
+        try:
+            async with self.semaphore:
+                self.transport.sendto(encode(obj), (tracker_ip, tracker_port))
+                return await asyncio.wait_for(self.data_queue[obj.transaction_id], self.timeout)
+        finally:
+            self.data_queue.pop(obj.transaction_id, None)
+
+    async def connect(self, tracker_ip, tracker_port):
+        transaction_id = random.getrandbits(32)
+        return decode(ConnectResponse,
+                      await self.request(ConnectRequest(0x41727101980, 0, transaction_id), tracker_ip, tracker_port))
+
+    @cache_concurrent
+    @async_timed_cache(CONNECTION_EXPIRES_AFTER_SECONDS)
+    async def ensure_connection_id(self, peer_id, tracker_ip, tracker_port):
+        # peer_id is just to ensure cache coherency
+        return (await self.connect(tracker_ip, tracker_port)).connection_id
+
+    async def announce(self, info_hash, peer_id, port, tracker_ip, tracker_port, stopped=False):
+        connection_id = await self.ensure_connection_id(peer_id, tracker_ip, tracker_port)
+        # this should make the key deterministic but unique per info hash + peer id
+        key = int.from_bytes(info_hash[:4], "big") ^ int.from_bytes(peer_id[:4], "big") ^ port
+        transaction_id = random.getrandbits(32)
+        req = AnnounceRequest(
+            connection_id, 1, transaction_id, info_hash, peer_id, 0, 0, 0, 3 if stopped else 1, 0, key, -1, port)
+        return decode(AnnounceResponse, await self.request(req, tracker_ip, tracker_port))
+
+    async def scrape(self, infohashes, tracker_ip, tracker_port, connection_id=None):
+        connection_id = await self.ensure_connection_id(None, tracker_ip, tracker_port)
+        transaction_id = random.getrandbits(32)
+        reply = await self.request(
+            ScrapeRequest(connection_id, 2, transaction_id, infohashes), tracker_ip, tracker_port)
+        return decode(ScrapeResponse, reply), connection_id
+
+    def datagram_received(self, data: bytes, addr: (str, int)) -> None:
+        if len(data) < 8:
+            return
+        transaction_id = int.from_bytes(data[4:8], byteorder="big", signed=False)
+        if transaction_id in self.data_queue:
+            if not self.data_queue[transaction_id].done():
+                if data[3] == 3:
+                    return self.data_queue[transaction_id].set_exception(Exception(decode(ErrorResponse, data).message))
+                return self.data_queue[transaction_id].set_result(data)
+        log.debug("unexpected packet (can be a response for a previously timed out request): %s", data.hex())
+
+    def connection_lost(self, exc: Exception = None) -> None:
+        self.transport = None
+
+
+class TrackerClient:
+    event_controller = StreamController()
+
+    def __init__(self, node_id, announce_port, get_servers, timeout=10.0):
+        self.client = UDPTrackerClientProtocol(timeout=timeout)
+        self.transport = None
+        self.peer_id = make_peer_id(node_id.hex() if node_id else None)
+        self.announce_port = announce_port
+        self._get_servers = get_servers
+        self.results = {}  # we can't probe the server before the interval, so we keep the result here until it expires
+        self.tasks = {}
+
+    async def start(self):
+        self.transport, _ = await asyncio.get_running_loop().create_datagram_endpoint(
+            lambda: self.client, local_addr=("0.0.0.0", 0))
+        self.event_controller.stream.listen(
+            lambda request: self.on_hash(request[1], request[2]) if request[0] == 'search' else None)
+
+    def stop(self):
+        while self.tasks:
+            self.tasks.popitem()[1].cancel()
+        if self.transport is not None:
+            self.transport.close()
+        self.client = None
+        self.transport = None
+        self.event_controller.close()
+
+    def on_hash(self, info_hash, on_announcement=None):
+        if info_hash not in self.tasks:
+            task = asyncio.create_task(self.get_peer_list(info_hash, on_announcement=on_announcement))
+            task.add_done_callback(lambda *_: self.tasks.pop(info_hash, None))
+            self.tasks[info_hash] = task
+
+    async def announce_many(self, *info_hashes, stopped=False):
+        await asyncio.gather(
+            *[self._announce_many(server, info_hashes, stopped=stopped) for server in self._get_servers()],
+            return_exceptions=True)
+
+    async def _announce_many(self, server, info_hashes, stopped=False):
+        tracker_ip = await resolve_host(*server, 'udp')
+        still_good_info_hashes = {
+            info_hash for (info_hash, (next_announcement, _)) in self.results.get(tracker_ip, {}).items()
+            if time.time() < next_announcement
+        }
+        results = await asyncio.gather(
+            *[self._probe_server(info_hash, tracker_ip, server[1], stopped=stopped)
+              for info_hash in info_hashes if info_hash not in still_good_info_hashes],
+            return_exceptions=True)
+        if results:
+            errors = sum([1 for result in results if result is None or isinstance(result, Exception)])
+            log.info("Tracker: finished announcing %d files to %s:%d, %d errors", len(results), *server, errors)
+
+    async def get_peer_list(self, info_hash, stopped=False, on_announcement=None, no_port=False):
+        found = []
+        probes = [self._probe_server(info_hash, *server, stopped, no_port) for server in self._get_servers()]
+        for done in asyncio.as_completed(probes):
+            result = await done
+            if result is not None:
+                await asyncio.gather(*filter(asyncio.iscoroutine, [on_announcement(result)] if on_announcement else []))
+                found.append(result)
+        return found
+
+    async def get_kademlia_peer_list(self, info_hash):
+        responses = await self.get_peer_list(info_hash, no_port=True)
+        return await announcement_to_kademlia_peers(*responses)
+
+    async def _probe_server(self, info_hash, tracker_host, tracker_port, stopped=False, no_port=False):
+        result = None
+        try:
+            tracker_host = await resolve_host(tracker_host, tracker_port, 'udp')
+        except socket.error:
+            log.warning("DNS failure while resolving tracker host: %s, skipping.", tracker_host)
+            return
+        self.results.setdefault(tracker_host, {})
+        if info_hash in self.results[tracker_host]:
+            next_announcement, result = self.results[tracker_host][info_hash]
+            if time.time() < next_announcement:
+                return result
+        try:
+            result = await self.client.announce(
+                info_hash, self.peer_id, 0 if no_port else self.announce_port, tracker_host, tracker_port, stopped)
+            self.results[tracker_host][info_hash] = (time.time() + result.interval, result)
+        except asyncio.TimeoutError:  # todo: this is UDP, timeout is common, we need a better metric for failures
+            self.results[tracker_host][info_hash] = (time.time() + 60.0, result)
+            log.debug("Tracker timed out: %s:%d", tracker_host, tracker_port)
+            return None
+        log.debug("Announced: %s found %d peers for %s", tracker_host, len(result.peers), info_hash.hex()[:8])
+        return result
+
+
+def enqueue_tracker_search(info_hash: bytes, peer_q: asyncio.Queue):
+    async def on_announcement(announcement: AnnounceResponse):
+        peers = await announcement_to_kademlia_peers(announcement)
+        log.info("Found %d peers from tracker for %s", len(peers), info_hash.hex()[:8])
+        peer_q.put_nowait(peers)
+    TrackerClient.event_controller.add(('search', info_hash, on_announcement))
+
+
+def announcement_to_kademlia_peers(*announcements: AnnounceResponse):
+    peers = [
+        (str(ipaddress.ip_address(peer.address)), peer.port)
+        for announcement in announcements for peer in announcement.peers if peer.port > 1024  # no privileged or 0
+    ]
+    return get_kademlia_peers_from_hosts(peers)
+
+
+class UDPTrackerServerProtocol(asyncio.DatagramProtocol):  # for testing. Not suitable for production
+    def __init__(self):
+        self.transport = None
+        self.known_conns = set()
+        self.peers = {}
+
+    def connection_made(self, transport: asyncio.DatagramTransport) -> None:
+        self.transport = transport
+
+    def add_peer(self, info_hash, ip_address: str, port: int):
+        self.peers.setdefault(info_hash, [])
+        self.peers[info_hash].append(encode_peer(ip_address, port))
+
+    def datagram_received(self, data: bytes, addr: (str, int)) -> None:
+        if len(data) < 16:
+            return
+        action = int.from_bytes(data[8:12], "big", signed=False)
+        if action == 0:
+            req = decode(ConnectRequest, data)
+            connection_id = random.getrandbits(32)
+            self.known_conns.add(connection_id)
+            return self.transport.sendto(encode(ConnectResponse(0, req.transaction_id, connection_id)), addr)
+        elif action == 1:
+            req = decode(AnnounceRequest, data)
+            if req.connection_id not in self.known_conns:
+                resp = encode(ErrorResponse(3, req.transaction_id, b'Connection ID missmatch.\x00'))
+            else:
+                compact_address = encode_peer(addr[0], req.port)
+                if req.event != 3:
+                    self.add_peer(req.info_hash, addr[0], req.port)
+                elif compact_address in self.peers.get(req.info_hash, []):
+                    self.peers[req.info_hash].remove(compact_address)
+                peers = [decode(CompactIPv4Peer, peer) for peer in self.peers[req.info_hash]]
+                resp = encode(AnnounceResponse(1, req.transaction_id, 1700, 0, len(peers), peers))
+            return self.transport.sendto(resp, addr)
+
+
+def encode_peer(ip_address: str, port: int):
+    compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), ip_address.split('.'), bytearray())
+    return compact_ip + port.to_bytes(2, "big", signed=False)

lbry/utils.py

@@ -21,7 +21,7 @@ import pkg_resources
 import certifi
 import aiohttp
 from prometheus_client import Counter
-from scribe.schema.claim import Claim
+from lbry.schema.claim import Claim
 
 
 log = logging.getLogger(__name__)
@@ -130,21 +130,16 @@ def get_sd_hash(stream_info):
 def json_dumps_pretty(obj, **kwargs):
     return json.dumps(obj, sort_keys=True, indent=2, separators=(',', ': '), **kwargs)
 
-
-def cancel_task(task: typing.Optional[asyncio.Task]):
-    if task and not task.done():
-        task.cancel()
-
-
-def cancel_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
-    for task in tasks:
-        cancel_task(task)
-
-
-def drain_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
-    while tasks:
-        cancel_task(tasks.pop())
-
+try:
+    # the standard contextlib.aclosing() is available in 3.10+
+    from contextlib import aclosing  # pylint: disable=unused-import
+except ImportError:
+    @contextlib.asynccontextmanager
+    async def aclosing(thing):
+        try:
+            yield thing
+        finally:
+            await thing.aclose()
 
 def async_timed_cache(duration: int):
     def wrapper(func):
@@ -455,8 +450,8 @@ def is_running_from_bundle():
 
 
 class LockWithMetrics(asyncio.Lock):
-    def __init__(self, acquire_metric, held_time_metric, loop=None):
-        super().__init__(loop=loop)
+    def __init__(self, acquire_metric, held_time_metric):
+        super().__init__()
         self._acquire_metric = acquire_metric
         self._lock_held_time_metric = held_time_metric
         self._lock_acquired_time = None
@@ -474,3 +469,18 @@ class LockWithMetrics(asyncio.Lock):
             return super().release()
         finally:
             self._lock_held_time_metric.observe(time.perf_counter() - self._lock_acquired_time)
+
+
+def get_colliding_prefix_bits(first_value: bytes, second_value: bytes):
+    """
+    Calculates the amount of colliding prefix bits between <first_value> and <second_value>.
+    This is given by the amount of bits that are the same until the first different one (via XOR),
+    starting from the most significant bit to the least significant bit.
+    :param first_value: first value to compare, bigger than size.
+    :param second_value: second value to compare, bigger than size.
+    :return: amount of prefix colliding bits.
+    """
+    assert len(first_value) == len(second_value), "length should be the same"
+    size = len(first_value) * 8
+    first_value, second_value = int.from_bytes(first_value, "big"), int.from_bytes(second_value, "big")
+    return size - (first_value ^ second_value).bit_length()

lbry/wallet/__init__.py

@@ -3,11 +3,11 @@ __lbcctl__ = 'lbcctl'
 __lbcwallet__ = 'lbcwallet'
 __lbcd_url__ = (
     'https://github.com/lbryio/lbcd/releases/download/' +
-    'v0.22.200-beta/lbcd_0.22.200-beta_TARGET_PLATFORM.tar.gz'
+    'v0.22.100-rc.0/lbcd_0.22.100-rc.0_TARGET_PLATFORM.tar.gz'
 )
 __lbcwallet_url__ = (
     'https://github.com/lbryio/lbcwallet/releases/download/' +
-    'v0.13.100-alpha-rc2/lbcwallet_0.13.100-alpha-rc2_TARGET_PLATFORM.tar.gz'
+    'v0.13.100-alpha.0/lbcwallet_0.13.100-alpha.0_TARGET_PLATFORM.tar.gz'
 )
 __spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
 

lbry/wallet/account.py

@@ -12,7 +12,7 @@ from typing import Type, Dict, Tuple, Optional, Any, List
 from lbry.error import InvalidPasswordError
 from lbry.crypto.crypt import aes_encrypt, aes_decrypt
 
-from scribe.schema.bip32 import PrivateKey, PublicKey, KeyPath, from_extended_key_string
+from .bip32 import PrivateKey, PublicKey, KeyPath, from_extended_key_string
 from .mnemonic import Mnemonic
 from .constants import COIN, TXO_TYPES
 from .transaction import Transaction, Input, Output

lbry/wallet/bip32.py

@@ -0,0 +1,342 @@
+from asn1crypto.keys import PrivateKeyInfo, ECPrivateKey
+from coincurve import PublicKey as cPublicKey, PrivateKey as cPrivateKey
+from coincurve.utils import (
+    pem_to_der, lib as libsecp256k1, ffi as libsecp256k1_ffi
+)
+from coincurve.ecdsa import CDATA_SIG_LENGTH
+
+from lbry.crypto.hash import hmac_sha512, hash160, double_sha256
+from lbry.crypto.base58 import Base58
+from .util import cachedproperty
+
+
+class KeyPath:
+    RECEIVE = 0
+    CHANGE = 1
+    CHANNEL = 2
+
+
+class DerivationError(Exception):
+    """ Raised when an invalid derivation occurs. """
+
+
+class _KeyBase:
+    """ A BIP32 Key, public or private. """
+
+    def __init__(self, ledger, chain_code, n, depth, parent):
+        if not isinstance(chain_code, (bytes, bytearray)):
+            raise TypeError('chain code must be raw bytes')
+        if len(chain_code) != 32:
+            raise ValueError('invalid chain code')
+        if not 0 <= n < 1 << 32:
+            raise ValueError('invalid child number')
+        if not 0 <= depth < 256:
+            raise ValueError('invalid depth')
+        if parent is not None:
+            if not isinstance(parent, type(self)):
+                raise TypeError('parent key has bad type')
+        self.ledger = ledger
+        self.chain_code = chain_code
+        self.n = n
+        self.depth = depth
+        self.parent = parent
+
+    def _hmac_sha512(self, msg):
+        """ Use SHA-512 to provide an HMAC, returned as a pair of 32-byte objects. """
+        hmac = hmac_sha512(self.chain_code, msg)
+        return hmac[:32], hmac[32:]
+
+    def _extended_key(self, ver_bytes, raw_serkey):
+        """ Return the 78-byte extended key given prefix version bytes and serialized key bytes. """
+        if not isinstance(ver_bytes, (bytes, bytearray)):
+            raise TypeError('ver_bytes must be raw bytes')
+        if len(ver_bytes) != 4:
+            raise ValueError('ver_bytes must have length 4')
+        if not isinstance(raw_serkey, (bytes, bytearray)):
+            raise TypeError('raw_serkey must be raw bytes')
+        if len(raw_serkey) != 33:
+            raise ValueError('raw_serkey must have length 33')
+
+        return (
+            ver_bytes + bytes((self.depth,))
+            + self.parent_fingerprint() + self.n.to_bytes(4, 'big')
+            + self.chain_code + raw_serkey
+        )
+
+    def identifier(self):
+        raise NotImplementedError
+
+    def extended_key(self):
+        raise NotImplementedError
+
+    def fingerprint(self):
+        """ Return the key's fingerprint as 4 bytes. """
+        return self.identifier()[:4]
+
+    def parent_fingerprint(self):
+        """ Return the parent key's fingerprint as 4 bytes. """
+        return self.parent.fingerprint() if self.parent else bytes((0,)*4)
+
+    def extended_key_string(self):
+        """ Return an extended key as a base58 string. """
+        return Base58.encode_check(self.extended_key())
+
+
+class PublicKey(_KeyBase):
+    """ A BIP32 public key. """
+
+    def __init__(self, ledger, pubkey, chain_code, n, depth, parent=None):
+        super().__init__(ledger, chain_code, n, depth, parent)
+        if isinstance(pubkey, cPublicKey):
+            self.verifying_key = pubkey
+        else:
+            self.verifying_key = self._verifying_key_from_pubkey(pubkey)
+
+    @classmethod
+    def from_compressed(cls, public_key_bytes, ledger=None) -> 'PublicKey':
+        return cls(ledger, public_key_bytes, bytes((0,)*32), 0, 0)
+
+    @classmethod
+    def _verifying_key_from_pubkey(cls, pubkey):
+        """ Converts a 33-byte compressed pubkey into an coincurve.PublicKey object. """
+        if not isinstance(pubkey, (bytes, bytearray)):
+            raise TypeError('pubkey must be raw bytes')
+        if len(pubkey) != 33:
+            raise ValueError('pubkey must be 33 bytes')
+        if pubkey[0] not in (2, 3):
+            raise ValueError('invalid pubkey prefix byte')
+        return cPublicKey(pubkey)
+
+    @cachedproperty
+    def pubkey_bytes(self):
+        """ Return the compressed public key as 33 bytes. """
+        return self.verifying_key.format(True)
+
+    @cachedproperty
+    def address(self):
+        """ The public key as a P2PKH address. """
+        return self.ledger.public_key_to_address(self.pubkey_bytes)
+
+    def ec_point(self):
+        return self.verifying_key.point()
+
+    def child(self, n: int) -> 'PublicKey':
+        """ Return the derived child extended pubkey at index N. """
+        if not 0 <= n < (1 << 31):
+            raise ValueError('invalid BIP32 public key child number')
+
+        msg = self.pubkey_bytes + n.to_bytes(4, 'big')
+        L_b, R_b = self._hmac_sha512(msg)  # pylint: disable=invalid-name
+        derived_key = self.verifying_key.add(L_b)
+        return PublicKey(self.ledger, derived_key, R_b, n, self.depth + 1, self)
+
+    def identifier(self):
+        """ Return the key's identifier as 20 bytes. """
+        return hash160(self.pubkey_bytes)
+
+    def extended_key(self):
+        """ Return a raw extended public key. """
+        return self._extended_key(
+            self.ledger.extended_public_key_prefix,
+            self.pubkey_bytes
+        )
+
+    def verify(self, signature, digest) -> bool:
+        """ Verify that a signature is valid for a 32 byte digest. """
+
+        if len(signature) != 64:
+            raise ValueError('Signature must be 64 bytes long.')
+
+        if len(digest) != 32:
+            raise ValueError('Digest must be 32 bytes long.')
+
+        key = self.verifying_key
+
+        raw_signature = libsecp256k1_ffi.new('secp256k1_ecdsa_signature *')
+
+        parsed = libsecp256k1.secp256k1_ecdsa_signature_parse_compact(
+            key.context.ctx, raw_signature, signature
+        )
+        assert parsed == 1
+
+        normalized_signature = libsecp256k1_ffi.new('secp256k1_ecdsa_signature *')
+
+        libsecp256k1.secp256k1_ecdsa_signature_normalize(
+            key.context.ctx, normalized_signature, raw_signature
+        )
+
+        verified = libsecp256k1.secp256k1_ecdsa_verify(
+            key.context.ctx, normalized_signature, digest, key.public_key
+        )
+
+        return bool(verified)
+
+
+class PrivateKey(_KeyBase):
+    """A BIP32 private key."""
+
+    HARDENED = 1 << 31
+
+    def __init__(self, ledger, privkey, chain_code, n, depth, parent=None):
+        super().__init__(ledger, chain_code, n, depth, parent)
+        if isinstance(privkey, cPrivateKey):
+            self.signing_key = privkey
+        else:
+            self.signing_key = self._signing_key_from_privkey(privkey)
+
+    @classmethod
+    def _signing_key_from_privkey(cls, private_key):
+        """ Converts a 32-byte private key into an coincurve.PrivateKey object. """
+        return cPrivateKey.from_int(PrivateKey._private_key_secret_exponent(private_key))
+
+    @classmethod
+    def _private_key_secret_exponent(cls, private_key):
+        """ Return the private key as a secret exponent if it is a valid private key. """
+        if not isinstance(private_key, (bytes, bytearray)):
+            raise TypeError('private key must be raw bytes')
+        if len(private_key) != 32:
+            raise ValueError('private key must be 32 bytes')
+        return int.from_bytes(private_key, 'big')
+
+    @classmethod
+    def from_seed(cls, ledger, seed) -> 'PrivateKey':
+        # This hard-coded message string seems to be coin-independent...
+        hmac = hmac_sha512(b'Bitcoin seed', seed)
+        privkey, chain_code = hmac[:32], hmac[32:]
+        return cls(ledger, privkey, chain_code, 0, 0)
+
+    @classmethod
+    def from_pem(cls, ledger, pem) -> 'PrivateKey':
+        der = pem_to_der(pem.encode())
+        try:
+            key_int = ECPrivateKey.load(der).native['private_key']
+        except ValueError:
+            key_int = PrivateKeyInfo.load(der).native['private_key']['private_key']
+        private_key = cPrivateKey.from_int(key_int)
+        return cls(ledger, private_key, bytes((0,)*32), 0, 0)
+
+    @classmethod
+    def from_bytes(cls, ledger, key_bytes) -> 'PrivateKey':
+        return cls(ledger, cPrivateKey(key_bytes), bytes((0,)*32), 0, 0)
+
+    @cachedproperty
+    def private_key_bytes(self):
+        """ Return the serialized private key (no leading zero byte). """
+        return self.signing_key.secret
+
+    @cachedproperty
+    def public_key(self) -> PublicKey:
+        """ Return the corresponding extended public key. """
+        verifying_key = self.signing_key.public_key
+        parent_pubkey = self.parent.public_key if self.parent else None
+        return PublicKey(
+            self.ledger, verifying_key, self.chain_code,
+            self.n, self.depth, parent_pubkey
+        )
+
+    def ec_point(self):
+        return self.public_key.ec_point()
+
+    def secret_exponent(self):
+        """ Return the private key as a secret exponent. """
+        return self.signing_key.to_int()
+
+    def wif(self):
+        """ Return the private key encoded in Wallet Import Format. """
+        return self.ledger.private_key_to_wif(self.private_key_bytes)
+
+    @property
+    def address(self):
+        """ The public key as a P2PKH address. """
+        return self.public_key.address
+
+    def child(self, n) -> 'PrivateKey':
+        """ Return the derived child extended private key at index N."""
+        if not 0 <= n < (1 << 32):
+            raise ValueError('invalid BIP32 private key child number')
+
+        if n >= self.HARDENED:
+            serkey = b'\0' + self.private_key_bytes
+        else:
+            serkey = self.public_key.pubkey_bytes
+
+        msg = serkey + n.to_bytes(4, 'big')
+        L_b, R_b = self._hmac_sha512(msg)  # pylint: disable=invalid-name
+        derived_key = self.signing_key.add(L_b)
+        return PrivateKey(self.ledger, derived_key, R_b, n, self.depth + 1, self)
+
+    def sign(self, data):
+        """ Produce a signature for piece of data by double hashing it and signing the hash. """
+        return self.signing_key.sign(data, hasher=double_sha256)
+
+    def sign_compact(self, digest):
+        """ Produce a compact signature. """
+        key = self.signing_key
+
+        signature = libsecp256k1_ffi.new('secp256k1_ecdsa_signature *')
+        signed = libsecp256k1.secp256k1_ecdsa_sign(
+            key.context.ctx, signature, digest, key.secret,
+            libsecp256k1_ffi.NULL, libsecp256k1_ffi.NULL
+        )
+
+        if not signed:
+            raise ValueError('The private key was invalid.')
+
+        serialized = libsecp256k1_ffi.new('unsigned char[%d]' % CDATA_SIG_LENGTH)
+        compacted = libsecp256k1.secp256k1_ecdsa_signature_serialize_compact(
+            key.context.ctx, serialized, signature
+        )
+        if compacted != 1:
+            raise ValueError('The signature could not be compacted.')
+
+        return bytes(libsecp256k1_ffi.buffer(serialized, CDATA_SIG_LENGTH))
+
+    def identifier(self):
+        """Return the key's identifier as 20 bytes."""
+        return self.public_key.identifier()
+
+    def extended_key(self):
+        """Return a raw extended private key."""
+        return self._extended_key(
+            self.ledger.extended_private_key_prefix,
+            b'\0' + self.private_key_bytes
+        )
+
+    def to_pem(self):
+        return self.signing_key.to_pem()
+
+
+def _from_extended_key(ledger, ekey):
+    """Return a PublicKey or PrivateKey from an extended key raw bytes."""
+    if not isinstance(ekey, (bytes, bytearray)):
+        raise TypeError('extended key must be raw bytes')
+    if len(ekey) != 78:
+        raise ValueError('extended key must have length 78')
+
+    depth = ekey[4]
+    n = int.from_bytes(ekey[9:13], 'big')
+    chain_code = ekey[13:45]
+
+    if ekey[:4] == ledger.extended_public_key_prefix:
+        pubkey = ekey[45:]
+        key = PublicKey(ledger, pubkey, chain_code, n, depth)
+    elif ekey[:4] == ledger.extended_private_key_prefix:
+        if ekey[45] != 0:
+            raise ValueError('invalid extended private key prefix byte')
+        privkey = ekey[46:]
+        key = PrivateKey(ledger, privkey, chain_code, n, depth)
+    else:
+        raise ValueError('version bytes unrecognised')
+
+    return key
+
+
+def from_extended_key_string(ledger, ekey_str):
+    """Given an extended key string, such as
+
+    xpub6BsnM1W2Y7qLMiuhi7f7dbAwQZ5Cz5gYJCRzTNainXzQXYjFwtuQXHd
+    3qfi3t3KJtHxshXezfjft93w4UE7BGMtKwhqEHae3ZA7d823DVrL
+
+    return a PublicKey or PrivateKey.
+    """
+    return _from_extended_key(ledger, Base58.decode_check(ekey_str))

lbry/wallet/checkpoints.py

@@ -1064,4 +1064,182 @@ HASHES = {
     1062000: 'c44d02a890aa66979b10d1cfa597c877f498841b4e12dd9a7bdf8d4a5fccab80',
     1063000: '1c093734f5f241b36c1b9971e2759983f88f4033405a2588b4ebfd6998ac7465',
     1064000: '9e354a83b71bbb9704053bfeea038a9c3d5daad080c6406c698b047c634706a6',
+    1065000: '563188accc4a6e311bd5046516a92a233f11f891b2304d37f151c5a6002b6958',
+    1066000: '333f1b4e996fac87e32dec667533715b31f1736b4342806a81d568b5c5238456',
+    1067000: 'df59a0b7319d5269bdf55043d91ec62bbb30829bb7054da623717a394b6ed678',
+    1068000: '06d8b674a205393edaf20c1d837baadc9caf0b0a675645246263cc163302241d',
+    1069000: 'ac065c48fad1383039d39e23c8367bad7cf9a37e07a5294cd7b04af5827b9961',
+    1070000: '90cd8b50f94208bc459081356474a961f6b764a1217f8fd291f5e4828081b730',
+    1071000: '3c0aa207ba9eea45458ab4fa26d6a027862592adb9bcce30915816e777dc6cfc',
+    1072000: '3d556c08f2300b67b704d3cbf46e22866e3ac164472b5930e2ada23b08475a0f',
+    1073000: 'a39b5c54c24efe3066aa203358b96baea405cd59aac6b0b48930e77799b4dd7d',
+    1074000: 'e8c8273d5a50a60e8744716c9f31496fb29eca87b4d68643f4ecd7ec4e400e23',
+    1075000: 'b8043ae41a1d0d7d4310c85764fcba1424733df347ffc2e8cbda1fe6ccbb5153',
+    1076000: '58468db1f91805e767d334824d6bffe54e0f900d1fb2a89b105086a493053b3d',
+    1077000: '04a78749b58465efa3a56d1735cd082c1f0f796e26486c7136950dbaf6effaa4',
+    1078000: 'e1dd6b58c75b01a67d4a4594dc7b4b2ee9e7d7fa7b25fd6246ce0e86eff33c75',
+    1079000: 'd239af017a6bb664485b14ad15e0eb703775e43018a045a8612b3697794460da',
+    1080000: '29ae5503f8c1249fefeb63fd967a71a70588ee0db1c97497e16366163a684341',
+    1081000: '05103ab27469e0859cbcd3daf42faa2bae798f522534697c7f2b34f7a050ee0f',
+    1082000: '4553d2cb7e90b6db11d242e287fe96822e6cd60e6388b94bf9006411f202ba03',
+    1083000: '97995acd178b2a142d571d5ae1c2a3deaf93a909fd91fb9c541d57f73e32dc99',
+    1084000: '9e3f23376af14d76ab24cd54e321dec019af73ad61067d959ff90043acc5ffcc',
+    1085000: '81c056b14f13cee0d6d6c8079fdd5a1a84c3a5c76cc9448612e8ef6d3531300e',
+    1086000: '8a0004f6809bdd075915a804e43991dfe8f22e05679d2fdaf8e373f101bac5c2',
+    1087000: '27c45a4c9ad24e038f2ebe40835a1c49ac7221d7185082866ee354351ba87c7a',
+    1088000: 'fd27e21747117b00b4ada1cba161ac49edb57cca540f86ac5ba885050f08f824',
+    1089000: 'bff867335767103bc3ed15ede5b9fde88016f8ede15dc5bf3e81ea40dcfc61ae',
+    1090000: '608f75016d1db08888dd59640f63e838c19bdfa833c0cc177ad3d2b818b0db5b',
+    1091000: '90750b452bd4dedaab6b57fecbfe88f71ce3d5437fad7f9ec0fdd270445c7526',
+    1092000: '98287b39f9f1233017dc5d932e5c77f0521ca84587eb3f39f0e7b6c297c749af',
+    1093000: '68a5846ed05c9bb142197849106838765f90f15c10b2cc938eef49b95eaa9d33',
+    1094000: '5660a1aac2fc763a417fc656c8887fc8186bf613ae1ccbb1a664fb43ce1fa1d6',
+    1095000: '62bad3db418b3f4cad3596881b645b72479c71deb0d39c7a4c8bd1577dc225fd',
+    1096000: 'e0e4b2b183591f10dd5614c289412f2fb5e320b7d3278f7c028f42f591872666',
+    1097000: 'a233a233fc2aa5dab9e75106d91388343ef969458ea974f1409a2ab5fc441911',
+    1098000: '16dfa5fa6cbd1188e562697b5f00ac206960d0851ed84adf37ae975fd5ffdd6a',
+    1099000: 'b8a870b7dc6d3263730c00f59d52aa6cce35dc59aa8fba715034cc2d14927260',
+    1100000: 'a3cd7749743da22a3846dcc2edbf1df21b938e829419389e3bc09284797c5b43',
+    1101000: '75b14c2a95e2a095949729b7c0b624bd725a2de98404a8e3247b60c977d0198e',
+    1102000: '4d3af64d37064dd5f57e25d61f248a1e21c1b1cadd7bb1404e35c9fbe06f1fd4',
+    1103000: 'd73c92bfed358dfcd7659228974ab75ea2fc86f2301ee47133adad8075203872',
+    1104000: '30cd82354f37bc0b412123867c7e1835206022a7501853bf8c0d3df02f291645',
+    1105000: '1d2ef984f26693dce77460cd2694e5da46e675077e91a1cea26051733b01a7ef',
+    1106000: '51c076c304222fe3ca308ba6968c46fef448f85be13a095cecb75b90e7954698',
+    1107000: '99e2221339e16acc34c9816f2ef7b866c2dd753aa3cbe484ae831959a23ece68',
+    1108000: '0f1227c250296bfe88eb7eb41703f99f633cfe02870816111e0cadfe778ddb19',
+    1109000: 'b35447f1ad76f95bc4f5886e4028d33acb3ad7b5000dd15516d3f11ce4baa990',
+    1110000: 'ac7baff996062bfaaaddd7d496b17e3ec1c8d34b2143095645ff22fb3888ae00',
+    1111000: '430bbbdcca36b2d69b6a2dd8b07c583a060a467e5f9acbc6de62462e1f7c7036',
+    1112000: 'e5274dea029dc44baff55c05b0555f91b74d29ffd40e3a8c4e2c5b57f9d40bef',
+    1113000: 'cf43863249fa42cfe108220dd40169dac702b0dd9cf5cb699cf2fc96feda8371',
+    1114000: 'fa1c0e551784d21c451564124d2d730e616724f3e535de3c186bcdeb47e80a8f',
+    1115000: '49fe6ecee35a397b83b5a704e950ad028cfb4b7e7a524021e789f4acc0fd6ffe',
+    1116000: '74ecded36751aa8b7901b31f0d16d75d111fc3c40b567f649c04f74ed028aa5c',
+    1117000: 'd9ca760a22190bdf545766b47d963c738a4edcc27f4d15ca801b35751577cfa7',
+    1118000: 'c28d42f871682800ac4e867608227cfb6bc4c00b618e83a8556f201a1c28813c',
+    1119000: 'c5fafc4e1785b0b9e84bb052e392154a5ba1aefe612998017e90772bcd554e08',
+    1120000: 'aa054d428bc9ccee0761da92163817163413065fe1e67ef79a056c5233ea3476',
+    1121000: '0df295bb944218503bd1bf66d2ece0c50fd22dae3391b80673a7ad1e4e5c3934',
+    1122000: 'a13abb350a26673b3933b1de307a60a6845ca594d502599548c6253e21a6d8e8',
+    1123000: 'a4bc6a3abf9ed1f4b14338ff0f03f83456312bc91a93fa89ae6db493050115e1',
+    1124000: '65869938df99adf0dda76200291ce09a54c9bcc787e4bb62cd72c367db58f4f0',
+    1125000: 'ea5e918233b14c3c73d488a906e3741c61bdcafe0393bd0404168fe80c950a46',
+    1126000: 'ce88cd35104fcec51bcee77302e03162dc694802536f5b668786b2245e61bca5',
+    1127000: 'ea19c0c8d205be4be87d02c5301c9ed331e7d75e25b93d1c2137c248882af515',
+    1128000: '006f32d63c2a3adcf4fbad0b0629c97f1beab6446a9c27fbde9472f2d066219e',
+    1129000: '218e5392e1ecf471c3bbc3d79c24dee30ac8db315dbeb61317318efb3f221163',
+    1130000: '30b9da0bd8364e9cd5551b2529341a01a3b7257a238d15b2560e2c99fdb324e8',
+    1131000: '8a7f382cfa023d2eba6639443e67206f8883b57d23ce7e1339234b8bb3098a82',
+    1132000: 'bf9af68a6fe2112d8fe311dfd52334ae2e7b0bac6675c9ebfddb1f386c212668',
+    1133000: '1a30951e2be633502a47c255a93ddbb9ed231d6bb4c55a807c0e910b437766b3',
+    1134000: 'a9bcaf3300b7915e701a8e396eb13f0c7287576323420be7aab3c3ba48020f76',
+    1135000: '337eed9ed072b5ad862af2d3d651f1b49fa852abc590b7e1c2dc381b496f438a',
+    1136000: '208761dbc29ec58302d722a05e937a3cf9e78bfb6495be395dd7b54f02e169dc',
+    1137000: '4e5b67ff3324b64e268049fdc3d82982b847ee359d409ade6368864c38a111e5',
+    1138000: '55d1d0833021a664e85eec8cc90a0985e67cc80d28841aaa8c2231ec28087ebb',
+    1139000: 'e750ada1ec9fa0f2f2461ed68958c7d116a699a82ec12911da5563139f8df19e',
+    1140000: '9cf81407b6ccc8046f0233f97484166945758f7392bb54841c912fcb34cf205c',
+    1141000: 'fccf32b2fae03e3b6b562483776625f9843cd68734c55659e2069cde7e383170',
+    1142000: 'c3608c215dd6569da6c1871c4d72a09ab1caa9663647f2a9454b5693d5d72a65',
+    1143000: 'bd39cb8c4e529d15bbea6baeec66afe52ca18afe32bd812f28fbb0676647cdff',
+    1144000: '6e42d02538565ce7e2d9bf31a304f1fd0ac122d35d17a030160575815901b0b1',
+    1145000: 'b9722e1de2904ce1219140fffb1f4f9f5a041f885faa634404238d103c738b4c',
+    1146000: 'd4de4271459966cee774f538a243d7db0689b213b296463d42e45c93194d7861',
+    1147000: '51fadf109f22bb85574d0fbcbd0b20992983e89aee3d415a7b1c37c44775d9a9',
+    1148000: '137e1fe8da31680d21a42e7421eb608a883a497314e4404625ce44b0edadde6a',
+    1149000: 'cb87867eb04203ce15e0763a2f4389376cea75e0a2877f55e2911c575bef07a8',
+    1150000: '977528ca7953a2c9c19fefaa3aab7ebdec3ac324d74a07d83764ba25d9be0689',
+    1151000: 'a09c51c832600ded63a19201df008075273ea248fd406886e93a2cbaa3bba46b',
+    1152000: '0e5367cfa0f00dd932a5bcc00dcc807fa6825161806bed588e16a57947b4b32d',
+    1153000: '55a9de3dcde2efb56a3c5fea7d22b98c1e180db9a4d4f4f6be7aae1f1cbd7608',
+    1154000: 'abc58cf71c4691ebfaef920252730cf69abbe9de88b424c03051b9b03e85d45a',
+    1155000: '4f074ce73c8a096620b8a32498362eb66a072eae95d561f2d53557cd513ae785',
+    1156000: '540a838a0f0a8834466b17dd456d35b8acae2ec8419f8bd9a704d9ea439062ac',
+    1157000: 'd5310ac671abdb658ea028db86c23fc729af965f91d67a37218c1412cf32a1f5',
+    1158000: '162d906a07e6c35e7c3ebf7069a200521605a97920f5b589d31b19bfd7766ee2',
+    1159000: '600bd8f5e1e62219e220f4dcb650db5812e79956f95ae8a50e83126932685ee0',
+    1160000: '91319398d1a805fac8582c8485e6d84e7490d6cfa6e44e2c630665b6bce0e6b8',
+    1161000: 'f7ad3cff6ee76e1e3df4abe70c600e4af66e1df55bf7b03aee12251d4455a1d4',
+    1162000: '85b9fbba669c2a4d3f85cdb5123f9538c05bd66172b7236d756703f99258454d',
+    1163000: '966085d767d1e5e2e8baf8eda8c11472ec5351181c418b503585284009aaea79',
+    1164000: '1c94e1b531215c019b12caf407296d8868481f49524b7180c7161b0363c1f789',
+    1165000: '803b6bf93735aeae2cf607824e2adf0d754b58da2516c2da1e485c697e472143',
+    1166000: '872561a82f7991633d0927d25cb659d096bbe556fe6dac7a0b6a679820733069',
+    1167000: '6bd7cdd605a3179b54c8af88d1638bf8133fab12cbf0a78d37cf21eddf4395a1',
+    1168000: '79946f5758c1817239cc642d27298bd710983551a8236e49832c6d818b097337',
+    1169000: 'b0994c60728e74de4aa361f37fa85e5296ce3188ae4e0b66d7b34fe86a239c9c',
+    1170000: 'a54188a5a64e0cf8da2406d16a0ac3983b087fc7d6231b6f8abf92cf11dc78cd',
+    1171000: 'ec2924d98e470cc6359821e6468df2c15d60301861d443188730342581230ef2',
+    1172000: 'b4ac11116aa73ce19428009a80e583e19dc9bcd380f7f7ce272a92921d5868d2',
+    1173000: '501d3551f762999dd5a799f3c5658fff2a7f3aff0511488272cd7693fefb8f9d',
+    1174000: '4660074ea48a78ae453cb14b694b2844cc0fb63ed9352ed20d11158bbb5c1f28',
+    1175000: '0727f6b1d9f8fe5677a9ffa0d475f53f5a419ef90b80896c22c2c95de22175de',
+    1176000: '150633d6a35496c24a93c9e19817e90f649c56b7e2558f99e97325bfd5df8b17',
+    1177000: '0849e19f22571b62dba8ff02f6b5a064a7ac36e7ed491321b3663567e8e17294',
+    1178000: '770dd463e7bad80f689f12934e4ae06e24378d1545dcf211fd143beaef49464e',
+    1179000: '059d383dcc60a49b658b674d92fc35cab07b06329c58d73818b6387cb0c06534',
+    1180000: 'e547cb3c636243ca9ae4cfb92c30a0f583eda84e329a5c1e5f64a26fc6fc791e',
+    1181000: '4521a4396ab02f73d45d7a3393ea1c602d255778d52c12079c88bfbad32aab43',
+    1182000: '051cfe993e4b0b34233403a9e8c397dd50e8b78a30fb07e9c260604ee9e624a9',
+    1183000: '44a69c99bb8b85e84ae279f2d8e5400d51cb3d5f0bcd178db49d55548cd66191',
+    1184000: '2a1d23c9bb3c71a533e0c9d25b03bfa7e9db8e014645f3e7fbede6d99fff0191',
+    1185000: 'bb90d6c6d77819163a9e909ee621d874707cdb21c91b1d9e861b204cf37d0ffa',
+    1186000: '4a92051b738ea0e28c64c64f1eb6f0405bc7c3427bef91ff20f4c43cf084d750',
+    1187000: 'f782ac330ca20fb5d8a094ee0f0f8c086a76e3f03ecc6a2c42f8fd07e52e0f41',
+    1188000: '94cb7b653dd3d838c186420158cf0e73db73ec28deaf67d9a2ca902caba4141a',
+    1189000: 'c8128e59b9ec948de890184578a113478ea63f7d57cb75c2c8d5c001a5a724c0',
+    1190000: '4da643bd35e5b98932ae21515a6bffb9c72f2cd8d514cd2d7eac1922af785c3f',
+    1191000: '0f922d86658ac3f53c5f9db360c68ab3f3253a925f23e1323820e3384214719a',
+    1192000: '4c3ab631cf5ba0c236f7c64af6f790fc24448319de6f75dbd28df4e2648d0b7d',
+    1193000: 'eda118d1fac3470a1f8f01f5c78108c8ecdcd6420be30f6d20f1d1831e7b6975',
+    1194000: '5723fff88abd9bb5088476fa5f4221a61c6f8a718703a92f13248ad350abeea2',
+    1195000: '1715846f82d011919e3446c6ce675a65fb80338bd791d4e735702c4767d9adc4',
+    1196000: 'b497667996aee2db61e88f442e728be15ab0b2b64cfd43198691fcf6cdafacc8',
+    1197000: '309a6170d837b8cb334fb888a64ed4e47e6592747e93c8e9d1bf7d608cfef87d',
+    1198000: '3ea918ef64a67dec20051519e6aefaeb7aca2d8583baca9ad5c5bd07073e513a',
+    1199000: '4ec7b7361b0243e5b2996a16e3b27acd662126b95fe542a487c7030e47ea3667',
+    1200000: 'b829c742686fcd642d0f9443336d7e2c4eab81667c90ce553df1350ed10b4233',
+    1201000: '44c022887f1e126fd281b1cae26b2017fa6415a64b105762c87643204ce165a5',
+    1202000: 'b11cc739eb28a14f4e47be125aa7e62d6d6f90c8f8014ee70044ed506d53d938',
+    1203000: '997a7c5fd7a98b39c9ca0790519924d73c3567656b605c97a6fdb7b406c3c64d',
+    1204000: '7d25d872e17195ee277243f7a5a39aa64d8750cec62e4777146acf61a8e76b04',
+    1205000: 'ce8486ae745a4645bee081ef3291d9505174bed05b0668d963b2998b7643dbb0',
+    1206000: '46a0bcea3c411c600dffe3e06e3d1dfbf5879a7ec4dcf3848e794cefcbf2bc0b',
+    1207000: '37e6297bf6e4e2bdd40401d4d7f95e3e3bdafd4a7f76b9c52865cefc6b82b20b',
+    1208000: 'd09e3982a9827b8cf56a5a2f4031dc6b082926c1fd57b63beaaa6cfd534eb902',
+    1209000: '54ae9010a9f146c83464e7ee60b30d9dbee36418561abc4e8d61bce9baa2d21d',
+    1210000: '5dcfd33f8e5ac21c9ba8553758b8cd8afae7961cad428530b5109c2db2ebf39f',
+    1211000: '91c952348bb2c3dfac0d6531a3dac770ea6dab571af257530e9c55493c96bdd9',
+    1212000: 'e62cc3fe044a7f5de4c04a8aed5619548f9d5c6fad9f989d3382cb96de1d780d',
+    1213000: '66b46ffdca8acf1dd04528dadb28b6ac4ce38807c1b84abd685d4ddb3dc59a34',
+    1214000: '2ce4091756ad23746bab4906f46545953cadaf61deae0d78e8a10d4eb51866b1',
+    1215000: '83ce3ca087799cdc4b4c5e7cfeb4a127708724a7ca76aa5f7f4ec1ed48b5fca6',
+    1216000: '7d07b739b7991fbd74926281bf51bba9d5721afab39598720f9ff5f7410a6721',
+    1217000: '76adf49491670d0e8379058eacf0228f330f3c18955dfea1ebe43bc11ee065f3',
+    1218000: '77f422e7301a81692dec69e5c6d35fa988a00a4d820ad0ebb1d595add36558cc',
+    1219000: '8ba9d944f8c468c81799294aeea8dc05ed1bb90bb26552fcd190bd88fedcddf2',
+    1220000: '00330367c255e0fe51b374597995c53353bc5700ad7d603cbd4197141933fe9c',
+    1221000: '3ba8b316b7964f31fdf628ed869a6fd023680cca6611257a31efe22e4d17e578',
+    1222000: '016e58d3fb6a29a3f9281789359460e776e9feb2f0db500482b6e231e1272aef',
+    1223000: 'fdfe767c29a3de7acd913b627d1e5fa887a1af9974f6a8a6474db822468c785c',
+    1224000: '92239f6207bff3689c554e92b24fe2e7be4a2203104ad8ef08b2c6bedd9aeccf',
+    1225000: '9a2f2dd9527b533d3d743efc55236e73e15192171bc8d0cd910918d1ab00aef7',
+    1226000: 'eb8269c75b8c5f66e6ea88ad70883dddcf8a75a45198ca7a46eb0ec606a791bb',
+    1227000: '5c82e624390cd57942dc9d64344eaa3d8991e0437e01802473053245b706290c',
+    1228000: '51e9a7d727f07fc01be7c03e3dd854eb666697f05bf89259baac628520d4402c',
+    1229000: 'c4bfdb651c9abdeda717fb9c8a4c8a6c9c0f78c13d3e6cae3f24f504d734c643',
+    1230000: '9f1ce781d16f2334567cbfb22fff42c14d2b9290cc2883746f435a1fb127021d',
+    1231000: '5c996634b377412ae0a3d8f541f3cc4a354aab72c198aa23a5cfc2678cbabf09',
+    1232000: '86702316a2d1730fbae01a08f36fffe5bf6d3ebb7d76b35a1617713766698b46',
+    1233000: 'fb16b63916c0287cb9b01d0c5aad626ced1b73c49a374c9009703aa90fd27a82',
+    1234000: '7c6f7904602ccd86bfb05cb8d6b5547c989c57cb2e214e93f1220fa4fe29bcb0',
+    1235000: '898b0f20811f52aa5a6bd0c35eff86fca3fbe3b066e423644fa77b2e269d9513',
+    1236000: '39128910ef624b6a8bbd390a311b5587c0991cda834eed996d814fe410cac352',
+    1237000: 'a0709afeedb64af4168ce8cf3dbda667a248df8e91da96acb2333686a2b89325',
+    1238000: 'e00075e7ba8c18cc277bfc5115ae6ff6b9678e6e99efd6e45f549ef8a3981a3d',
+    1239000: '3fba891600738f2d37e279209d52bbe6dc7ce005eeed62048247c96f370e7cd5',
+    1240000: 'def9bf1bec9325db90bb070f532972cfdd74e814c2b5e74a4d5a7c09a963a5f1',
+    1241000: '6a5d187e32bc189ac786959e1fe846031b97ae1ce202c22e1bdb1d2a963005fd',
+    1242000: 'a74d7c0b104eaf76c53a3a31ce51b75bbd8e05b5e84c31f593f505a13d83634c',
 }

lbry/wallet/coinselection.py

@@ -141,7 +141,7 @@ class CoinSelector:
                     _) -> List[OutputEffectiveAmountEstimator]:
         """ Accumulate UTXOs at random until there is enough to cover the target. """
         target = self.target + self.cost_of_change
-        self.random.shuffle(txos, self.random.random)
+        self.random.shuffle(txos, random=self.random.random)  # pylint: disable=deprecated-argument
         selection = []
         amount = 0
         for coin in txos:

lbry/wallet/database.py

@@ -13,7 +13,7 @@ from datetime import date
 from prometheus_client import Gauge, Counter, Histogram
 from lbry.utils import LockWithMetrics
 
-from scribe.schema.bip32 import PublicKey
+from .bip32 import PublicKey
 from .transaction import Transaction, Output, OutputScript, TXRefImmutable, Input
 from .constants import TXO_TYPES, CLAIM_TYPES
 from .util import date_to_julian_day
@@ -1211,6 +1211,7 @@ class Database(SQLiteMixin):
         return addresses
 
     async def get_address_count(self, cols=None, read_only=False, **constraints):
+        self._clean_txo_constraints_for_aggregation(constraints)
         count = await self.select_addresses('COUNT(*) as total', read_only=read_only, **constraints)
         return count[0]['total'] or 0
 

lbry/wallet/ledger.py

@@ -10,8 +10,8 @@ from collections import defaultdict
 from binascii import hexlify, unhexlify
 from typing import Dict, Tuple, Type, Iterable, List, Optional, DefaultDict, NamedTuple
 
-from scribe.schema.result import Outputs, INVALID, NOT_FOUND
-from scribe.schema.url import URL
+from lbry.schema.result import Outputs, INVALID, NOT_FOUND
+from lbry.schema.url import URL
 from lbry.crypto.hash import hash160, double_sha256, sha256
 from lbry.crypto.base58 import Base58
 from lbry.utils import LRUCacheWithMetrics
@@ -26,7 +26,7 @@ from lbry.wallet.transaction import Transaction, Output
 from lbry.wallet.header import Headers, UnvalidatedHeaders
 from lbry.wallet.checkpoints import HASHES
 from lbry.wallet.constants import TXO_TYPES, CLAIM_TYPES, COIN, NULL_HASH32
-from scribe.schema.bip32 import PublicKey, PrivateKey
+from lbry.wallet.bip32 import PublicKey, PrivateKey
 from lbry.wallet.coinselection import CoinSelector
 
 log = logging.getLogger(__name__)
@@ -106,7 +106,7 @@ class Ledger(metaclass=LedgerRegistry):
     target_timespan = 150
 
     default_fee_per_byte = 50
-    default_fee_per_name_char = 200000
+    default_fee_per_name_char = 0
 
     checkpoints = HASHES
 
@@ -329,10 +329,10 @@ class Ledger(metaclass=LedgerRegistry):
     async def start(self):
         if not os.path.exists(self.path):
             os.mkdir(self.path)
-        await asyncio.wait([
+        await asyncio.wait(map(asyncio.create_task, [
             self.db.open(),
             self.headers.open()
-        ])
+        ]))
         fully_synced = self.on_ready.first
         asyncio.create_task(self.network.start())
         await self.network.on_connected.first
@@ -466,9 +466,9 @@ class Ledger(metaclass=LedgerRegistry):
     async def subscribe_accounts(self):
         if self.network.is_connected and self.accounts:
             log.info("Subscribe to %i accounts", len(self.accounts))
-            await asyncio.wait([
+            await asyncio.wait(map(asyncio.create_task, [
                 self.subscribe_account(a) for a in self.accounts
-            ])
+            ]))
 
     async def subscribe_account(self, account: Account):
         for address_manager in account.address_managers.values():
@@ -722,6 +722,15 @@ class Ledger(metaclass=LedgerRegistry):
                 return account.address_managers[details['chain']]
         return None
 
+    async def broadcast_or_release(self, tx, blocking=False):
+        try:
+            await self.broadcast(tx)
+        except:
+            await self.release_tx(tx)
+            raise
+        if blocking:
+            await self.wait(tx, timeout=None)
+
     def broadcast(self, tx):
         # broadcast can't be a retriable call yet
         return self.network.broadcast(hexlify(tx.raw).decode())
@@ -780,13 +789,9 @@ class Ledger(metaclass=LedgerRegistry):
             include_is_my_output=False,
             include_sent_supports=False,
             include_sent_tips=False,
-            include_received_tips=False,
-            hub_server=False) -> Tuple[List[Output], dict, int, int]:
+            include_received_tips=False) -> Tuple[List[Output], dict, int, int]:
         encoded_outputs = await query
-        if hub_server:
-            outputs = Outputs.from_grpc(encoded_outputs)
-        else:
-            outputs = Outputs.from_base64(encoded_outputs or '')  # TODO: why is the server returning None?
+        outputs = Outputs.from_base64(encoded_outputs or '')  # TODO: why is the server returning None?
         txs: List[Transaction] = []
         if len(outputs.txs) > 0:
             async for tx in self.request_transactions(tuple(outputs.txs), cached=True):
@@ -862,13 +867,10 @@ class Ledger(metaclass=LedgerRegistry):
                         txo.received_tips = tips
         return txos, blocked, outputs.offset, outputs.total
 
-    async def resolve(self, accounts, urls, new_sdk_server=None, **kwargs):
+    async def resolve(self, accounts, urls, **kwargs):
         txos = []
         urls_copy = list(urls)
-        if new_sdk_server:
-            resolve = partial(self.network.new_resolve, new_sdk_server)
-        else:
-            resolve = partial(self.network.retriable_call, self.network.resolve)
+        resolve = partial(self.network.retriable_call, self.network.resolve)
         while urls_copy:
             batch, urls_copy = urls_copy[:100], urls_copy[100:]
             txos.extend(
@@ -893,17 +895,14 @@ class Ledger(metaclass=LedgerRegistry):
         return await self.network.sum_supports(new_sdk_server, **kwargs)
 
     async def claim_search(
-            self, accounts, include_purchase_receipt=False, include_is_my_output=False,
-            new_sdk_server=None, **kwargs) -> Tuple[List[Output], dict, int, int]:
-        if new_sdk_server:
-            claim_search = partial(self.network.new_claim_search, new_sdk_server)
-        else:
-            claim_search = self.network.claim_search
+            self, accounts,
+            include_purchase_receipt=False,
+            include_is_my_output=False,
+            **kwargs) -> Tuple[List[Output], dict, int, int]:
         return await self._inflate_outputs(
-            claim_search(**kwargs), accounts,
+            self.network.claim_search(**kwargs), accounts,
             include_purchase_receipt=include_purchase_receipt,
-            include_is_my_output=include_is_my_output,
-            hub_server=new_sdk_server is not None
+            include_is_my_output=include_is_my_output
         )
 
     # async def get_claim_by_claim_id(self, accounts, claim_id, **kwargs) -> Output:
@@ -939,9 +938,7 @@ class Ledger(metaclass=LedgerRegistry):
                              "%d change addresses (gap: %d), %d channels, %d certificates and %d claims. ",
                              account.id, balance, total_receiving, account.receiving.gap, total_change,
                              account.change.gap, channel_count, len(account.channel_keys), claim_count)
-        except Exception as err:
-            if isinstance(err, asyncio.CancelledError):  # TODO: remove when updated to 3.8
-                raise
+        except Exception:
             log.exception(
                 'Failed to display wallet state, please file issue '
                 'for this bug along with the traceback you see below:')
@@ -964,9 +961,7 @@ class Ledger(metaclass=LedgerRegistry):
             claim_ids = [p.purchased_claim_id for p in purchases]
             try:
                 resolved, _, _, _ = await self.claim_search([], claim_ids=claim_ids)
-            except Exception as err:
-                if isinstance(err, asyncio.CancelledError):  # TODO: remove when updated to 3.8
-                    raise
+            except Exception:
                 log.exception("Resolve failed while looking up purchased claim ids:")
                 resolved = []
             lookup = {claim.claim_id: claim for claim in resolved}
@@ -1046,9 +1041,7 @@ class Ledger(metaclass=LedgerRegistry):
         claim_ids = collection.claim.collection.claims.ids[offset:page_size + offset]
         try:
             resolve_results, _, _, _ = await self.claim_search([], claim_ids=claim_ids)
-        except Exception as err:
-            if isinstance(err, asyncio.CancelledError):  # TODO: remove when updated to 3.8
-                raise
+        except Exception:
             log.exception("Resolve failed while looking up collection claim ids:")
             return []
         claims = []

lbry/wallet/manager.py

@@ -3,7 +3,6 @@ import json
 import typing
 import logging
 import asyncio
-from distutils.util import strtobool
 
 from binascii import unhexlify
 from decimal import Decimal
@@ -183,7 +182,6 @@ class WalletManager:
         }[config.blockchain_name]
 
         ledger_config = {
-            'use_go_hub': not strtobool(os.environ.get('ENABLE_LEGACY_SEARCH') or 'yes'),
             'auto_connect': True,
             'explicit_servers': [],
             'hub_timeout': config.hub_timeout,
@@ -238,7 +236,6 @@ class WalletManager:
 
     async def reset(self):
         self.ledger.config = {
-            'use_go_hub': not strtobool(os.environ.get('ENABLE_LEGACY_SEARCH') or 'yes'),
             'auto_connect': True,
             'explicit_servers': [],
             'default_servers': Config.lbryum_servers.default,
@@ -320,10 +317,4 @@ class WalletManager:
         )
 
     async def broadcast_or_release(self, tx, blocking=False):
-        try:
-            await self.ledger.broadcast(tx)
-        except:
-            await self.ledger.release_tx(tx)
-            raise
-        if blocking:
-            await self.ledger.wait(tx, timeout=None)
+        await self.ledger.broadcast_or_release(tx, blocking=blocking)

lbry/wallet/network.py

@@ -7,9 +7,6 @@ from time import perf_counter
 from collections import defaultdict
 from typing import Dict, Optional, Tuple
 import aiohttp
-import grpc
-from scribe.schema.types.v2 import hub_pb2_grpc
-from scribe.schema.types.v2.hub_pb2 import SearchRequest
 
 from lbry import __version__
 from lbry.utils import resolve_host
@@ -120,7 +117,7 @@ class ClientSession(BaseClientSession):
                     )
                 else:
                     await asyncio.sleep(max(0, max_idle - (now - self.last_send)))
-        except Exception as err:
+        except (Exception, asyncio.CancelledError) as err:
             if isinstance(err, asyncio.CancelledError):
                 log.info("closing connection to %s:%i", *self.server)
             else:
@@ -217,7 +214,7 @@ class Network:
         def loop_task_done_callback(f):
             try:
                 f.result()
-            except Exception:
+            except (Exception, asyncio.CancelledError):
                 if self.running:
                     log.exception("wallet server connection loop crashed")
 
@@ -315,7 +312,8 @@ class Network:
         sleep_delay = 30
         while self.running:
             await asyncio.wait(
-                [asyncio.sleep(30), self._urgent_need_reconnect.wait()], return_when=asyncio.FIRST_COMPLETED
+                map(asyncio.create_task, [asyncio.sleep(30), self._urgent_need_reconnect.wait()]),
+                return_when=asyncio.FIRST_COMPLETED
             )
             if self._urgent_need_reconnect.is_set():
                 sleep_delay = 30
@@ -341,14 +339,13 @@ class Network:
                 try:
                     if not self._urgent_need_reconnect.is_set():
                         await asyncio.wait(
-                            [self._keepalive_task, self._urgent_need_reconnect.wait()],
+                            [self._keepalive_task, asyncio.create_task(self._urgent_need_reconnect.wait())],
                             return_when=asyncio.FIRST_COMPLETED
                         )
                     else:
                         await self._keepalive_task
                     if self._urgent_need_reconnect.is_set():
                         log.warning("urgent reconnect needed")
-                        self._urgent_need_reconnect.clear()
                     if self._keepalive_task and not self._keepalive_task.done():
                         self._keepalive_task.cancel()
                 except asyncio.CancelledError:
@@ -394,7 +391,6 @@ class Network:
                 log.warning("Wallet server call timed out, retrying.")
             except ConnectionError:
                 log.warning("connection error")
-
         raise asyncio.CancelledError()  # if we got here, we are shutting down
 
     def _update_remote_height(self, header_args):
@@ -477,21 +473,6 @@ class Network:
     def claim_search(self, session_override=None, **kwargs):
         return self.rpc('blockchain.claimtrie.search', kwargs, False, session_override)
 
-    async def new_resolve(self, server, urls):
-        message = {"method": "resolve", "params": {"urls": urls, "protobuf": True}}
-        async with self.aiohttp_session.post(server, json=message) as r:
-            result = await r.json()
-            return result['result']
-
-    async def new_claim_search(self, server, **kwargs):
-        async with grpc.aio.insecure_channel(server) as channel:
-            stub = hub_pb2_grpc.HubStub(channel)
-            try:
-                response = await stub.Search(SearchRequest(**kwargs))
-            except grpc.aio.AioRpcError as error:
-                raise RPCError(error.code(), error.details())
-            return response
-
     async def sum_supports(self, server, **kwargs):
         message = {"method": "support_sum", "params": kwargs}
         async with self.aiohttp_session.post(server, json=message) as r:

lbry/wallet/orchstr8/__init__.py

@@ -1,5 +1,2 @@
-__hub_url__ = (
-    "https://github.com/lbryio/hub/releases/download/v0.2022.01.21.1/hub"
-)
 from lbry.wallet.orchstr8.node import Conductor
 from lbry.wallet.orchstr8.service import ConductorService

lbry/wallet/orchstr8/node.py

@@ -1,5 +1,5 @@
+# pylint: disable=import-error
 import os
-import signal
 import json
 import shutil
 import asyncio
@@ -9,24 +9,29 @@ import logging
 import tempfile
 import subprocess
 import platform
-from distutils.util import strtobool
 
 from binascii import hexlify
 from typing import Type, Optional
 import urllib.request
 from uuid import uuid4
+
+
 import lbry
 from lbry.wallet import Wallet, Ledger, RegTestLedger, WalletManager, Account, BlockHeightEvent
 from lbry.conf import KnownHubsList, Config
-from lbry.wallet.orchstr8 import __hub_url__
-
-from scribe.env import Env
-from scribe.hub.service import HubServerService
-from scribe.elasticsearch.service import ElasticSyncService
-from scribe.blockchain.service import BlockchainProcessorService
 
 log = logging.getLogger(__name__)
 
+try:
+    from hub.herald.env import ServerEnv
+    from hub.scribe.env import BlockchainEnv
+    from hub.elastic_sync.env import ElasticEnv
+    from hub.herald.service import HubServerService
+    from hub.elastic_sync.service import ElasticSyncService
+    from hub.scribe.service import BlockchainProcessorService
+except ImportError:
+    pass
+
 
 def get_lbcd_node_from_ledger(ledger_module):
     return LBCDNode(
@@ -54,13 +59,10 @@ class Conductor:
         self.wallet_node = WalletNode(
             self.manager_module, RegTestLedger, default_seed=seed
         )
-        self.hub_node = HubNode(__hub_url__, "hub", self.spv_node)
-
         self.lbcd_started = False
         self.lbcwallet_started = False
         self.spv_started = False
         self.wallet_started = False
-        self.hub_started = False
 
         self.log = log.getChild('conductor')
 
@@ -74,17 +76,6 @@ class Conductor:
             await self.lbcd_node.stop(cleanup)
             self.lbcd_started = False
 
-    async def start_hub(self):
-        if not self.hub_started:
-            await self.hub_node.start()
-            await self.lbcwallet_node.running.wait()
-            self.hub_started = True
-
-    async def stop_hub(self, cleanup=True):
-        if self.hub_started:
-            await self.hub_node.stop(cleanup)
-            self.hub_started = False
-
     async def start_spv(self):
         if not self.spv_started:
             await self.spv_node.start(self.lbcwallet_node)
@@ -125,13 +116,11 @@ class Conductor:
         await self.start_lbcd()
         await self.start_lbcwallet()
         await self.start_spv()
-        await self.start_hub()
         await self.start_wallet()
 
     async def stop(self):
         all_the_stops = [
             self.stop_wallet,
-            self.stop_hub,
             self.stop_spv,
             self.stop_lbcwallet,
             self.stop_lbcd
@@ -176,7 +165,6 @@ class WalletNode:
         self.manager = self.manager_class.from_config({
             'ledgers': {
                 self.ledger_class.get_id(): {
-                    'use_go_hub': not strtobool(os.environ.get('ENABLE_LEGACY_SEARCH') or 'yes'),
                     'api_port': self.port,
                     'explicit_servers': [(spv_node.hostname, spv_node.port)],
                     'default_servers': Config.lbryum_servers.default,
@@ -184,6 +172,7 @@ class WalletNode:
                     'known_hubs': config.known_hubs if config else KnownHubsList(),
                     'hub_timeout': 30,
                     'concurrent_hub_requests': 32,
+                    'fee_per_name_char': 200000
                 }
             },
             'wallets': [wallet_file_name]
@@ -225,6 +214,7 @@ class SPVNode:
         self.port = 50001 + node_number  # avoid conflict with default daemon
         self.udp_port = self.port
         self.elastic_notifier_port = 19080 + node_number
+        self.elastic_services = f'localhost:9200/localhost:{self.elastic_notifier_port}'
         self.session_timeout = 600
         self.stopped = True
         self.index_name = uuid4().hex
@@ -246,25 +236,35 @@ class SPVNode:
                 'host': self.hostname,
                 'tcp_port': self.port,
                 'udp_port': self.udp_port,
-                'elastic_notifier_port': self.elastic_notifier_port,
+                'elastic_services': self.elastic_services,
                 'session_timeout': self.session_timeout,
                 'max_query_workers': 0,
                 'es_index_prefix': self.index_name,
-                'chain': 'regtest'
+                'chain': 'regtest',
+                'index_address_status': False
             }
             if extraconf:
                 conf.update(extraconf)
-            env = Env(**conf)
-            self.writer = BlockchainProcessorService(env)
-            self.server = HubServerService(env)
-            self.es_writer = ElasticSyncService(env)
+            self.writer = BlockchainProcessorService(
+                BlockchainEnv(db_dir=self.data_path, daemon_url=lbcwallet_node.rpc_url,
+                              reorg_limit=100, max_query_workers=0, chain='regtest', index_address_status=False)
+            )
+            self.server = HubServerService(ServerEnv(**conf))
+            self.es_writer = ElasticSyncService(
+                ElasticEnv(
+                    db_dir=self.data_path, reorg_limit=100, max_query_workers=0, chain='regtest',
+                    elastic_notifier_port=self.elastic_notifier_port,
+                    es_index_prefix=self.index_name,
+                    filtering_channel_ids=(extraconf or {}).get('filtering_channel_ids'),
+                    blocking_channel_ids=(extraconf or {}).get('blocking_channel_ids')
+                )
+            )
             await self.writer.start()
             await self.es_writer.start()
             await self.server.start()
         except Exception as e:
             self.stopped = True
-            if not isinstance(e, asyncio.CancelledError):
-                log.exception("failed to start spv node")
+            log.exception("failed to start spv node")
             raise e
 
     async def stop(self, cleanup=True):
@@ -673,139 +673,3 @@ class LBCWalletNode:
 
     def get_raw_transaction(self, txid):
         return self._cli_cmnd('getrawtransaction', txid, '1')
-
-
-class HubProcess(asyncio.SubprocessProtocol):
-    def __init__(self, ready, stopped):
-        self.ready = ready
-        self.stopped = stopped
-        self.log = log.getChild('hub')
-        self.transport = None
-
-    def pipe_data_received(self, fd, data):
-        self.stopped.clear()
-        self.ready.set()
-        if self.log:
-            self.log.info(data.decode())
-        if b'error' in data.lower():
-            self.ready.set()
-            raise SystemError(data.decode())
-        if b'listening on' in data:
-            self.ready.set()
-        str_lines = str(data.decode()).split("\n")
-        for line in str_lines:
-            if 'releaseTime' in line:
-                print(line)
-
-    def process_exited(self):
-        self.ready.clear()
-        self.stopped.set()
-
-    async def stop(self):
-        t = asyncio.create_task(self.stopped.wait())
-        try:
-            self.transport.send_signal(signal.SIGINT)
-            await asyncio.wait_for(t, 3)
-            # log.warning("stopped go hub")
-        except asyncio.TimeoutError:
-            if not t.done():
-                t.cancel()
-            self.transport.terminate()
-            await self.stopped.wait()
-            log.warning("terminated go hub")
-
-
-class HubNode:
-    def __init__(self, url, daemon, spv_node):
-        self.spv_node = spv_node
-        self.latest_release_url = url
-        self.project_dir = os.path.dirname(os.path.dirname(__file__))
-        self.bin_dir = os.path.join(self.project_dir, 'bin')
-        self.daemon_bin = os.path.join(self.bin_dir, daemon)
-        self.cli_bin = os.path.join(self.bin_dir, daemon)
-        self.log = log.getChild('hub')
-        self.transport = None
-        self.protocol = None
-        self.hostname = 'localhost'
-        self.rpcport = 50051  # avoid conflict with default rpc port
-        self._stopped = asyncio.Event()
-        self.running = asyncio.Event()
-
-    @property
-    def stopped(self):
-        return not self.running.is_set()
-
-    @property
-    def exists(self):
-        return (
-            os.path.exists(self.cli_bin) and
-            os.path.exists(self.daemon_bin)
-        )
-
-    def download(self):
-        downloaded_file = os.path.join(
-            self.bin_dir,
-            self.latest_release_url[self.latest_release_url.rfind('/')+1:]
-        )
-
-        if not os.path.exists(self.bin_dir):
-            os.mkdir(self.bin_dir)
-
-        if not os.path.exists(downloaded_file):
-            self.log.info('Downloading: %s', self.latest_release_url)
-            with urllib.request.urlopen(self.latest_release_url) as response:
-                with open(downloaded_file, 'wb') as out_file:
-                    shutil.copyfileobj(response, out_file)
-
-        self.log.info('Extracting: %s', downloaded_file)
-
-        if downloaded_file.endswith('.zip'):
-            with zipfile.ZipFile(downloaded_file) as dotzip:
-                dotzip.extractall(self.bin_dir)
-                # zipfile bug https://bugs.python.org/issue15795
-                os.chmod(self.cli_bin, 0o755)
-                os.chmod(self.daemon_bin, 0o755)
-
-        elif downloaded_file.endswith('.tar.gz'):
-            with tarfile.open(downloaded_file) as tar:
-                tar.extractall(self.bin_dir)
-
-        os.chmod(self.daemon_bin, 0o755)
-
-        return self.exists
-
-    def ensure(self):
-        return self.exists or self.download()
-
-    async def start(self):
-        assert self.ensure()
-        loop = asyncio.get_event_loop()
-        asyncio.get_child_watcher().attach_loop(loop)
-        command = [
-            self.daemon_bin, 'serve', '--esindex', self.spv_node.index_name + 'claims', '--debug'
-        ]
-        self.log.info(' '.join(command))
-        self.protocol = HubProcess(self.running, self._stopped)
-        try:
-            self.transport, _ = await loop.subprocess_exec(
-                lambda: self.protocol, *command
-            )
-            self.protocol.transport = self.transport
-        except Exception as e:
-            log.exception('failed to start go hub', exc_info=e)
-            raise e
-        await self.protocol.ready.wait()
-
-    async def stop(self, cleanup=True):
-        try:
-            if self.protocol:
-                await self.protocol.stop()
-        except Exception as e:
-            log.exception('failed to stop go hub', exc_info=e)
-            raise e
-        finally:
-            if cleanup:
-                self.cleanup()
-
-    def cleanup(self):
-        pass

lbry/wallet/rpc/session.py

@@ -395,8 +395,8 @@ class RPCSession(SessionBase):
         namespace=NAMESPACE, labelnames=("version",)
     )
 
-    def __init__(self, *, framer=None, loop=None, connection=None):
-        super().__init__(framer=framer, loop=loop)
+    def __init__(self, *, framer=None, connection=None):
+        super().__init__(framer=framer)
         self.connection = connection or self.default_connection()
         self.client_version = 'unknown'
 

lbry/wallet/script.py

@@ -17,6 +17,7 @@ OP_HASH160 = 0xa9
 OP_EQUALVERIFY = 0x88
 OP_CHECKSIG = 0xac
 OP_CHECKMULTISIG = 0xae
+OP_CHECKLOCKTIMEVERIFY = 0xb1
 OP_EQUAL = 0x87
 OP_PUSHDATA1 = 0x4c
 OP_PUSHDATA2 = 0x4d
@@ -276,7 +277,7 @@ class Template:
             elif isinstance(opcode, PUSH_INTEGER):
                 data = values[opcode.name]
                 source.write_many(push_data(
-                    data.to_bytes((data.bit_length() + 7) // 8, byteorder='little')
+                    data.to_bytes((data.bit_length() + 8) // 8, byteorder='little', signed=True)
                 ))
             elif isinstance(opcode, PUSH_SUBSCRIPT):
                 data = values[opcode.name]
@@ -357,19 +358,27 @@ class InputScript(Script):
     REDEEM_PUBKEY_HASH = Template('pubkey_hash', (
         PUSH_SINGLE('signature'), PUSH_SINGLE('pubkey')
     ))
-    REDEEM_SCRIPT = Template('script', (
+    MULTI_SIG_SCRIPT = Template('multi_sig', (
         SMALL_INTEGER('signatures_count'), PUSH_MANY('pubkeys'), SMALL_INTEGER('pubkeys_count'),
         OP_CHECKMULTISIG
     ))
-    REDEEM_SCRIPT_HASH = Template('script_hash', (
-        OP_0, PUSH_MANY('signatures'), PUSH_SUBSCRIPT('script', REDEEM_SCRIPT)
+    REDEEM_SCRIPT_HASH_MULTI_SIG = Template('script_hash+multi_sig', (
+        OP_0, PUSH_MANY('signatures'), PUSH_SUBSCRIPT('script', MULTI_SIG_SCRIPT)
+    ))
+    TIME_LOCK_SCRIPT = Template('timelock', (
+        PUSH_INTEGER('height'), OP_CHECKLOCKTIMEVERIFY, OP_DROP,
+        # rest is identical to OutputScript.PAY_PUBKEY_HASH:
+        OP_DUP, OP_HASH160, PUSH_SINGLE('pubkey_hash'), OP_EQUALVERIFY, OP_CHECKSIG
+    ))
+    REDEEM_SCRIPT_HASH_TIME_LOCK = Template('script_hash+timelock', (
+        PUSH_SINGLE('signature'), PUSH_SINGLE('pubkey'), PUSH_SUBSCRIPT('script', TIME_LOCK_SCRIPT)
     ))
 
     templates = [
         REDEEM_PUBKEY,
         REDEEM_PUBKEY_HASH,
-        REDEEM_SCRIPT_HASH,
-        REDEEM_SCRIPT
+        REDEEM_SCRIPT_HASH_TIME_LOCK,
+        REDEEM_SCRIPT_HASH_MULTI_SIG,
     ]
 
     @classmethod
@@ -380,20 +389,38 @@ class InputScript(Script):
         })
 
     @classmethod
-    def redeem_script_hash(cls, signatures, pubkeys):
-        return cls(template=cls.REDEEM_SCRIPT_HASH, values={
+    def redeem_multi_sig_script_hash(cls, signatures, pubkeys):
+        return cls(template=cls.REDEEM_SCRIPT_HASH_MULTI_SIG, values={
             'signatures': signatures,
-            'script': cls.redeem_script(signatures, pubkeys)
+            'script': cls(template=cls.MULTI_SIG_SCRIPT, values={
+                'signatures_count': len(signatures),
+                'pubkeys': pubkeys,
+                'pubkeys_count': len(pubkeys)
+            })
         })
 
     @classmethod
-    def redeem_script(cls, signatures, pubkeys):
-        return cls(template=cls.REDEEM_SCRIPT, values={
-            'signatures_count': len(signatures),
-            'pubkeys': pubkeys,
-            'pubkeys_count': len(pubkeys)
+    def redeem_time_lock_script_hash(cls, signature, pubkey, height=None, pubkey_hash=None, script_source=None):
+        if height and pubkey_hash:
+            script = cls(template=cls.TIME_LOCK_SCRIPT, values={
+                'height': height,
+                'pubkey_hash': pubkey_hash
+            })
+        elif script_source:
+            script = cls(source=script_source, template=cls.TIME_LOCK_SCRIPT)
+            script.parse(script.template)
+        else:
+            raise ValueError("script_source or both height and pubkey_hash are required.")
+        return cls(template=cls.REDEEM_SCRIPT_HASH_TIME_LOCK, values={
+            'signature': signature,
+            'pubkey': pubkey,
+            'script': script
         })
 
+    @property
+    def is_script_hash(self):
+        return self.template.name.startswith('script_hash+')
+
 
 class OutputScript(Script):
 
@@ -460,21 +487,6 @@ class OutputScript(Script):
         UPDATE_CLAIM_OPCODES + PAY_SCRIPT_HASH.opcodes
     ))
 
-    SELL_SCRIPT = Template('sell_script', (
-        OP_VERIFY, OP_DROP, OP_DROP, OP_DROP, PUSH_INTEGER('price'), OP_PRICECHECK
-    ))
-    SELL_CLAIM = Template('sell_claim+pay_script_hash', (
-        OP_SELL_CLAIM, PUSH_SINGLE('claim_id'), PUSH_SUBSCRIPT('sell_script', SELL_SCRIPT),
-        PUSH_SUBSCRIPT('receive_script', InputScript.REDEEM_SCRIPT), OP_2DROP, OP_2DROP
-    ) + PAY_SCRIPT_HASH.opcodes)
-
-    BUY_CLAIM = Template('buy_claim+pay_script_hash', (
-        OP_BUY_CLAIM, PUSH_SINGLE('sell_id'),
-        PUSH_SINGLE('claim_id'), PUSH_SINGLE('claim_version'),
-        PUSH_SINGLE('owner_pubkey_hash'), PUSH_SINGLE('negotiation_signature'),
-        OP_2DROP, OP_2DROP, OP_2DROP,
-    ) + PAY_SCRIPT_HASH.opcodes)
-
     templates = [
         PAY_PUBKEY_FULL,
         PAY_PUBKEY_HASH,
@@ -489,8 +501,6 @@ class OutputScript(Script):
         SUPPORT_CLAIM_DATA_SCRIPT,
         UPDATE_CLAIM_PUBKEY,
         UPDATE_CLAIM_SCRIPT,
-        SELL_CLAIM, SELL_SCRIPT,
-        BUY_CLAIM,
     ]
 
     @classmethod
@@ -550,30 +560,6 @@ class OutputScript(Script):
             'pubkey_hash': pubkey_hash
         })
 
-    @classmethod
-    def sell_script(cls, price):
-        return cls(template=cls.SELL_SCRIPT, values={
-            'price': price,
-        })
-
-    @classmethod
-    def sell_claim(cls, claim_id, price, signatures, pubkeys):
-        return cls(template=cls.SELL_CLAIM, values={
-            'claim_id': claim_id,
-            'sell_script': OutputScript.sell_script(price),
-            'receive_script': InputScript.redeem_script(signatures, pubkeys)
-        })
-
-    @classmethod
-    def buy_claim(cls, sell_id, claim_id, claim_version, owner_pubkey_hash, negotiation_signature):
-        return cls(template=cls.BUY_CLAIM, values={
-            'sell_id': sell_id,
-            'claim_id': claim_id,
-            'claim_version': claim_version,
-            'owner_pubkey_hash': owner_pubkey_hash,
-            'negotiation_signature': negotiation_signature,
-        })
-
     @property
     def is_pay_pubkey_hash(self):
         return self.template.name.endswith('pay_pubkey_hash')
@@ -602,17 +588,6 @@ class OutputScript(Script):
     def is_support_claim_data(self):
         return self.template.name.startswith('support_claim+data+')
 
-    @property
-    def is_sell_claim(self):
-        return self.template.name.startswith('sell_claim+')
-
-    @property
-    def is_buy_claim(self):
-        return self.template.name.startswith('buy_claim+')
-
     @property
     def is_claim_involved(self):
-        return any((
-            self.is_claim_name, self.is_support_claim, self.is_update_claim,
-            self.is_sell_claim, self.is_buy_claim
-        ))
+        return any((self.is_claim_name, self.is_support_claim, self.is_update_claim))

lbry/wallet/transaction.py

@@ -7,18 +7,18 @@ from typing import List, Iterable, Optional, Tuple
 from lbry.error import InsufficientFundsError
 from lbry.crypto.hash import hash160, sha256
 from lbry.crypto.base58 import Base58
-from scribe.schema.url import normalize_name
-from scribe.schema.claim import Claim
-from scribe.schema.base import Signable
-from scribe.schema.purchase import Purchase
-from scribe.schema.support import Support
-from scribe.schema.bip32 import PrivateKey, PublicKey
+from lbry.schema.url import normalize_name
+from lbry.schema.claim import Claim
+from lbry.schema.base import Signable
+from lbry.schema.purchase import Purchase
+from lbry.schema.support import Support
 
 from .script import InputScript, OutputScript
 from .constants import COIN, DUST, NULL_HASH32
 from .bcd_data_stream import BCDataStream
 from .hash import TXRef, TXRefImmutable
 from .util import ReadOnlyList
+from .bip32 import PrivateKey, PublicKey
 
 if typing.TYPE_CHECKING:
     from lbry.wallet.account import Account
@@ -145,6 +145,14 @@ class Input(InputOutput):
         script = InputScript.redeem_pubkey_hash(cls.NULL_SIGNATURE, cls.NULL_PUBLIC_KEY)
         return cls(txo.ref, script)
 
+    @classmethod
+    def spend_time_lock(cls, txo: 'Output', script_source: bytes) -> 'Input':
+        """ Create an input to spend time lock script."""
+        script = InputScript.redeem_time_lock_script_hash(
+            cls.NULL_SIGNATURE, cls.NULL_PUBLIC_KEY, script_source=script_source
+        )
+        return cls(txo.ref, script)
+
     @property
     def amount(self) -> int:
         """ Amount this input adds to the transaction. """
@@ -710,8 +718,11 @@ class Transaction:
         stream.write_compact_size(len(self._inputs))
         for i, txin in enumerate(self._inputs):
             if signing_input == i:
-                assert txin.txo_ref.txo is not None
-                txin.serialize_to(stream, txin.txo_ref.txo.script.source)
+                if txin.script.is_script_hash:
+                    txin.serialize_to(stream, txin.script.values['script'].source)
+                else:
+                    assert txin.txo_ref.txo is not None
+                    txin.serialize_to(stream, txin.txo_ref.txo.script.source)
             else:
                 txin.serialize_to(stream, b'')
         self._serialize_outputs(stream)
@@ -854,16 +865,19 @@ class Transaction:
     def signature_hash_type(hash_type):
         return hash_type
 
-    async def sign(self, funding_accounts: Iterable['Account']):
+    async def sign(self, funding_accounts: Iterable['Account'], extra_keys: dict = None):
         self._reset()
         ledger, wallet = self.ensure_all_have_same_ledger_and_wallet(funding_accounts)
         for i, txi in enumerate(self._inputs):
             assert txi.script is not None
             assert txi.txo_ref.txo is not None
             txo_script = txi.txo_ref.txo.script
-            if txo_script.is_pay_pubkey_hash:
-                address = ledger.hash160_to_address(txo_script.values['pubkey_hash'])
-                private_key = await ledger.get_private_key_for_address(wallet, address)
+            if txo_script.is_pay_pubkey_hash or txo_script.is_pay_script_hash:
+                if 'pubkey_hash' in txo_script.values:
+                    address = ledger.hash160_to_address(txo_script.values.get('pubkey_hash', ''))
+                    private_key = await ledger.get_private_key_for_address(wallet, address)
+                else:
+                    private_key = next(iter(extra_keys.values()))
                 assert private_key is not None, 'Cannot find private key for signing output.'
                 tx = self._serialize_for_signature(i)
                 txi.script.values['signature'] = \
@@ -937,6 +951,15 @@ class Transaction:
         data = Output.add_purchase_data(Purchase(claim_id))
         return cls.create([], [payment, data], funding_accounts, change_account)
 
+    @classmethod
+    async def spend_time_lock(cls, time_locked_txo: Output, script: bytes, account: 'Account'):
+        txi = Input.spend_time_lock(time_locked_txo, script)
+        txi.sequence = 0xFFFFFFFE
+        tx = await cls.create([txi], [], [account], account, sign=False)
+        tx.locktime = txi.script.values['script'].values['height']
+        tx._reset()
+        return tx
+
     @property
     def my_inputs(self):
         for txi in self.inputs:

lbry/wallet/udp.py

@@ -4,7 +4,7 @@ from time import perf_counter
 import logging
 from typing import Optional, Tuple, NamedTuple
 from lbry.utils import LRUCache, is_valid_public_ipv4
-from scribe.schema.attrs import country_str_to_int, country_int_to_str
+from lbry.schema.attrs import country_str_to_int, country_int_to_str
 # from prometheus_client import Counter
 
 
@@ -23,7 +23,7 @@ class SPVPing(NamedTuple):
     pad_bytes: bytes
 
     def encode(self):
-        return struct.pack(b'!lB64s', *self)
+        return struct.pack(b'!lB64s', *self)  # pylint: disable=not-an-iterable
 
     @staticmethod
     def make() -> bytes:
@@ -49,7 +49,7 @@ class SPVPong(NamedTuple):
     country: int
 
     def encode(self):
-        return struct.pack(PONG_ENCODING, *self)
+        return struct.pack(PONG_ENCODING, *self)  # pylint: disable=not-an-iterable
 
     @staticmethod
     def encode_address(address: str):
@@ -175,11 +175,9 @@ class SPVServerStatusProtocol(asyncio.DatagramProtocol):
 
 class StatusServer:
     def __init__(self):
-        1/0
         self._protocol: Optional[SPVServerStatusProtocol] = None
 
     async def start(self, height: int, tip: bytes, country: str, interface: str, port: int, allow_lan: bool = False):
-        1/0
         if self.is_running:
             return
         loop = asyncio.get_event_loop()

lbry/wallet/usage_payment.py

@@ -2,6 +2,7 @@ import asyncio
 import logging
 
 from lbry.error import (
+    InsufficientFundsError,
     ServerPaymentFeeAboveMaxAllowedError,
     ServerPaymentInvalidAddressError,
     ServerPaymentWalletLockedError
@@ -24,41 +25,66 @@ class WalletServerPayer:
         self.max_fee = max_fee
         self._on_payment_controller = StreamController()
         self.on_payment = self._on_payment_controller.stream
-        self.on_payment.listen(None, on_error=lambda e: logging.warning(e.args[0]))
+        self.on_payment.listen(None, on_error=lambda e: log.warning(e.args[0]))
 
     async def pay(self):
+        while self.running:
+            try:
+                await self._pay()
+            except (asyncio.TimeoutError, ConnectionError):
+                if not self.running:
+                    break
+                delay = max(self.payment_period / 24, 10)
+                log.warning("Payement failed. Will retry after %g seconds.", delay)
+                asyncio.sleep(delay)
+            except BaseException as e:
+                if not isinstance(e, asyncio.CancelledError):
+                    log.exception("Unexpected exception. Payment task exiting early.")
+                self.running = False
+                raise
+
+    async def _pay(self):
         while self.running:
             await asyncio.sleep(self.payment_period)
-            features = await self.ledger.network.retriable_call(self.ledger.network.get_server_features)
+            features = await self.ledger.network.get_server_features()
+            log.debug("pay loop: received server features: %s", str(features))
             address = features['payment_address']
             amount = str(features['daily_fee'])
             if not address or not amount:
+                log.debug("pay loop: no address or no amount")
                 continue
 
             if not self.ledger.is_pubkey_address(address):
+                log.info("pay loop: address not pubkey")
                 self._on_payment_controller.add_error(ServerPaymentInvalidAddressError(address))
                 continue
 
             if self.wallet.is_locked:
+                log.info("pay loop: wallet is locked")
                 self._on_payment_controller.add_error(ServerPaymentWalletLockedError())
                 continue
 
             amount = lbc_to_dewies(features['daily_fee'])  # check that this is in lbc and not dewies
             limit = lbc_to_dewies(self.max_fee)
             if amount > limit:
+                log.info("pay loop: amount (%d) > limit (%d)", amount, limit)
                 self._on_payment_controller.add_error(
                     ServerPaymentFeeAboveMaxAllowedError(features['daily_fee'], self.max_fee)
                 )
                 continue
 
-            tx = await Transaction.create(
-                [],
-                [Output.pay_pubkey_hash(amount, self.ledger.address_to_hash160(address))],
-                self.wallet.get_accounts_or_all(None),
-                self.wallet.get_account_or_default(None)
-            )
+            try:
+                tx = await Transaction.create(
+                    [],
+                    [Output.pay_pubkey_hash(amount, self.ledger.address_to_hash160(address))],
+                    self.wallet.get_accounts_or_all(None),
+                    self.wallet.get_account_or_default(None)
+                )
+            except InsufficientFundsError:
+                self._on_payment_controller.add_error(InsufficientFundsError())
+                continue
 
-            await self.ledger.broadcast(tx)
+            await self.ledger.broadcast_or_release(tx, blocking=True)
             if self.analytics_manager:
                 await self.analytics_manager.send_credits_sent()
             self._on_payment_controller.add(tx)
@@ -70,7 +96,18 @@ class WalletServerPayer:
         self.wallet = wallet
         self.running = True
         self.task = asyncio.ensure_future(self.pay())
-        self.task.add_done_callback(lambda _: log.info("Stopping wallet server payments."))
+        self.task.add_done_callback(self._done_callback)
+
+    def _done_callback(self, f):
+        if f.cancelled():
+            reason = "Cancelled"
+        elif f.exception():
+            reason = f'Exception: {f.exception()}'
+        elif not self.running:
+            reason = "Stopped"
+        else:
+            reason = ""
+        log.info("Stopping wallet server payments. %s", reason)
 
     async def stop(self):
         if self.running:

lbry/wallet/wallet.py

@@ -10,6 +10,7 @@ from collections import UserDict
 from hashlib import sha256
 from operator import attrgetter
 from lbry.crypto.crypt import better_aes_encrypt, better_aes_decrypt
+from lbry.error import InvalidPasswordError
 from .account import Account
 
 if typing.TYPE_CHECKING:
@@ -138,6 +139,10 @@ class Wallet:
             'accounts': [a.to_dict(encrypt_password) for a in self.accounts]
         }
 
+    def to_json(self):
+        assert not self.is_locked, "Cannot serialize a wallet with locked/encrypted accounts."
+        return json.dumps(self.to_dict())
+
     def save(self):
         if self.preferences.get(ENCRYPT_ON_DISK, False):
             if self.encryption_password is not None:
@@ -164,21 +169,32 @@ class Wallet:
 
     def pack(self, password):
         assert not self.is_locked, "Cannot pack a wallet with locked/encrypted accounts."
-        new_data = json.dumps(self.to_dict())
-        new_data_compressed = zlib.compress(new_data.encode())
+        new_data_compressed = zlib.compress(self.to_json().encode())
         return better_aes_encrypt(password, new_data_compressed)
 
     @classmethod
     def unpack(cls, password, encrypted):
         decrypted = better_aes_decrypt(password, encrypted)
-        decompressed = zlib.decompress(decrypted)
+        try:
+            decompressed = zlib.decompress(decrypted)
+        except zlib.error as e:
+            if "incorrect header check" in e.args[0].lower():
+                raise InvalidPasswordError()
+            if "unknown compression method" in e.args[0].lower():
+                raise InvalidPasswordError()
+            if "invalid window size" in e.args[0].lower():
+                raise InvalidPasswordError()
+            raise
         return json.loads(decompressed)
 
     def merge(self, manager: 'WalletManager',
-              password: str, data: str) -> List['Account']:
+              password: str, data: str) -> (List['Account'], List['Account']):
         assert not self.is_locked, "Cannot sync apply on a locked wallet."
-        added_accounts = []
-        decrypted_data = self.unpack(password, data)
+        added_accounts, merged_accounts = [], []
+        if password is None:
+            decrypted_data = json.loads(data)
+        else:
+            decrypted_data = self.unpack(password, data)
         self.preferences.merge(decrypted_data.get('preferences', {}))
         for account_dict in decrypted_data['accounts']:
             ledger = manager.get_or_create_ledger(account_dict['ledger'])
@@ -191,10 +207,11 @@ class Wallet:
                     break
             if local_match is not None:
                 local_match.merge(account_dict)
+                merged_accounts.append(local_match)
             else:
                 new_account = Account.from_dict(ledger, self, account_dict)
                 added_accounts.append(new_account)
-        return added_accounts
+        return added_accounts, merged_accounts
 
     @property
     def is_locked(self) -> bool:
@@ -203,11 +220,12 @@ class Wallet:
                 return True
         return False
 
-    def unlock(self, password):
+    async def unlock(self, password):
         for account in self.accounts:
             if account.encrypted:
                 if not account.decrypt(password):
                     return False
+                await account.deterministic_channel_keys.ensure_cache_primed()
         self.encryption_password = password
         return True
 

scripts/dht_crawler.py

@@ -0,0 +1,520 @@
+import sys
+import datetime
+import logging
+import asyncio
+import os.path
+import random
+import time
+import typing
+from dataclasses import dataclass, astuple, replace
+
+from aiohttp import web
+from prometheus_client import Gauge, generate_latest as prom_generate_latest, Counter, Histogram
+
+import lbry.dht.error
+from lbry.dht.constants import generate_id
+from lbry.dht.node import Node
+from lbry.dht.peer import make_kademlia_peer, PeerManager, decode_tcp_peer_from_compact_address
+from lbry.dht.protocol.distance import Distance
+from lbry.dht.protocol.iterative_find import FindValueResponse, FindNodeResponse, FindResponse
+from lbry.extras.daemon.storage import SQLiteMixin
+from lbry.conf import Config
+from lbry.utils import resolve_host
+
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-4s %(name)s:%(lineno)d: %(message)s")
+log = logging.getLogger(__name__)
+
+
+class SDHashSamples:
+    def __init__(self, samples_file_path):
+        with open(samples_file_path, "rb") as sample_file:
+            self._samples = sample_file.read()
+        assert len(self._samples) % 48 == 0
+        self.size = len(self._samples) // 48
+
+    def read_samples(self, count=1):
+        for _ in range(count):
+            offset = 48 * random.randrange(0, self.size)
+            yield self._samples[offset:offset + 48]
+
+
+class PeerStorage(SQLiteMixin):
+    CREATE_TABLES_QUERY = """
+    PRAGMA JOURNAL_MODE=WAL;
+    CREATE TABLE IF NOT EXISTS peer (
+        peer_id INTEGER NOT NULL,
+        node_id VARCHAR(96),
+        address VARCHAR,
+        udp_port INTEGER,
+        tcp_port INTEGER,
+        first_online DATETIME,
+        errors INTEGER,
+        last_churn INTEGER,
+        added_on DATETIME NOT NULL,
+        last_check DATETIME,
+        last_seen DATETIME,
+        latency INTEGER,
+        PRIMARY KEY (peer_id)
+    );
+    CREATE TABLE IF NOT EXISTS connection (
+        from_peer_id INTEGER NOT NULL,
+        to_peer_id INTEGER NOT NULL,
+        PRIMARY KEY (from_peer_id, to_peer_id),
+        FOREIGN KEY(from_peer_id) REFERENCES peer (peer_id),
+        FOREIGN KEY(to_peer_id) REFERENCES peer (peer_id)
+    );
+"""
+
+    async def open(self):
+        await super().open()
+        self.db.writer_connection.row_factory = dict_row_factory
+
+    async def all_peers(self):
+        return [
+            DHTPeer(**peer) for peer in await self.db.execute_fetchall(
+                "select * from peer where latency > 0 or last_seen > datetime('now', '-1 hour')")
+        ]
+
+    async def save_peers(self, *peers):
+        log.info("Saving graph nodes (peers) to DB")
+        await self.db.executemany(
+            "INSERT OR REPLACE INTO peer("
+            "node_id, address, udp_port, tcp_port, first_online, errors, last_churn,"
+            "added_on, last_check, last_seen, latency, peer_id) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
+            [astuple(peer) for peer in peers]
+        )
+        log.info("Finished saving graph nodes (peers) to DB")
+
+    async def save_connections(self, connections_map):
+        log.info("Saving graph edges (connections) to DB")
+        await self.db.executemany(
+            "DELETE FROM connection WHERE from_peer_id = ?", [(key,) for key in connections_map])
+        for from_peer_id in connections_map:
+            await self.db.executemany(
+                "INSERT INTO connection(from_peer_id, to_peer_id) VALUES(?,?)",
+                [(from_peer_id, to_peer_id) for to_peer_id in connections_map[from_peer_id]])
+        log.info("Finished saving graph edges (connections) to DB")
+
+
+@dataclass(frozen=True)
+class DHTPeer:
+    node_id: str
+    address: str
+    udp_port: int
+    tcp_port: int = None
+    first_online: datetime.datetime = None
+    errors: int = None
+    last_churn: int = None
+    added_on: datetime.datetime = None
+    last_check: datetime.datetime = None
+    last_seen: datetime.datetime = None
+    latency: int = None
+    peer_id: int = None
+
+    @classmethod
+    def from_kad_peer(cls, peer, peer_id):
+        node_id = peer.node_id.hex() if peer.node_id else None
+        return DHTPeer(
+            node_id=node_id, address=peer.address, udp_port=peer.udp_port, tcp_port=peer.tcp_port,
+            peer_id=peer_id, added_on=datetime.datetime.utcnow())
+
+    def to_kad_peer(self):
+        node_id = bytes.fromhex(self.node_id) if self.node_id else None
+        return make_kademlia_peer(node_id, self.address, self.udp_port, self.tcp_port)
+
+
+def new_node(address="0.0.0.0", udp_port=0, node_id=None):
+    node_id = node_id or generate_id()
+    loop = asyncio.get_event_loop()
+    return Node(loop, PeerManager(loop), node_id, udp_port, udp_port, 3333, address)
+
+
+class Crawler:
+    unique_total_hosts_metric = Gauge(
+        "unique_total_hosts", "Number of unique hosts seen in the last interval", namespace="dht_crawler_node",
+    )
+    reachable_hosts_metric = Gauge(
+        "reachable_hosts", "Number of hosts that replied in the last interval", namespace="dht_crawler_node",
+    )
+    total_historic_hosts_metric = Gauge(
+        "history_total_hosts", "Number of hosts seen since first run.", namespace="dht_crawler_node",
+    )
+    pending_check_hosts_metric = Gauge(
+        "pending_hosts", "Number of hosts on queue to be checked.", namespace="dht_crawler_node",
+    )
+    hosts_with_errors_metric = Gauge(
+        "error_hosts", "Number of hosts that raised errors during contact.", namespace="dht_crawler_node",
+    )
+    ROUTING_TABLE_SIZE_HISTOGRAM_BUCKETS = tuple(map(float, range(100))) + (
+        500., 1000., 2000., float('inf')
+    )
+    connections_found_metric = Histogram(
+        "connections_found", "Number of hosts returned by the last successful contact.", namespace="dht_crawler_node",
+        buckets=ROUTING_TABLE_SIZE_HISTOGRAM_BUCKETS
+    )
+    known_connections_found_metric = Histogram(
+        "known_connections_found", "Number of already known hosts returned by last contact.",
+        namespace="dht_crawler_node", buckets=ROUTING_TABLE_SIZE_HISTOGRAM_BUCKETS
+    )
+    reachable_connections_found_metric = Histogram(
+        "reachable_connections_found", "Number of reachable known hosts returned by last contact.",
+        namespace="dht_crawler_node", buckets=ROUTING_TABLE_SIZE_HISTOGRAM_BUCKETS
+    )
+    LATENCY_HISTOGRAM_BUCKETS = (
+        0., 5., 10., 15., 30., 60., 120., 180., 240., 300., 600., 1200., 1800., 4000., 6000., float('inf')
+    )
+    host_latency_metric = Histogram(
+        "host_latency", "Time spent on the last request, in milliseconds.", namespace="dht_crawler_node",
+        buckets=LATENCY_HISTOGRAM_BUCKETS
+    )
+    probed_streams_metric = Counter(
+        "probed_streams", "Amount of streams probed.", namespace="dht_crawler_node",
+    )
+    announced_streams_metric = Counter(
+        "announced_streams", "Amount of streams where announcements were found.", namespace="dht_crawler_node",
+    )
+    working_streams_metric = Counter(
+        "working_streams", "Amount of streams with reachable hosts.", namespace="dht_crawler_node",
+    )
+
+    def __init__(self, db_path: str, sd_hash_samples: SDHashSamples):
+        self.node = new_node()
+        self.db = PeerStorage(db_path)
+        self.sd_hashes = sd_hash_samples
+        self._memory_peers = {}
+        self._reachable_by_node_id = {}
+        self._connections = {}
+
+    async def open(self):
+        await self.db.open()
+        self._memory_peers = {
+            (peer.address, peer.udp_port): peer for peer in await self.db.all_peers()
+        }
+        self.refresh_reachable_set()
+
+    def refresh_reachable_set(self):
+        self._reachable_by_node_id = {
+            bytes.fromhex(peer.node_id): peer for peer in self._memory_peers.values() if (peer.latency or 0) > 0
+        }
+
+    async def probe_files(self):
+        if not self.sd_hashes:
+            return
+        while True:
+            for sd_hash in self.sd_hashes.read_samples(10_000):
+                self.refresh_reachable_set()
+                distance = Distance(sd_hash)
+                node_ids = list(self._reachable_by_node_id.keys())
+                node_ids.sort(key=lambda node_id: distance(node_id))
+                k_closest = [self._reachable_by_node_id[node_id] for node_id in node_ids[:8]]
+                found = False
+                working = False
+                for response in asyncio.as_completed(
+                        [self.request_peers(peer.address, peer.udp_port, peer.node_id, sd_hash) for peer in k_closest]):
+                    response = await response
+                    if response and response.found:
+                        found = True
+                        blob_peers = []
+                        for compact_addr in response.found_compact_addresses:
+                            try:
+                                blob_peers.append(decode_tcp_peer_from_compact_address(compact_addr))
+                            except ValueError as e:
+                                log.error("Error decoding compact peers: %s", e)
+                        for blob_peer in blob_peers:
+                            response = await self.request_peers(blob_peer.address, blob_peer.tcp_port, blob_peer.node_id, sd_hash)
+                            if response:
+                                working = True
+                                log.info("Found responsive peer for %s: %s:%d(%d)",
+                                         sd_hash.hex()[:8], blob_peer.address,
+                                         blob_peer.udp_port or -1, blob_peer.tcp_port or -1)
+                            else:
+                                log.info("Found dead peer for %s: %s:%d(%d)",
+                                         sd_hash.hex()[:8], blob_peer.address,
+                                         blob_peer.udp_port or -1, blob_peer.tcp_port or -1)
+                self.probed_streams_metric.inc()
+                if found:
+                    self.announced_streams_metric.inc()
+                if working:
+                    self.working_streams_metric.inc()
+                log.info("Done querying stream %s for peers. Found: %s, working: %s", sd_hash.hex()[:8], found, working)
+                await asyncio.sleep(.5)
+
+    @property
+    def refresh_limit(self):
+        return datetime.datetime.utcnow() - datetime.timedelta(hours=1)
+
+    @property
+    def all_peers(self):
+        return [
+            peer for peer in self._memory_peers.values()
+            if (peer.last_seen and peer.last_seen > self.refresh_limit) or (peer.latency or 0) > 0
+        ]
+
+    @property
+    def active_peers_count(self):
+        return len(self.all_peers)
+
+    @property
+    def checked_peers_count(self):
+        return len([peer for peer in self.all_peers if peer.last_check and peer.last_check > self.refresh_limit])
+
+    @property
+    def unreachable_peers_count(self):
+        return len([peer for peer in self.all_peers
+                    if peer.last_check and peer.last_check > self.refresh_limit and not peer.latency])
+
+    @property
+    def peers_with_errors_count(self):
+        return len([peer for peer in self.all_peers if (peer.errors or 0) > 0])
+
+    def get_peers_needing_check(self):
+        to_check = [peer for peer in self.all_peers if peer.last_check is None or peer.last_check < self.refresh_limit]
+        return to_check
+
+    def remove_expired_peers(self):
+        for key, peer in list(self._memory_peers.items()):
+            if (peer.latency or 0) < 1 and peer.last_seen < self.refresh_limit:
+                del self._memory_peers[key]
+
+    def add_peers(self, *peers):
+        for peer in peers:
+            db_peer = self.get_from_peer(peer)
+            if db_peer and db_peer.node_id is None and peer.node_id is not None:
+                db_peer = replace(db_peer, node_id=peer.node_id.hex())
+            elif not db_peer:
+                db_peer = DHTPeer.from_kad_peer(peer, len(self._memory_peers) + 1)
+            db_peer = replace(db_peer, last_seen=datetime.datetime.utcnow())
+            self._memory_peers[(peer.address, peer.udp_port)] = db_peer
+
+    async def flush_to_db(self):
+        await self.db.save_peers(*self._memory_peers.values())
+        connections_to_save = self._connections
+        self._connections = {}
+        # await self.db.save_connections(connections_to_save)  heavy call
+        self.remove_expired_peers()
+
+    def get_from_peer(self, peer):
+        return self._memory_peers.get((peer.address, peer.udp_port), None)
+
+    def set_latency(self, peer, latency=None):
+        if latency:
+            self.host_latency_metric.observe(latency / 1_000_000.0)
+        db_peer = self.get_from_peer(peer)
+        if not db_peer:
+            return
+        db_peer = replace(db_peer, latency=latency)
+        if not db_peer.node_id and peer.node_id:
+            db_peer = replace(db_peer, node_id=peer.node_id.hex())
+        if db_peer.first_online and latency is None:
+            db_peer = replace(db_peer, last_churn=(datetime.datetime.utcnow() - db_peer.first_online).seconds)
+        elif latency is not None and db_peer.first_online is None:
+            db_peer = replace(db_peer, first_online=datetime.datetime.utcnow())
+        db_peer = replace(db_peer, last_check=datetime.datetime.utcnow())
+        self._memory_peers[(db_peer.address, db_peer.udp_port)] = db_peer
+
+    def inc_errors(self, peer):
+        db_peer = self.get_from_peer(peer)
+        self._memory_peers[(peer.address, peer.node_id)] = replace(db_peer, errors=(db_peer.errors or 0) + 1)
+
+    def associate_peers(self, peer, other_peers):
+        self._connections[self.get_from_peer(peer).peer_id] = [
+            self.get_from_peer(other_peer).peer_id for other_peer in other_peers]
+
+    async def request_peers(self, host, port, node_id, key=None) -> typing.Optional[FindResponse]:
+        key = key or node_id
+        peer = make_kademlia_peer(key, await resolve_host(host, port, 'udp'), port)
+        for attempt in range(3):
+            try:
+                req_start = time.perf_counter_ns()
+                if key == node_id:
+                    response = await self.node.protocol.get_rpc_peer(peer).find_node(key)
+                    response = FindNodeResponse(key, response)
+                    latency = time.perf_counter_ns() - req_start
+                    self.set_latency(peer, latency)
+                else:
+                    response = await self.node.protocol.get_rpc_peer(peer).find_value(key)
+                    response = FindValueResponse(key, response)
+                await asyncio.sleep(0.05)
+                return response
+            except asyncio.TimeoutError:
+                if key == node_id:
+                    self.set_latency(peer, None)
+                continue
+            except lbry.dht.error.TransportNotConnected:
+                log.info("Transport unavailable, waiting 1s to retry")
+                await asyncio.sleep(1)
+            except lbry.dht.error.RemoteException as e:
+                log.info('Peer errored: %s:%d attempt #%d - %s',
+                         host, port, (attempt + 1), str(e))
+                if key == node_id:
+                    self.inc_errors(peer)
+                    self.set_latency(peer, None)
+                continue
+
+    async def crawl_routing_table(self, host, port, node_id=None):
+        start = time.time()
+        log.debug("querying %s:%d", host, port)
+        address = await resolve_host(host, port, 'udp')
+        key = node_id or self.node.protocol.peer_manager.get_node_id_for_endpoint(address, port)
+        peer = make_kademlia_peer(key, address, port)
+        self.add_peers(peer)
+        if not key:
+            latency = None
+            for _ in range(3):
+                try:
+                    ping_start = time.perf_counter_ns()
+                    await self.node.protocol.get_rpc_peer(peer).ping()
+                    await asyncio.sleep(0.05)
+                    key = key or self.node.protocol.peer_manager.get_node_id_for_endpoint(address, port)
+                    peer = make_kademlia_peer(key, address, port)
+                    latency = time.perf_counter_ns() - ping_start
+                    break
+                except asyncio.TimeoutError:
+                    pass
+                except lbry.dht.error.RemoteException:
+                    self.inc_errors(peer)
+                    pass
+            self.set_latency(peer, latency if peer.node_id else None)
+            if not latency or not peer.node_id:
+                if latency and not peer.node_id:
+                    log.warning("No node id from %s:%d", host, port)
+                return set()
+        distance = Distance(key)
+        max_distance = int.from_bytes(bytes([0xff] * 48), 'big')
+        peers = set()
+        factor = 2048
+        for i in range(1000):
+            response = await self.request_peers(address, port, key)
+            new_peers = list(response.get_close_kademlia_peers(peer)) if response else None
+            if not new_peers:
+                break
+            new_peers.sort(key=lambda peer: distance(peer.node_id))
+            peers.update(new_peers)
+            far_key = new_peers[-1].node_id
+            if distance(far_key) <= distance(key):
+                current_distance = distance(key)
+                next_jump = current_distance + int(max_distance // factor)  # jump closer
+                factor /= 2
+                if factor > 8 and next_jump < max_distance:
+                    key = int.from_bytes(peer.node_id, 'big') ^ next_jump
+                    if key.bit_length() > 384:
+                        break
+                    key = key.to_bytes(48, 'big')
+                else:
+                    break
+            else:
+                key = far_key
+                factor = 2048
+        if peers:
+            log.info("Done querying %s:%d in %.2f seconds: %d peers found over %d requests.",
+                     host, port, (time.time() - start), len(peers), i)
+        if peers:
+            self.connections_found_metric.observe(len(peers))
+            known_peers = 0
+            reachable_connections = 0
+            for peer in peers:
+                known_peer = self.get_from_peer(peer)
+                known_peers += 1 if known_peer else 0
+                reachable_connections += 1 if known_peer and (known_peer.latency or 0) > 0 else 0
+            self.known_connections_found_metric.observe(known_peers)
+            self.reachable_connections_found_metric.observe(reachable_connections)
+        self.add_peers(*peers)
+        self.associate_peers(peer, peers)
+        return peers
+
+    async def process(self):
+        to_process = {}
+
+        def submit(_peer):
+            f = asyncio.ensure_future(
+                self.crawl_routing_table(_peer.address, _peer.udp_port, bytes.fromhex(_peer.node_id)))
+            to_process[_peer.peer_id] = f
+            f.add_done_callback(lambda _: to_process.pop(_peer.peer_id))
+
+        to_check = self.get_peers_needing_check()
+        last_flush = datetime.datetime.utcnow()
+        while True:
+            for peer in to_check[:200]:
+                if peer.peer_id not in to_process:
+                    submit(peer)
+                    await asyncio.sleep(.05)
+            await asyncio.sleep(0)
+            self.unique_total_hosts_metric.set(self.checked_peers_count)
+            self.reachable_hosts_metric.set(self.checked_peers_count - self.unreachable_peers_count)
+            self.total_historic_hosts_metric.set(len(self._memory_peers))
+            self.pending_check_hosts_metric.set(len(to_check))
+            self.hosts_with_errors_metric.set(self.peers_with_errors_count)
+            log.info("%d known, %d contacted recently, %d unreachable, %d error, %d processing, %d on queue",
+                     self.active_peers_count, self.checked_peers_count, self.unreachable_peers_count,
+                     self.peers_with_errors_count, len(to_process), len(to_check))
+            if to_process:
+                await asyncio.wait(to_process.values(), return_when=asyncio.FIRST_COMPLETED)
+            to_check = self.get_peers_needing_check()
+            if (datetime.datetime.utcnow() - last_flush).seconds > 60:
+                log.info("flushing to db")
+                await self.flush_to_db()
+                last_flush = datetime.datetime.utcnow()
+            while not to_check and not to_process:
+                port = self.node.listening_port.get_extra_info('socket').getsockname()[1]
+                self.node.stop()
+                await self.node.start_listening()
+                log.info("Idle, sleeping a minute. Port changed to %d", port)
+                await asyncio.sleep(60.0)
+                to_check = self.get_peers_needing_check()
+
+
+class SimpleMetrics:
+    def __init__(self, port):
+        self.prometheus_port = port
+
+    async def handle_metrics_get_request(self, _):
+        try:
+            return web.Response(
+                text=prom_generate_latest().decode(),
+                content_type='text/plain; version=0.0.4'
+            )
+        except Exception:
+            log.exception('could not generate prometheus data')
+            raise
+
+    async def start(self):
+        prom_app = web.Application()
+        prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
+        metrics_runner = web.AppRunner(prom_app)
+        await metrics_runner.setup()
+        prom_site = web.TCPSite(metrics_runner, "0.0.0.0", self.prometheus_port)
+        await prom_site.start()
+
+
+def dict_row_factory(cursor, row):
+    d = {}
+    for idx, col in enumerate(cursor.description):
+        if col[0] in ('added_on', 'first_online', 'last_seen', 'last_check'):
+            d[col[0]] = datetime.datetime.fromisoformat(row[idx]) if row[idx] else None
+        else:
+            d[col[0]] = row[idx]
+    return d
+
+
+async def test():
+    db_path = "/tmp/peers.db" if len(sys.argv) == 1 else sys.argv[-1]
+    asyncio.get_event_loop().set_debug(True)
+    metrics = SimpleMetrics('8080')
+    await metrics.start()
+    conf = Config()
+    hosting_samples = SDHashSamples("test.sample") if os.path.isfile("test.sample") else None
+    crawler = Crawler(db_path, hosting_samples)
+    await crawler.open()
+    await crawler.flush_to_db()
+    await crawler.node.start_listening()
+    if crawler.active_peers_count < 100:
+        probes = []
+        for (host, port) in conf.known_dht_nodes:
+            probes.append(asyncio.create_task(crawler.crawl_routing_table(host, port)))
+        await asyncio.gather(*probes)
+        await crawler.flush_to_db()
+    await asyncio.gather(crawler.process(), crawler.probe_files())
+
+if __name__ == '__main__':
+    asyncio.run(test())

scripts/dht_node.py

@@ -83,7 +83,7 @@ async def main(host: str, port: int, db_file_path: str, bootstrap_node: Optional
     await storage.open()
     node = Node(
         loop, PeerManager(loop), node_id, port, port, 3333, None,
-        storage=storage
+        storage=storage, is_bootstrap_node=True
     )
     if prometheus_port > 0:
         metrics = SimpleMetrics(prometheus_port, node if export else None)

scripts/example_update_es.py

@@ -1,46 +0,0 @@
-import asyncio
-from pprint import pprint
-
-from elasticsearch import AsyncElasticsearch
-from elasticsearch._async.helpers import async_scan, async_bulk
-
-DB = {}
-INDEX = 'claims'
-
-
-async def generate_support_amounts(client: AsyncElasticsearch):
-    async for doc in async_scan(client):
-        DB[doc['_id']] = doc['_source']['support_amount']
-        if len(DB) > 10:
-            break
-    pprint(DB)
-
-
-def generate_support_to_trending():
-    for claim_id, amount in DB.items():
-        yield {'doc': {"trending_mixed": amount}, '_id': claim_id, '_index': INDEX, '_op_type': 'update'}
-
-
-async def write_trending(client: AsyncElasticsearch):
-    await async_bulk(client, generate_support_to_trending())
-
-
-def get_client(host='localhost', port=9201):
-    hosts = [{'host': host, 'port': port}]
-    return AsyncElasticsearch(hosts, timeout=port)
-
-
-async def run():
-    client = get_client()
-    await generate_support_amounts(client)
-    await write_trending(client)
-    for claim_id, value in DB.items():
-        if value > 0:
-            break
-    doc = await client.get(INDEX, claim_id)
-    pprint(doc)
-    pprint(DB[claim_id])
-    await client.close()
-
-
-asyncio.get_event_loop().run_until_complete(run())

scripts/hook-libtorrent.py

@@ -0,0 +1,24 @@
+"""
+Hook for libtorrent.
+"""
+
+import os
+import glob
+import os.path
+from PyInstaller.utils.hooks import get_module_file_attribute
+from PyInstaller import compat
+
+
+def get_binaries():
+    if compat.is_win:
+        files = ('c:/Windows/System32/libssl-1_1-x64.dll', 'c:/Windows/System32/libcrypto-1_1-x64.dll')
+        for file in files:
+            if not os.path.isfile(file):
+                print(f"MISSING {file}")
+        return [(file, '.') for file in files]
+    return []
+
+
+binaries = get_binaries()
+for file in glob.glob(os.path.join(get_module_file_attribute('libtorrent'), 'libtorrent*pyd*')):
+    binaries.append((file, 'libtorrent'))

scripts/release.py

@@ -2,7 +2,7 @@ import os
 import re
 import io
 import sys
-import json
+import yaml
 import argparse
 import unittest
 from datetime import date
@@ -25,25 +25,17 @@ AREA_RENAME = {
 
 
 def get_github():
-    config_path = os.path.expanduser('~/.lbry-release-tool.json')
+    config_path = os.path.expanduser('~/.config/gh/hosts.yml')
     if os.path.exists(config_path):
         with open(config_path, 'r') as config_file:
-            config = json.load(config_file)
-            return github3.login(token=config['token'])
+            config = yaml.load(config_file, Loader=yaml.FullLoader)
+            return github3.login(token=config['github.com']['oauth_token'])
 
-    token = os.environ.get("GH_TOKEN")
-    if not token:
-        print('GitHub Credentials')
-        username = input('username: ')
-        password = getpass('password: ')
-        gh = github3.authorize(
-            username, password, ['repo'], 'lbry release tool',
-            two_factor_callback=lambda: input('Enter 2FA: ')
-        )
-        with open(config_path, 'w') as config_file:
-            json.dump({'token': gh.token}, config_file)
-        token = gh.token
-    return github3.login(token=token)
+    print('To run release tool you need to first login using the github cli:')
+    print('')
+    print('  $ gh auth login')
+    print('')
+    sys.exit(1)
 
 
 def get_labels(pr, prefix):

scripts/sd_hash_sampler.py

@@ -0,0 +1,44 @@
+import asyncio
+from typing import Iterable
+
+from lbry.extras.daemon.client import daemon_rpc
+from lbry.conf import Config
+conf = Config()
+
+
+async def sample_prefix(prefix: bytes):
+    result = await daemon_rpc(conf, "claim_search", sd_hash=prefix.hex(), page_size=50)
+    total_pages = result['total_pages']
+    print(total_pages)
+    sd_hashes = set()
+    for page in range(1, total_pages + 1):
+        if page > 1:
+            result = await daemon_rpc(conf, "claim_search", sd_hash=prefix.hex(), page=page, page_size=50)
+        for item in result['items']:
+            sd_hash = item.get('value', {}).get('source', {}).get('sd_hash')
+            if not sd_hash:
+                print('err', item)
+                continue
+            sd_hashes.add(sd_hash)
+        print('page', page, len(sd_hashes))
+    return sd_hashes
+
+
+def save_sample(name: str, samples: Iterable[str]):
+    with open(name, 'wb') as outfile:
+        for sample in samples:
+            outfile.write(bytes.fromhex(sample))
+        outfile.flush()
+        print(outfile.tell())
+
+
+async def main():
+    samples = set()
+    futs = [asyncio.ensure_future(sample_prefix(bytes([i]))) for i in range(256)]
+    for i, completed in enumerate(asyncio.as_completed(futs)):
+        samples.update(await completed)
+        print(i, len(samples))
+    print(save_sample("test.sample", samples))
+
+if __name__ == "__main__":
+    asyncio.run(main())