Revert "TEMP: Try python 3.8."

This reverts commit 8def4d5177.

.github/workflows/main.yml

@@ -1,79 +1,206 @@
 name: ci
-on: pull_request
+on: ["push", "pull_request", "workflow_dispatch"]
+
 jobs:
 
   lint:
     name: lint
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v1
-      - uses: actions/setup-python@v1
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         with:
-          python-version: '3.7'
-      - run: make install tools
+          python-version: '3.9'
+      - name: extract pip cache
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
+          restore-keys: ${{ runner.os }}-pip-
+      - run: pip install --user --upgrade pip wheel
+      - run: pip install -e .[lint]
       - run: make lint
 
   tests-unit:
     name: "tests / unit"
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os:
+          - ubuntu-20.04
+          - macos-latest
+          - windows-latest
+    runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v1
-      - uses: actions/setup-python@v1
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         with:
-          python-version: '3.7'
-      - run: make install tools
-      - working-directory: lbry
+          python-version: '3.9'
+      - name: set pip cache dir
+        shell: bash
+        run: echo "PIP_CACHE_DIR=$(pip cache dir)" >> $GITHUB_ENV
+      - name: extract pip cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.PIP_CACHE_DIR }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
+          restore-keys: ${{ runner.os }}-pip-
+      - id: os-name
+        uses: ASzc/change-string-case-action@v5
+        with:
+          string: ${{ runner.os }}
+      - run: python -m pip install --user --upgrade pip wheel
+      - if: startsWith(runner.os, 'linux')
+        run: pip install -e .[test]
+      - if: startsWith(runner.os, 'linux')
         env:
           HOME: /tmp
-        run: coverage run -p --source=lbry -m unittest discover -vv tests.unit
+        run: make test-unit-coverage
+      - if: startsWith(runner.os, 'linux') != true
+        run: pip install -e .[test]
+      - if: startsWith(runner.os, 'linux') != true
+        env:
+          HOME: /tmp
+        run: coverage run --source=lbry -m unittest tests/unit/test_conf.py
+      - name: submit coverage report
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COVERALLS_FLAG_NAME: tests-unit-${{ steps.os-name.outputs.lowercase }}
+          COVERALLS_PARALLEL: true
+        run: |
+          pip install coveralls
+          coveralls --service=github
 
   tests-integration:
     name: "tests / integration"
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         test:
           - datanetwork
           - blockchain
+          - claims
+          - takeovers
+          - transactions
           - other
     steps:
-      - uses: actions/checkout@v1
-      - uses: actions/setup-python@v1
+      - name: Configure sysctl limits
+        run: |
+          sudo swapoff -a
+          sudo sysctl -w vm.swappiness=1
+          sudo sysctl -w fs.file-max=262144
+          sudo sysctl -w vm.max_map_count=262144
+      - name: Runs Elasticsearch
+        uses: elastic/elastic-github-actions/elasticsearch@master
         with:
-          python-version: '3.7'
+          stack-version: 7.12.1
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
       - if: matrix.test == 'other'
-        run: sudo apt install -y --no-install-recommends ffmpeg
-      - run: pip install tox-travis
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends ffmpeg
+      - name: extract pip cache
+        uses: actions/cache@v3
+        with:
+          path: ./.tox
+          key: tox-integration-${{ matrix.test }}-${{ hashFiles('setup.py') }}
+          restore-keys: txo-integration-${{ matrix.test }}-
+      - run: pip install tox coverage coveralls
+      - if: matrix.test == 'claims'
+        run: rm -rf .tox
       - run: tox -e ${{ matrix.test }}
+      - name: submit coverage report
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COVERALLS_FLAG_NAME: tests-integration-${{ matrix.test }}
+          COVERALLS_PARALLEL: true
+        run: |
+          coverage combine tests
+          coveralls --service=github
+
+
+  coverage:
+    needs: ["tests-unit", "tests-integration"]
+    runs-on: ubuntu-20.04
+    steps:
+      - name: finalize coverage report submission
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          pip install coveralls
+          coveralls --service=github --finish
 
   build:
     needs: ["lint", "tests-unit", "tests-integration"]
-    name: "build"
+    name: "build / binary"
     strategy:
       matrix:
         os:
-          - ubuntu-latest
+          - ubuntu-20.04
           - macos-latest
           - windows-latest
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v1
-      - uses: actions/setup-python@v1
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         with:
-          python-version: '3.7'
-      - name: Setup
-        run: |
-          pip install pyinstaller
-          pip install -e .
-      - if: startsWith(matrix.os, 'windows') == false
+          python-version: '3.9'
+      - id: os-name
+        uses: ASzc/change-string-case-action@v5
+        with:
+          string: ${{ runner.os }}
+      - name: set pip cache dir
+        shell: bash
+        run: echo "PIP_CACHE_DIR=$(pip cache dir)" >> $GITHUB_ENV
+      - name: extract pip cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.PIP_CACHE_DIR }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
+          restore-keys: ${{ runner.os }}-pip-
+      - run: pip install pyinstaller==4.6
+      - run: pip install -e .
+      - if: startsWith(github.ref, 'refs/tags/v')
+        run: python docker/set_build.py
+      - if: startsWith(runner.os, 'linux') || startsWith(runner.os, 'mac')
         name: Build & Run (Unix)
         run: |
           pyinstaller --onefile --name lbrynet lbry/extras/cli.py
-          chmod +x dist/lbrynet
           dist/lbrynet --version
-      - if: startsWith(matrix.os, 'windows')
+      - if: startsWith(runner.os, 'windows')
         name: Build & Run (Windows)
         run: |
-          pip install pywin32
+          pip install pywin32==301
           pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico --onefile --name lbrynet lbry/extras/cli.py
           dist/lbrynet.exe --version
+      - uses: actions/upload-artifact@v3
+        with:
+          name: lbrynet-${{ steps.os-name.outputs.lowercase }}
+          path: dist/
+
+  release:
+    name: "release"
+    if: startsWith(github.ref, 'refs/tags/v')
+    needs: ["build"]
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/download-artifact@v2
+      - name: upload binaries
+        env:
+          GITHUB_TOKEN: ${{ secrets.RELEASE_API_TOKEN }}
+        run: |
+          pip install githubrelease
+          chmod +x lbrynet-macos/lbrynet
+          chmod +x lbrynet-linux/lbrynet
+          zip --junk-paths lbrynet-mac.zip lbrynet-macos/lbrynet
+          zip --junk-paths lbrynet-linux.zip lbrynet-linux/lbrynet
+          zip --junk-paths lbrynet-windows.zip lbrynet-windows/lbrynet.exe
+          ls -lh
+          githubrelease release lbryio/lbry-sdk info ${GITHUB_REF#refs/tags/}
+          githubrelease asset lbryio/lbry-sdk upload ${GITHUB_REF#refs/tags/} \
+            lbrynet-mac.zip lbrynet-linux.zip lbrynet-windows.zip
+          githubrelease release lbryio/lbry-sdk publish ${GITHUB_REF#refs/tags/}
+

.github/workflows/release.yml

@@ -0,0 +1,22 @@
+name: slack
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  release:
+    name: "slack notification"
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: LoveToKnow/slackify-markdown-action@v1.0.0
+        id: markdown
+        with:
+          text: "There is a new SDK release: ${{github.event.release.html_url}}\n${{ github.event.release.body }}"
+      - uses: slackapi/slack-github-action@v1.14.0
+        env:
+          CHANGELOG: '<!channel> ${{ steps.markdown.outputs.text }}'
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_RELEASE_BOT_WEBHOOK }}
+        with:
+          payload: '{"type": "mrkdwn", "text": ${{ toJSON(env.CHANGELOG) }} }'
+

.gitignore

@@ -6,12 +6,17 @@
 /.coverage*
 /lbry-venv
 /venv
+/lbry/blockchain
 
 lbry.egg-info
 __pycache__
 _trial_temp/
+trending*.log
 
-/tests/integration/blockchain/files
+/tests/integration/claims/files
 /tests/.coverage.*
 
-/lbry/wallet/bin
+/lbry/wallet/bin
+
+/.vscode
+/.gitignore

.gitlab-ci.yml

@@ -1,210 +0,0 @@
-default:
-  image: python:3.7
-
-
-#cache:
-#  directories:
-#    - $HOME/venv
-#    - $HOME/.cache/pip
-#    - $HOME/Library/Caches/pip
-#    - $HOME/Library/Caches/Homebrew
-#    - $TRAVIS_BUILD_DIR/.tox
-
-
-stages:
-  - test
-  - build
-  - assets
-  - release
-
-
-.tagged:
-  rules:
-    - if: '$CI_COMMIT_TAG =~ /^v[0-9\.]+$/'
-      when: on_success
-
-
-
-test:lint:
-  stage: test
-  script:
-    - make install tools
-    - make lint
-
-test:unit:
-  stage: test
-  script:
-    - make install tools
-    - HOME=/tmp coverage run -p --source=lbry -m unittest discover -vv tests.unit
-
-test:datanetwork-integration:
-  stage: test
-  script:
-    - pip install tox-travis
-    - tox -e datanetwork
-
-test:blockchain-integration:
-  stage: test
-  script:
-    - pip install tox-travis
-    - tox -e blockchain
-
-test:other-integration:
-  stage: test
-  script:
-    - apt-get update
-    - apt-get install -y --no-install-recommends ffmpeg
-    - pip install tox-travis
-    - tox -e other
-
-test:json-api:
-  stage: test
-  script:
-    - make install tools
-    - HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
-
-
-
-.build:
-  stage: build
-  artifacts:
-    expire_in: 1 day
-    paths:
-      - lbrynet-${OS}.zip
-  script:
-    - pip install --upgrade 'setuptools<45.0.0'
-    - pip install pyinstaller
-    - pip install -e .
-    - python3.7 docker/set_build.py # must come after lbry is installed because it imports lbry
-    - pyinstaller --onefile --name lbrynet lbry/extras/cli.py
-    - chmod +x dist/lbrynet
-    - zip --junk-paths ${CI_PROJECT_DIR}/lbrynet-${OS}.zip dist/lbrynet  # gitlab expects artifacts to be in $CI_PROJECT_DIR
-    - openssl dgst -sha256 ${CI_PROJECT_DIR}/lbrynet-${OS}.zip | egrep -o [0-9a-f]+$ # get sha256 of asset. works on mac and ubuntu
-    - dist/lbrynet --version
-
-build:linux:
-  extends: .build
-  image: ubuntu:16.04
-  variables:
-    OS: linux
-  before_script:
-    - apt-get update
-    - apt-get install -y --no-install-recommends software-properties-common zip curl build-essential
-    - add-apt-repository -y ppa:deadsnakes/ppa
-    - apt-get update
-    - apt-get install -y --no-install-recommends python3.7-dev
-    - python3.7 <(curl -q https://bootstrap.pypa.io/get-pip.py) # make sure we get pip with python3.7
-
-build:mac:
-  extends: .build
-  tags: [macos] # makes gitlab use the mac runner
-  variables:
-    OS: mac
-    GIT_DEPTH: 5
-    VENV: /tmp/gitlab-lbry-sdk-venv
-  before_script:
-#    - brew upgrade python || true
-    - python3 --version | grep -q '^Python 3\.7\.' # dont upgrade python on every run. just make sure we're on the right Python
-#    - pip3 install --user --upgrade pip virtualenv
-    - pip3 --version | grep -q '\(python 3\.7\)'
-    - virtualenv --python=python3.7 "${VENV}"
-    - source "${VENV}/bin/activate"
-  after_script:
-    - rm -rf "${VENV}"
-
-build:windows:
-  extends: .build
-  tags: [windows] # makes gitlab use the windows runner
-  variables:
-    OS: windows
-    GIT_DEPTH: 5
-  before_script:
-    - ./docker/install_choco.ps1
-    - choco install -y --x86 python3 7zip checksum
-    - python --version # | findstr /B "Python 3\.7\." # dont upgrade python on every run. just make sure we're on the right Python
-    - pip --version # | findstr /E '\(python 3\.7\)'
-    - pip install virtualenv pywin32
-    - virtualenv venv
-    - venv/Scripts/activate.ps1
-    - pip install pip==19.3.1; $true # $true ignores errors. need this to get the correct coincurve wheel. see commit notes for details.
-  after_script:
-    - rmdir -Recurse venv
-  script:
-    - pip install --upgrade 'setuptools<45.0.0'
-    - pip install pyinstaller==3.5
-    - pip install -e .
-    - python docker/set_build.py # must come after lbry is installed because it imports lbry
-    - pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico -F -n lbrynet lbry/extras/cli.py
-    - 7z a -tzip $env:CI_PROJECT_DIR/lbrynet-${OS}.zip ./dist/lbrynet.exe
-    - checksum --type=sha256 --file=$env:CI_PROJECT_DIR/lbrynet-${OS}.zip
-    - dist/lbrynet.exe --version
-
-
-
-# s3 = upload asset to s3 (build.lbry.io)
-.s3:
-  stage: assets
-  variables:
-    GIT_STRATEGY: none
-  script:
-    - "[ -f lbrynet-${OS}.zip ]" # check that asset exists before trying to upload
-    - pip install awscli
-    - S3_PATH="daemon/gitlab-build-${CI_PIPELINE_ID}_commit-${CI_COMMIT_SHA:0:7}$( if [ ! -z ${CI_COMMIT_TAG} ]; then echo _tag-${CI_COMMIT_TAG}; else echo _branch-${CI_COMMIT_REF_NAME}; fi )"
-    - AWS_ACCESS_KEY_ID=${ARTIFACTS_KEY} AWS_SECRET_ACCESS_KEY=${ARTIFACTS_SECRET} AWS_REGION=${ARTIFACTS_REGION}
-      aws s3 cp lbrynet-${OS}.zip s3://${ARTIFACTS_BUCKET}/${S3_PATH}/lbrynet-${OS}.zip
-
-s3:linux:
-  extends: .s3
-  variables: {OS: linux}
-  needs: ["build:linux"]
-
-s3:mac:
-  extends: .s3
-  variables: {OS: mac}
-  needs: ["build:mac"]
-
-s3:windows:
-  extends: .s3
-  variables: {OS: windows}
-  needs: ["build:windows"]
-
-
-
-# github = upload assets to github when there's a tagged release
-.github:
-  extends: .tagged
-  stage: assets
-  variables:
-    GIT_STRATEGY: none
-  script:
-    - "[ -f lbrynet-${OS}.zip ]" # check that asset exists before trying to upload. githubrelease won't error if its missing
-    - pip install githubrelease
-    - githubrelease --no-progress --github-token ${GITHUB_CI_USER_ACCESS_TOKEN} asset lbryio/lbry-sdk upload ${CI_COMMIT_TAG} lbrynet-${OS}.zip
-
-github:linux:
-  extends: .github
-  variables: {OS: linux}
-  needs: ["build:linux"]
-
-github:mac:
-  extends: .github
-  variables: {OS: mac}
-  needs: ["build:mac"]
-
-github:windows:
-  extends: .github
-  variables: {OS: windows}
-  needs: ["build:windows"]
-
-
-
-publish:
-  extends: .tagged
-  stage: release
-  variables:
-    GIT_STRATEGY: none
-  script:
-    - pip install githubrelease
-    - githubrelease --no-progress --github-token ${GITHUB_CI_USER_ACCESS_TOKEN} release lbryio/lbry-sdk publish ${CI_COMMIT_TAG}
-    - >
-      curl -X POST -H 'Content-type: application/json' --data '{"text":"<!channel> There is a new SDK release: https://github.com/lbryio/lbry-sdk/releases/tag/'"${CI_COMMIT_TAG}"'\n'"$(curl -s "https://api.github.com/repos/lbryio/lbry-sdk/releases/tags/${CI_COMMIT_TAG}" | egrep '\w*\"body\":' | cut -d':' -f 2- | tail -c +3 | head -c -2)"'", "channel":"tech"}' "$(echo ${SLACK_WEBHOOK_URL_BASE64} | base64 -d)"

INSTALL.md

@@ -9,20 +9,29 @@ Here's a video walkthrough of this setup, which is itself hosted by the LBRY net
 
 ## Prerequisites
 
-Running `lbrynet` from source requires Python 3.7 or higher. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/).
+Running `lbrynet` from source requires Python 3.7. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/).
 
-After installing python 3, you'll need to install some additional libraries depending on your operating system.
+After installing Python 3.7, you'll need to install some additional libraries depending on your operating system.
+
+Because of [issue #2769](https://github.com/lbryio/lbry-sdk/issues/2769)
+at the moment the `lbrynet` daemon will only work correctly with Python 3.7.
+If Python 3.8+ is used, the daemon will start but the RPC server
+may not accept messages, returning the following:
+```
+Could not connect to daemon. Are you sure it's running?
+```
 
 ### macOS
 
 macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
 
 These environment variables also need to be set:
-1. PYTHONUNBUFFERED=1
-2. EVENT_NOKQUEUE=1
+```
+PYTHONUNBUFFERED=1
+EVENT_NOKQUEUE=1
+```
 
 Remaining dependencies can then be installed by running:
-
 ```
 brew install python protobuf
 ```
@@ -31,14 +40,17 @@ Assistance installing Python3: https://docs.python-guide.org/starting/install3/o
 
 ### Linux
 
-On Ubuntu (16.04 minimum, we recommend 18.04), install the following:
-
+On Ubuntu (we recommend 18.04 or 20.04), install the following:
 ```
 sudo add-apt-repository ppa:deadsnakes/ppa
 sudo apt-get update
 sudo apt-get install build-essential python3.7 python3.7-dev git python3.7-venv libssl-dev python-protobuf
 ```
 
+The [deadsnakes PPA](https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa) provides Python 3.7
+for those Ubuntu distributions that no longer have it in their
+official repositories.
+
 On Raspbian, you will also need to install `python-pyparsing`.
 
 If you're running another Linux distro, install the equivalent of the above packages for your system.
@@ -47,62 +59,119 @@ If you're running another Linux distro, install the equivalent of the above pack
 
 ### Linux/Mac
 
-To install on Linux/Mac:
+Clone the repository:
+```bash
+git clone https://github.com/lbryio/lbry-sdk.git
+cd lbry-sdk
+```
 
- ```
- Clone the repository:
- $ git clone https://github.com/lbryio/lbry-sdk.git
- $ cd lbry-sdk
+Create a Python virtual environment for lbry-sdk:
+```bash
+python3.7 -m venv lbry-venv
+```
 
- Create a Python virtual environment for lbry-sdk:
- $ python3.7 -m venv lbry-venv
- 
- Activating lbry-sdk virtual environment:
- $ source lbry-venv/bin/activate
- 
- Make sure you're on Python 3.7+ (as the default Python in virtual environment):
- $ python --version
+Activate virtual environment:
+```bash
+source lbry-venv/bin/activate
+```
 
- Install packages:
- $ make install
+Make sure you're on Python 3.7+ as default in the virtual environment:
+```bash
+python --version
+```
 
- If you are on Linux and using PyCharm, generates initial configs:
- $ make idea
- ```
+Install packages:
+```bash
+make install
+```
 
-To verify your installation, `which lbrynet` should return a path inside of the `lbry-venv` folder created by the `python3.7 -m venv lbry-venv` command.
+If you are on Linux and using PyCharm, generates initial configs:
+```bash
+make idea
+```
+
+To verify your installation, `which lbrynet` should return a path inside
+of the `lbry-venv` folder.
+```bash
+(lbry-venv) $ which lbrynet
+/opt/lbry-sdk/lbry-venv/bin/lbrynet
+```
+
+To exit the virtual environment simply use the command `deactivate`.
 
 ### Windows
 
-To install on Windows:
+Clone the repository:
+```bash
+git clone https://github.com/lbryio/lbry-sdk.git
+cd lbry-sdk
+```
 
- ```
- Clone the repository:
- > git clone https://github.com/lbryio/lbry-sdk.git
- > cd lbry-sdk
+Create a Python virtual environment for lbry-sdk:
+```bash
+python -m venv lbry-venv
+```
 
- Create a Python virtual environment for lbry-sdk:
- > python -m venv lbry-venv
+Activate virtual environment:
+```bash
+lbry-venv\Scripts\activate
+```
 
- Activating lbry-sdk virtual environment:
- > lbry-venv\Scripts\activate
-
- Install packages:
- > pip install -e .
- ```
+Install packages:
+```bash
+pip install -e .
+```
 
 ## Run the tests
+### Elasticsearch
 
-To run the unit tests from the repo directory:
+For running integration tests, Elasticsearch is required to be available at localhost:9200/
 
- ```
- python -m unittest discover tests.unit
- ```
+The easiest way to start it is using docker with:
+```bash
+make elastic-docker
+```
+
+Alternative installation methods are available [at Elasticsearch website](https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html).
+
+To run the unit and integration tests from the repo directory:
+```
+python -m unittest discover tests.unit
+python -m unittest discover tests.integration
+```
 
 ## Usage
 
 To start the API server:
-    `lbrynet start`
+```
+lbrynet start
+```
 
+Whenever the code inside [lbry-sdk/lbry](./lbry)
+is modified we should run `make install` to recompile the `lbrynet`
+executable with the newest code.
+
+## Development
+
+When developing, remember to enter the environment,
+and if you wish start the server interactively.
+```bash
+$ source lbry-venv/bin/activate
+
+(lbry-venv) $ python lbry/extras/cli.py start
+```
+
+Parameters can be passed in the same way.
+```bash
+(lbry-venv) $ python lbry/extras/cli.py wallet balance
+```
+
+If a Python debugger (`pdb` or `ipdb`) is installed we can also start it
+in this way, set up break points, and step through the code.
+```bash
+(lbry-venv) $ pip install ipdb
+
+(lbry-venv) $ ipdb lbry/extras/cli.py
+```
 
 Happy hacking!

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2015-2020 LBRY Inc
+Copyright (c) 2015-2022 LBRY Inc
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
 "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,

Makefile

@@ -1,23 +1,26 @@
-.PHONY: install tools lint test idea
+.PHONY: install tools lint test test-unit test-unit-coverage test-integration idea
 
 install:
-	CFLAGS="-DSQLITE_MAX_VARIABLE_NUMBER=2500000" pip install -U https://github.com/rogerbinns/apsw/releases/download/3.30.1-r1/apsw-3.30.1-r1.zip \
-		--global-option=fetch \
-		--global-option=--version --global-option=3.30.1 --global-option=--all \
-		--global-option=build --global-option=--enable --global-option=fts5
 	pip install -e .
 
-tools:
-	pip install mypy==0.701
-	pip install coverage astroid pylint
-
 lint:
 	pylint --rcfile=setup.cfg lbry
 	#mypy --ignore-missing-imports lbry
 
-test:
+test: test-unit test-integration
+
+test-unit:
+	python -m unittest discover tests.unit
+
+test-unit-coverage:
+	coverage run --source=lbry -m unittest discover -vv tests.unit
+
+test-integration:
 	tox
 
 idea:
 	mkdir -p .idea
 	cp -r scripts/idea/* .idea
+
+elastic-docker:
+	docker run -d -v lbryhub:/usr/share/elasticsearch/data -p 9200:9200 -p 9300:9300 -e"ES_JAVA_OPTS=-Xms512m -Xmx512m" -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.12.1

README.md

@@ -1,10 +1,10 @@
-# <img src="https://raw.githubusercontent.com/lbryio/lbry-sdk/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [![Gitlab CI Badge](https://ci.lbry.tech/lbry/lbry-sdk/badges/master/pipeline.svg)](https://ci.lbry.tech/lbry/lbry-sdk)
+# <img src="https://raw.githubusercontent.com/lbryio/lbry-sdk/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [![build](https://github.com/lbryio/lbry-sdk/actions/workflows/main.yml/badge.svg)](https://github.com/lbryio/lbry-sdk/actions/workflows/main.yml) [![coverage](https://coveralls.io/repos/github/lbryio/lbry-sdk/badge.svg)](https://coveralls.io/github/lbryio/lbry-sdk)
 
 LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
 
 LBRY SDK for Python is currently the most fully featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components include:
 
- * Built on Python 3.7+ and `asyncio`.
+ * Built on Python 3.7 and `asyncio`.
  * Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbry.dht](https://github.com/lbryio/lbry-sdk/tree/master/lbry/dht)).
  * Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbry.blob_exchange](https://github.com/lbryio/lbry-sdk/tree/master/lbry/blob_exchange)).
  * Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbry.schema](https://github.com/lbryio/lbry-sdk/tree/master/lbry/schema)).
@@ -41,7 +41,7 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
 
 ## Security
 
-We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.com/faq/gpg-key) if you need it.
+We take security seriously. Please contact security@lbry.com regarding any security issues. [Our PGP key is here](https://lbry.com/faq/pgp-key) if you need it.
 
 ## Contact
 

SECURITY.md

@@ -0,0 +1,9 @@
+# Security Policy
+
+## Supported Versions
+
+While we are not at v1.0 yet, only the latest release will be supported.
+
+## Reporting a Vulnerability
+
+See https://lbry.com/faq/security

docker/Dockerfile.dht_node

@@ -0,0 +1,43 @@
+FROM debian:10-slim
+
+ARG user=lbry
+ARG projects_dir=/home/$user
+ARG db_dir=/database
+
+ARG DOCKER_TAG
+ARG DOCKER_COMMIT=docker
+ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT
+
+RUN apt-get update && \
+    apt-get -y --no-install-recommends install \
+      wget \
+      automake libtool \
+      tar unzip \
+      build-essential \
+      pkg-config \
+      libleveldb-dev \
+      python3.7 \
+      python3-dev \
+      python3-pip \
+      python3-wheel \
+      python3-setuptools && \
+    update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN groupadd -g 999 $user && useradd -m -u 999 -g $user $user
+
+COPY . $projects_dir
+RUN chown -R $user:$user $projects_dir
+RUN mkdir -p $db_dir
+RUN chown -R $user:$user $db_dir
+
+USER $user
+WORKDIR $projects_dir
+
+RUN python3 -m pip install -U setuptools pip
+RUN make install
+RUN python3 docker/set_build.py
+RUN rm ~/.cache -rf
+VOLUME $db_dir
+ENTRYPOINT ["python3", "scripts/dht_node.py"]
+

docker/Dockerfile.wallet_server

@@ -1,4 +1,4 @@
-FROM ubuntu:19.10
+FROM debian:10-slim
 
 ARG user=lbry
 ARG db_dir=/database
@@ -13,10 +13,14 @@ RUN apt-get update && \
       wget \
       tar unzip \
       build-essential \
-      python3 \
+      automake libtool \
+      pkg-config \
+      libleveldb-dev \
+      python3.7 \
       python3-dev \
       python3-pip \
       python3-wheel \
+      python3-cffi \
       python3-setuptools && \
     update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 && \
     rm -rf /var/lib/apt/lists/*

docker/Dockerfile.web

@@ -0,0 +1,45 @@
+FROM debian:10-slim
+
+ARG user=lbry
+ARG downloads_dir=/database
+ARG projects_dir=/home/$user
+
+ARG DOCKER_TAG
+ARG DOCKER_COMMIT=docker
+ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT
+
+RUN apt-get update && \
+    apt-get -y --no-install-recommends install \
+      wget \
+      automake libtool \
+      tar unzip \
+      build-essential \
+      pkg-config \
+      libleveldb-dev \
+      python3.7 \
+      python3-dev \
+      python3-pip \
+      python3-wheel \
+      python3-setuptools && \
+    update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN groupadd -g 999 $user && useradd -m -u 999 -g $user $user
+RUN mkdir -p $downloads_dir
+RUN chown -R $user:$user $downloads_dir
+
+COPY . $projects_dir
+RUN chown -R $user:$user $projects_dir
+
+USER $user
+WORKDIR $projects_dir
+
+RUN pip install uvloop
+RUN make install
+RUN python3 docker/set_build.py
+RUN rm ~/.cache -rf
+
+# entry point
+VOLUME $downloads_dir
+COPY ./docker/webconf.yaml /webconf.yaml
+ENTRYPOINT ["/home/lbry/.local/bin/lbrynet", "start", "--config=/webconf.yaml"]

docker/README.md

@@ -0,0 +1,9 @@
+### How to run with docker-compose
+1. Edit config file and after that fix permissions with
+```
+sudo chown -R 999:999 webconf.yaml
+```
+2. Start SDK with
+```
+docker-compose up -d
+```

docker/docker-compose-wallet-server.yml

@@ -1,34 +1,49 @@
 version: "3"
 
 volumes:
-  lbrycrd:
   wallet_server:
+  es01:
 
 services:
-  lbrycrd:
-    image: lbry/lbrycrd:${LBRYCRD_TAG:-latest-release}
-    restart: always
-    ports: # accessible from host
-      - "9246:9246" # rpc port
-    expose: # internal to docker network. also this doesn't do anything. its for documentation only.
-      - "9245" # node-to-node comms port
-    volumes:
-      - "lbrycrd:/data/.lbrycrd"
-    environment:
-      - RUN_MODE=default
-      - SNAPSHOT_URL=${LBRYCRD_SNAPSHOT_URL-https://lbry.com/snapshot/blockchain}
-      - RPC_ALLOW_IP=0.0.0.0/0
   wallet_server:
+    depends_on: 
+      - es01
     image: lbry/wallet-server:${WALLET_SERVER_TAG:-latest-release}
-    depends_on:
-      - lbrycrd
     restart: always
+    network_mode: host
     ports:
       - "50001:50001" # rpc port
-      - "50005:50005" # websocket port
-      #- "2112:2112"   # uncomment to enable prometheus
+      - "2112:2112"   # uncomment to enable prometheus
     volumes:
       - "wallet_server:/database"
     environment:
-      - SNAPSHOT_URL=${WALLET_SERVER_SNAPSHOT_URL-https://lbry.com/snapshot/wallet}
-      - DAEMON_URL=http://lbry:lbry@lbrycrd:9245
+      - DAEMON_URL=http://lbry:lbry@127.0.0.1:9245
+      - MAX_QUERY_WORKERS=4
+      - CACHE_MB=1024
+      - CACHE_ALL_TX_HASHES=
+      - CACHE_ALL_CLAIM_TXOS=
+      - MAX_SEND=1000000000000000000
+      - MAX_RECEIVE=1000000000000000000
+      - MAX_SESSIONS=100000
+      - HOST=0.0.0.0
+      - TCP_PORT=50001
+      - PROMETHEUS_PORT=2112
+      - FILTERING_CHANNEL_IDS=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8
+      - BLOCKING_CHANNEL_IDS=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6
+  es01:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0
+    container_name: es01
+    environment:
+      - node.name=es01
+      - discovery.type=single-node
+      - indices.query.bool.max_clause_count=8192
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms4g -Xmx4g"  # no more than 32, remember to disable swap
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - es01:/usr/share/elasticsearch/data
+    ports:
+      - 127.0.0.1:9200:9200

docker/docker-compose.yml

@@ -0,0 +1,9 @@
+version: '3'
+services:
+    websdk:
+        image: vshyba/websdk
+        ports:
+            - '5279:5279'
+            - '5280:5280'
+        volumes:
+            - ./webconf.yaml:/webconf.yaml

docker/set_build.py

@@ -20,7 +20,7 @@ def _check_and_set(d: dict, key: str, value: str):
 def main():
     build_info = {item: build_info_mod.__dict__[item] for item in dir(build_info_mod) if not item.startswith("__")}
 
-    commit_hash = os.getenv('DOCKER_COMMIT', os.getenv('CI_COMMIT_SHA', os.getenv('TRAVIS_COMMIT')))
+    commit_hash = os.getenv('DOCKER_COMMIT', os.getenv('GITHUB_SHA'))
     if commit_hash is None:
         raise ValueError("Commit hash not found in env vars")
     _check_and_set(build_info, "COMMIT_HASH", commit_hash[:6])
@@ -30,8 +30,10 @@ def main():
         _check_and_set(build_info, "DOCKER_TAG", docker_tag)
         _check_and_set(build_info, "BUILD", "docker")
     else:
-        ci_tag = os.getenv('CI_COMMIT_TAG', os.getenv('TRAVIS_TAG'))
-        _check_and_set(build_info, "BUILD", "release" if re.match(r'v\d+\.\d+\.\d+$', str(ci_tag)) else "qa")
+        if re.match(r'refs/tags/v\d+\.\d+\.\d+$', str(os.getenv('GITHUB_REF'))):
+            _check_and_set(build_info, "BUILD", "release")
+        else:
+            _check_and_set(build_info, "BUILD", "qa")
 
     log.debug("build info: %s", ", ".join([f"{k}={v}" for k, v in build_info.items()]))
     with open(build_info_mod.__file__, 'w') as f:

docker/wallet_server_entrypoint.sh

@@ -6,7 +6,7 @@ set -euo pipefail
 
 SNAPSHOT_URL="${SNAPSHOT_URL:-}" #off by default. latest snapshot at https://lbry.com/snapshot/wallet
 
-if [[ -n "$SNAPSHOT_URL" ]] && [[ ! -f /database/claims.db ]]; then
+if [[ -n "$SNAPSHOT_URL" ]] && [[ ! -f /database/lbry-leveldb ]]; then
   files="$(ls)"
   echo "Downloading wallet snapshot from $SNAPSHOT_URL"
   wget --no-verbose --trust-server-names --content-disposition "$SNAPSHOT_URL"
@@ -20,4 +20,6 @@ if [[ -n "$SNAPSHOT_URL" ]] && [[ ! -f /database/claims.db ]]; then
   rm "$filename"
 fi
 
-/home/lbry/.local/bin/torba-server "$@"
+/home/lbry/.local/bin/lbry-hub-elastic-sync
+echo 'starting server'
+/home/lbry/.local/bin/lbry-hub "$@"

docker/webconf.yaml

@@ -0,0 +1,9 @@
+allowed_origin: "*"
+max_key_fee: "0.0 USD"
+save_files: false
+save_blobs: false
+streaming_server: "0.0.0.0:5280"
+api: "0.0.0.0:5279"
+data_dir: /tmp
+download_dir: /tmp
+wallet_dir: /tmp

lbry/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.59.2"
+__version__ = "0.113.0"
 version = tuple(map(int, __version__.split('.')))  # pylint: disable=invalid-name

lbry/blob/blob_file.py

@@ -1,5 +1,6 @@
 import os
 import re
+import time
 import asyncio
 import binascii
 import logging
@@ -70,21 +71,27 @@ class AbstractBlob:
         'writers',
         'verified',
         'writing',
-        'readers'
+        'readers',
+        'added_on',
+        'is_mine',
     ]
 
-    def __init__(self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
-                 blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
-                 blob_directory: typing.Optional[str] = None):
+    def __init__(
+        self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
+        blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
+        blob_directory: typing.Optional[str] = None, added_on: typing.Optional[int] = None, is_mine: bool = False,
+    ):
         self.loop = loop
         self.blob_hash = blob_hash
         self.length = length
         self.blob_completed_callback = blob_completed_callback
         self.blob_directory = blob_directory
         self.writers: typing.Dict[typing.Tuple[typing.Optional[str], typing.Optional[int]], HashBlobWriter] = {}
-        self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
-        self.writing: asyncio.Event = asyncio.Event(loop=self.loop)
+        self.verified: asyncio.Event = asyncio.Event()
+        self.writing: asyncio.Event = asyncio.Event()
         self.readers: typing.List[typing.BinaryIO] = []
+        self.added_on = added_on or time.time()
+        self.is_mine = is_mine
 
         if not is_valid_blobhash(blob_hash):
             raise InvalidBlobHashError(blob_hash)
@@ -110,7 +117,7 @@ class AbstractBlob:
                 if reader in self.readers:
                     self.readers.remove(reader)
 
-    def _write_blob(self, blob_bytes: bytes):
+    def _write_blob(self, blob_bytes: bytes) -> asyncio.Task:
         raise NotImplementedError()
 
     def set_length(self, length) -> None:
@@ -180,35 +187,42 @@ class AbstractBlob:
 
     @classmethod
     async def create_from_unencrypted(
-            cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
-            unencrypted: bytes, blob_num: int,
-            blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None) -> BlobInfo:
+        cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
+        unencrypted: bytes, blob_num: int, added_on: int, is_mine: bool,
+        blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None,
+    ) -> BlobInfo:
         """
         Create an encrypted BlobFile from plaintext bytes
         """
 
         blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted)
         length = len(blob_bytes)
-        blob = cls(loop, blob_hash, length, blob_completed_callback, blob_dir)
+        blob = cls(loop, blob_hash, length, blob_completed_callback, blob_dir, added_on, is_mine)
         writer = blob.get_blob_writer()
         writer.write(blob_bytes)
         await blob.verified.wait()
-        return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash)
+        return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), added_on, blob_hash, is_mine)
 
     def save_verified_blob(self, verified_bytes: bytes):
         if self.verified.is_set():
             return
-        if self.is_writeable():
-            self._write_blob(verified_bytes)
+
+        def update_events(_):
             self.verified.set()
+            self.writing.clear()
+
+        if self.is_writeable():
+            self.writing.set()
+            task = self._write_blob(verified_bytes)
+            task.add_done_callback(update_events)
             if self.blob_completed_callback:
-                self.blob_completed_callback(self)
+                task.add_done_callback(lambda _: self.blob_completed_callback(self))
 
     def get_blob_writer(self, peer_address: typing.Optional[str] = None,
                         peer_port: typing.Optional[int] = None) -> HashBlobWriter:
         if (peer_address, peer_port) in self.writers and not self.writers[(peer_address, peer_port)].closed():
             raise OSError(f"attempted to download blob twice from {peer_address}:{peer_port}")
-        fut = asyncio.Future(loop=self.loop)
+        fut = asyncio.Future()
         writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
         self.writers[(peer_address, peer_port)] = writer
 
@@ -242,11 +256,13 @@ class BlobBuffer(AbstractBlob):
     """
     An in-memory only blob
     """
-    def __init__(self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
-                 blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
-                 blob_directory: typing.Optional[str] = None):
+    def __init__(
+        self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
+        blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
+        blob_directory: typing.Optional[str] = None, added_on: typing.Optional[int] = None, is_mine: bool = False
+    ):
         self._verified_bytes: typing.Optional[BytesIO] = None
-        super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
+        super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory, added_on, is_mine)
 
     @contextlib.contextmanager
     def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
@@ -261,9 +277,11 @@ class BlobBuffer(AbstractBlob):
             self.verified.clear()
 
     def _write_blob(self, blob_bytes: bytes):
-        if self._verified_bytes:
-            raise OSError("already have bytes for blob")
-        self._verified_bytes = BytesIO(blob_bytes)
+        async def write():
+            if self._verified_bytes:
+                raise OSError("already have bytes for blob")
+            self._verified_bytes = BytesIO(blob_bytes)
+        return self.loop.create_task(write())
 
     def delete(self):
         if self._verified_bytes:
@@ -281,10 +299,12 @@ class BlobFile(AbstractBlob):
     """
     A blob existing on the local file system
     """
-    def __init__(self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
-                 blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
-                 blob_directory: typing.Optional[str] = None):
-        super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
+    def __init__(
+        self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
+        blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
+        blob_directory: typing.Optional[str] = None, added_on: typing.Optional[int] = None, is_mine: bool = False
+    ):
+        super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory, added_on, is_mine)
         if not blob_directory or not os.path.isdir(blob_directory):
             raise OSError(f"invalid blob directory '{blob_directory}'")
         self.file_path = os.path.join(self.blob_directory, self.blob_hash)
@@ -319,22 +339,28 @@ class BlobFile(AbstractBlob):
             handle.close()
 
     def _write_blob(self, blob_bytes: bytes):
-        with open(self.file_path, 'wb') as f:
-            f.write(blob_bytes)
+        def _write_blob():
+            with open(self.file_path, 'wb') as f:
+                f.write(blob_bytes)
+
+        async def write_blob():
+            await self.loop.run_in_executor(None, _write_blob)
+
+        return self.loop.create_task(write_blob())
 
     def delete(self):
+        super().delete()
         if os.path.isfile(self.file_path):
             os.remove(self.file_path)
-        return super().delete()
 
     @classmethod
     async def create_from_unencrypted(
-            cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
-            unencrypted: bytes, blob_num: int,
-            blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'],
-                                                                     asyncio.Task]] = None) -> BlobInfo:
+        cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
+        unencrypted: bytes, blob_num: int, added_on: float, is_mine: bool,
+        blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None
+    ) -> BlobInfo:
         if not blob_dir or not os.path.isdir(blob_dir):
             raise OSError(f"cannot create blob in directory: '{blob_dir}'")
         return await super().create_from_unencrypted(
-            loop, blob_dir, key, iv, unencrypted, blob_num, blob_completed_callback
+            loop, blob_dir, key, iv, unencrypted, blob_num, added_on, is_mine, blob_completed_callback
         )

lbry/blob/blob_info.py

@@ -7,13 +7,19 @@ class BlobInfo:
         'blob_num',
         'length',
         'iv',
+        'added_on',
+        'is_mine'
     ]
 
-    def __init__(self, blob_num: int, length: int, iv: str, blob_hash: typing.Optional[str] = None):
+    def __init__(
+            self, blob_num: int, length: int, iv: str, added_on,
+             blob_hash: typing.Optional[str] = None, is_mine=False):
         self.blob_hash = blob_hash
         self.blob_num = blob_num
         self.length = length
         self.iv = iv
+        self.added_on = added_on
+        self.is_mine = is_mine
 
     def as_dict(self) -> typing.Dict:
         d = {

lbry/blob/blob_manager.py

@@ -2,7 +2,7 @@ import os
 import typing
 import asyncio
 import logging
-from lbry.utils import LRUCache
+from lbry.utils import LRUCacheWithMetrics
 from lbry.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
 from lbry.stream.descriptor import StreamDescriptor
 from lbry.connection_manager import ConnectionManager
@@ -32,34 +32,34 @@ class BlobManager:
             else self._node_data_store.completed_blobs
         self.blobs: typing.Dict[str, AbstractBlob] = {}
         self.config = config
-        self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCache(
+        self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCacheWithMetrics(
             self.config.blob_lru_cache_size)
         self.connection_manager = ConnectionManager(loop)
 
-    def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None):
+    def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None, is_mine: bool = False):
         if self.config.save_blobs or (
                 is_valid_blobhash(blob_hash) and os.path.isfile(os.path.join(self.blob_dir, blob_hash))):
             return BlobFile(
-                self.loop, blob_hash, length, self.blob_completed, self.blob_dir
+                self.loop, blob_hash, length, self.blob_completed, self.blob_dir, is_mine=is_mine
             )
         return BlobBuffer(
-            self.loop, blob_hash, length, self.blob_completed, self.blob_dir
+            self.loop, blob_hash, length, self.blob_completed, self.blob_dir, is_mine=is_mine
         )
 
-    def get_blob(self, blob_hash, length: typing.Optional[int] = None):
+    def get_blob(self, blob_hash, length: typing.Optional[int] = None, is_mine: bool = False):
         if blob_hash in self.blobs:
             if self.config.save_blobs and isinstance(self.blobs[blob_hash], BlobBuffer):
                 buffer = self.blobs.pop(blob_hash)
                 if blob_hash in self.completed_blob_hashes:
                     self.completed_blob_hashes.remove(blob_hash)
-                self.blobs[blob_hash] = self._get_blob(blob_hash, length)
+                self.blobs[blob_hash] = self._get_blob(blob_hash, length, is_mine)
                 if buffer.is_readable():
                     with buffer.reader_context() as reader:
                         self.blobs[blob_hash].write_blob(reader.read())
             if length and self.blobs[blob_hash].length is None:
                 self.blobs[blob_hash].set_length(length)
         else:
-            self.blobs[blob_hash] = self._get_blob(blob_hash, length)
+            self.blobs[blob_hash] = self._get_blob(blob_hash, length, is_mine)
         return self.blobs[blob_hash]
 
     def is_blob_verified(self, blob_hash: str, length: typing.Optional[int] = None) -> bool:
@@ -83,6 +83,8 @@ class BlobManager:
         to_add = await self.storage.sync_missing_blobs(in_blobfiles_dir)
         if to_add:
             self.completed_blob_hashes.update(to_add)
+        # check blobs that aren't set as finished but were seen on disk
+        await self.ensure_completed_blobs_status(in_blobfiles_dir - to_add)
         if self.config.track_bandwidth:
             self.connection_manager.start()
         return True
@@ -105,13 +107,26 @@ class BlobManager:
         if isinstance(blob, BlobFile):
             if blob.blob_hash not in self.completed_blob_hashes:
                 self.completed_blob_hashes.add(blob.blob_hash)
-            return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=True))
+            return self.loop.create_task(self.storage.add_blobs(
+                (blob.blob_hash, blob.length, blob.added_on, blob.is_mine), finished=True)
+            )
         else:
-            return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=False))
+            return self.loop.create_task(self.storage.add_blobs(
+                (blob.blob_hash, blob.length, blob.added_on, blob.is_mine), finished=False)
+            )
 
-    def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]:
-        """Returns of the blobhashes_to_check, which are valid"""
-        return [blob_hash for blob_hash in blob_hashes if self.is_blob_verified(blob_hash)]
+    async def ensure_completed_blobs_status(self, blob_hashes: typing.Iterable[str]):
+        """Ensures that completed blobs from a given list of blob hashes are set as 'finished' in the database."""
+        to_add = []
+        for blob_hash in blob_hashes:
+            if not self.is_blob_verified(blob_hash):
+                continue
+            blob = self.get_blob(blob_hash)
+            to_add.append((blob.blob_hash, blob.length, blob.added_on, blob.is_mine))
+            if len(to_add) > 500:
+                await self.storage.add_blobs(*to_add, finished=True)
+                to_add.clear()
+        return await self.storage.add_blobs(*to_add, finished=True)
 
     def delete_blob(self, blob_hash: str):
         if not is_valid_blobhash(blob_hash):

lbry/blob/disk_space_manager.py

@@ -0,0 +1,77 @@
+import asyncio
+import logging
+
+log = logging.getLogger(__name__)
+
+
+class DiskSpaceManager:
+
+    def __init__(self, config, db, blob_manager, cleaning_interval=30 * 60, analytics=None):
+        self.config = config
+        self.db = db
+        self.blob_manager = blob_manager
+        self.cleaning_interval = cleaning_interval
+        self.running = False
+        self.task = None
+        self.analytics = analytics
+        self._used_space_bytes = None
+
+    async def get_free_space_mb(self, is_network_blob=False):
+        limit_mb = self.config.network_storage_limit if is_network_blob else self.config.blob_storage_limit
+        space_used_mb = await self.get_space_used_mb()
+        space_used_mb = space_used_mb['network_storage'] if is_network_blob else space_used_mb['content_storage']
+        return max(0, limit_mb - space_used_mb)
+
+    async def get_space_used_bytes(self):
+        self._used_space_bytes = await self.db.get_stored_blob_disk_usage()
+        return self._used_space_bytes
+
+    async def get_space_used_mb(self, cached=True):
+        cached = cached and self._used_space_bytes is not None
+        space_used_bytes = self._used_space_bytes if cached else await self.get_space_used_bytes()
+        return {key: int(value/1024.0/1024.0) for key, value in space_used_bytes.items()}
+
+    async def clean(self):
+        await self._clean(False)
+        await self._clean(True)
+
+    async def _clean(self, is_network_blob=False):
+        space_used_mb = await self.get_space_used_mb(cached=False)
+        if is_network_blob:
+            space_used_mb = space_used_mb['network_storage']
+        else:
+            space_used_mb = space_used_mb['content_storage'] + space_used_mb['private_storage']
+        storage_limit_mb = self.config.network_storage_limit if is_network_blob else self.config.blob_storage_limit
+        if self.analytics:
+            asyncio.create_task(
+                self.analytics.send_disk_space_used(space_used_mb, storage_limit_mb, is_network_blob)
+            )
+        delete = []
+        available = storage_limit_mb - space_used_mb
+        if storage_limit_mb == 0 if not is_network_blob else available >= 0:
+            return 0
+        for blob_hash, file_size, _ in await self.db.get_stored_blobs(is_mine=False, is_network_blob=is_network_blob):
+            delete.append(blob_hash)
+            available += int(file_size/1024.0/1024.0)
+            if available >= 0:
+                break
+        if delete:
+            await self.db.stop_all_files()
+            await self.blob_manager.delete_blobs(delete, delete_from_db=True)
+        self._used_space_bytes = None
+        return len(delete)
+
+    async def cleaning_loop(self):
+        while self.running:
+            await asyncio.sleep(self.cleaning_interval)
+            await self.clean()
+
+    async def start(self):
+        self.running = True
+        self.task = asyncio.create_task(self.cleaning_loop())
+        self.task.add_done_callback(lambda _: log.info("Stopping blob cleanup service."))
+
+    async def stop(self):
+        if self.running:
+            self.running = False
+            self.task.cancel()

lbry/blob_exchange/client.py

@@ -32,7 +32,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
         self.buf = b''
 
         # this is here to handle the race when the downloader is closed right as response_fut gets a result
-        self.closed = asyncio.Event(loop=self.loop)
+        self.closed = asyncio.Event()
 
     def data_received(self, data: bytes):
         if self.connection_manager:
@@ -111,7 +111,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
             self.transport.write(msg)
             if self.connection_manager:
                 self.connection_manager.sent_data(f"{self.peer_address}:{self.peer_port}", len(msg))
-            response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop)
+            response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout)
             availability_response = response.get_availability_response()
             price_response = response.get_price_response()
             blob_response = response.get_blob_response()
@@ -151,7 +151,9 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
                 f" timeout in {self.peer_timeout}"
             log.debug(msg)
             msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
-            await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
+            await asyncio.wait_for(self.writer.finished, self.peer_timeout)
+            # wait for the io to finish
+            await self.blob.verified.wait()
             log.info("%s at %fMB/s", msg,
                      round((float(self._blob_bytes_received) /
                             float(time.perf_counter() - start_time)) / 1000000.0, 2))
@@ -185,7 +187,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
         try:
             self._blob_bytes_received = 0
             self.blob, self.writer = blob, blob.get_blob_writer(self.peer_address, self.peer_port)
-            self._response_fut = asyncio.Future(loop=self.loop)
+            self._response_fut = asyncio.Future()
             return await self._download_blob()
         except OSError:
             # i'm not sure how to fix this race condition - jack
@@ -242,7 +244,7 @@ async def request_blob(loop: asyncio.AbstractEventLoop, blob: Optional['Abstract
     try:
         if not connected_protocol:
             await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
-                                   peer_connect_timeout, loop=loop)
+                                   peer_connect_timeout)
             connected_protocol = protocol
         if blob is None or blob.get_is_verified() or not blob.is_writeable():
             # blob is None happens when we are just opening a connection

lbry/blob_exchange/downloader.py

@@ -3,6 +3,7 @@ import typing
 import logging
 from lbry.utils import cache_concurrent
 from lbry.blob_exchange.client import request_blob
+from lbry.dht.node import get_kademlia_peers_from_hosts
 if typing.TYPE_CHECKING:
     from lbry.conf import Config
     from lbry.dht.node import Node
@@ -29,7 +30,7 @@ class BlobDownloader:
         self.failures: typing.Dict['KademliaPeer', int] = {}
         self.connection_failures: typing.Set['KademliaPeer'] = set()
         self.connections: typing.Dict['KademliaPeer', 'BlobExchangeClientProtocol'] = {}
-        self.is_running = asyncio.Event(loop=self.loop)
+        self.is_running = asyncio.Event()
 
     def should_race_continue(self, blob: 'AbstractBlob'):
         max_probes = self.config.max_connections_per_download * (1 if self.connections else 10)
@@ -63,8 +64,8 @@ class BlobDownloader:
             self.scores[peer] = bytes_received / elapsed if bytes_received and elapsed else 1
 
     async def new_peer_or_finished(self):
-        active_tasks = list(self.active_connections.values()) + [asyncio.sleep(1)]
-        await asyncio.wait(active_tasks, loop=self.loop, return_when='FIRST_COMPLETED')
+        active_tasks = list(self.active_connections.values()) + [asyncio.create_task(asyncio.sleep(1))]
+        await asyncio.wait(active_tasks, return_when='FIRST_COMPLETED')
 
     def cleanup_active(self):
         if not self.active_connections and not self.connections:
@@ -87,7 +88,6 @@ class BlobDownloader:
         if blob.get_is_verified():
             return blob
         self.is_running.set()
-        tried_for_this_blob: typing.Set['KademliaPeer'] = set()
         try:
             while not blob.get_is_verified() and self.is_running.is_set():
                 batch: typing.Set['KademliaPeer'] = set(self.connections.keys())
@@ -97,24 +97,15 @@ class BlobDownloader:
                     "%s running, %d peers, %d ignored, %d active, %s connections", blob_hash[:6],
                     len(batch), len(self.ignored), len(self.active_connections), len(self.connections)
                 )
-                re_add: typing.Set['KademliaPeer'] = set()
                 for peer in sorted(batch, key=lambda peer: self.scores.get(peer, 0), reverse=True):
                     if peer in self.ignored:
                         continue
-                    if peer in tried_for_this_blob:
+                    if peer in self.active_connections or not self.should_race_continue(blob):
                         continue
-                    if peer in self.active_connections:
-                        if peer not in re_add:
-                            re_add.add(peer)
-                        continue
-                    if not self.should_race_continue(blob):
-                        break
                     log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
                     t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id))
                     self.active_connections[peer] = t
-                    tried_for_this_blob.add(peer)
-                if not re_add:
-                    self.peer_queue.put_nowait(list(batch))
+                self.peer_queue.put_nowait(list(batch))
                 await self.new_peer_or_finished()
                 self.cleanup_active()
             log.debug("downloaded %s", blob_hash[:8])
@@ -133,11 +124,14 @@ class BlobDownloader:
             protocol.close()
 
 
-async def download_blob(loop, config: 'Config', blob_manager: 'BlobManager', node: 'Node',
+async def download_blob(loop, config: 'Config', blob_manager: 'BlobManager', dht_node: 'Node',
                         blob_hash: str) -> 'AbstractBlob':
-    search_queue = asyncio.Queue(loop=loop, maxsize=config.max_connections_per_download)
+    search_queue = asyncio.Queue(maxsize=config.max_connections_per_download)
     search_queue.put_nowait(blob_hash)
-    peer_queue, accumulate_task = node.accumulate_peers(search_queue)
+    peer_queue, accumulate_task = dht_node.accumulate_peers(search_queue)
+    fixed_peers = None if not config.fixed_peers else await get_kademlia_peers_from_hosts(config.fixed_peers)
+    if fixed_peers:
+        loop.call_later(config.fixed_peer_delay, peer_queue.put_nowait, fixed_peers)
     downloader = BlobDownloader(loop, config, blob_manager, peer_queue)
     try:
         return await downloader.download_blob(blob_hash)

lbry/blob_exchange/server.py

@@ -1,6 +1,7 @@
 import asyncio
 import binascii
 import logging
+import socket
 import typing
 from json.decoder import JSONDecodeError
 from lbry.blob_exchange.serialization import BlobResponse, BlobRequest, blob_response_types
@@ -24,19 +25,19 @@ class BlobServerProtocol(asyncio.Protocol):
         self.idle_timeout = idle_timeout
         self.transfer_timeout = transfer_timeout
         self.server_task: typing.Optional[asyncio.Task] = None
-        self.started_listening = asyncio.Event(loop=self.loop)
+        self.started_listening = asyncio.Event()
         self.buf = b''
         self.transport: typing.Optional[asyncio.Transport] = None
         self.lbrycrd_address = lbrycrd_address
         self.peer_address_and_port: typing.Optional[str] = None
-        self.started_transfer = asyncio.Event(loop=self.loop)
-        self.transfer_finished = asyncio.Event(loop=self.loop)
+        self.started_transfer = asyncio.Event()
+        self.transfer_finished = asyncio.Event()
         self.close_on_idle_task: typing.Optional[asyncio.Task] = None
 
     async def close_on_idle(self):
         while self.transport:
             try:
-                await asyncio.wait_for(self.started_transfer.wait(), self.idle_timeout, loop=self.loop)
+                await asyncio.wait_for(self.started_transfer.wait(), self.idle_timeout)
             except asyncio.TimeoutError:
                 log.debug("closing idle connection from %s", self.peer_address_and_port)
                 return self.close()
@@ -100,7 +101,7 @@ class BlobServerProtocol(asyncio.Protocol):
                 log.debug("send %s to %s:%i", blob_hash, peer_address, peer_port)
                 self.started_transfer.set()
                 try:
-                    sent = await asyncio.wait_for(blob.sendfile(self), self.transfer_timeout, loop=self.loop)
+                    sent = await asyncio.wait_for(blob.sendfile(self), self.transfer_timeout)
                     if sent and sent > 0:
                         self.blob_manager.connection_manager.sent_data(self.peer_address_and_port, sent)
                         log.info("sent %s (%i bytes) to %s:%i", blob_hash, sent, peer_address, peer_port)
@@ -137,7 +138,7 @@ class BlobServerProtocol(asyncio.Protocol):
             try:
                 request = BlobRequest.deserialize(self.buf + data)
                 self.buf = remainder
-            except JSONDecodeError:
+            except (UnicodeDecodeError, JSONDecodeError):
                 log.error("request from %s is not valid json (%i bytes): %s", self.peer_address_and_port,
                           len(self.buf + data), '' if not data else binascii.hexlify(self.buf + data).decode())
                 self.close()
@@ -156,7 +157,7 @@ class BlobServer:
         self.loop = loop
         self.blob_manager = blob_manager
         self.server_task: typing.Optional[asyncio.Task] = None
-        self.started_listening = asyncio.Event(loop=self.loop)
+        self.started_listening = asyncio.Event()
         self.lbrycrd_address = lbrycrd_address
         self.idle_timeout = idle_timeout
         self.transfer_timeout = transfer_timeout
@@ -167,6 +168,13 @@ class BlobServer:
             raise Exception("already running")
 
         async def _start_server():
+            # checking if the port is in use
+            # thx https://stackoverflow.com/a/52872579
+            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+                if s.connect_ex(('localhost', port)) == 0:
+                    # the port is already in use!
+                    log.error("Failed to bind TCP %s:%d", interface, port)
+
             server = await self.loop.create_server(
                 lambda: self.server_protocol_class(self.loop, self.blob_manager, self.lbrycrd_address,
                                                    self.idle_timeout, self.transfer_timeout),

lbry/conf.py

@@ -1,13 +1,12 @@
 import os
 import re
 import sys
-import typing
 import logging
+from typing import List, Dict, Tuple, Union, TypeVar, Generic, Optional
 from argparse import ArgumentParser
 from contextlib import contextmanager
-
-import yaml
 from appdirs import user_data_dir, user_config_dir
+import yaml
 from lbry.error import InvalidCurrencyError
 from lbry.dht import constants
 from lbry.wallet.coinselection import STRATEGIES
@@ -16,7 +15,7 @@ log = logging.getLogger(__name__)
 
 
 NOT_SET = type('NOT_SET', (object,), {})  # pylint: disable=invalid-name
-T = typing.TypeVar('T')
+T = TypeVar('T')
 
 CURRENCIES = {
     'BTC': {'type': 'crypto'},
@@ -25,11 +24,11 @@ CURRENCIES = {
 }
 
 
-class Setting(typing.Generic[T]):
+class Setting(Generic[T]):
 
-    def __init__(self, doc: str, default: typing.Optional[T] = None,
-                 previous_names: typing.Optional[typing.List[str]] = None,
-                 metavar: typing.Optional[str] = None):
+    def __init__(self, doc: str, default: Optional[T] = None,
+                 previous_names: Optional[List[str]] = None,
+                 metavar: Optional[str] = None):
         self.doc = doc
         self.default = default
         self.previous_names = previous_names or []
@@ -46,7 +45,7 @@ class Setting(typing.Generic[T]):
     def no_cli_name(self):
         return f"--no-{self.name.replace('_', '-')}"
 
-    def __get__(self, obj: typing.Optional['BaseConfig'], owner) -> T:
+    def __get__(self, obj: Optional['BaseConfig'], owner) -> T:
         if obj is None:
             return self
         for location in obj.search_order:
@@ -54,7 +53,7 @@ class Setting(typing.Generic[T]):
                 return location[self.name]
         return self.default
 
-    def __set__(self, obj: 'BaseConfig', val: typing.Union[T, NOT_SET]):
+    def __set__(self, obj: 'BaseConfig', val: Union[T, NOT_SET]):
         if val == NOT_SET:
             for location in obj.modify_order:
                 if self.name in location:
@@ -64,6 +63,18 @@ class Setting(typing.Generic[T]):
             for location in obj.modify_order:
                 location[self.name] = val
 
+    def is_set(self, obj: 'BaseConfig') -> bool:
+        for location in obj.search_order:
+            if self.name in location:
+                return True
+        return False
+
+    def is_set_to_default(self, obj: 'BaseConfig') -> bool:
+        for location in obj.search_order:
+            if self.name in location:
+                return location[self.name] == self.default
+        return False
+
     def validate(self, value):
         raise NotImplementedError()
 
@@ -88,7 +99,7 @@ class String(Setting[str]):
             f"Setting '{self.name}' must be a string."
 
     # TODO: removes this after pylint starts to understand generics
-    def __get__(self, obj: typing.Optional['BaseConfig'], owner) -> str:  # pylint: disable=useless-super-delegation
+    def __get__(self, obj: Optional['BaseConfig'], owner) -> str:  # pylint: disable=useless-super-delegation
         return super().__get__(obj, owner)
 
 
@@ -192,7 +203,7 @@ class MaxKeyFee(Setting[dict]):
         )
         parser.add_argument(
             self.no_cli_name,
-            help=f"Disable maximum key fee check.",
+            help="Disable maximum key fee check.",
             dest=self.name,
             const=None,
             action="store_const",
@@ -201,7 +212,7 @@ class MaxKeyFee(Setting[dict]):
 
 
 class StringChoice(String):
-    def __init__(self, doc: str, valid_values: typing.List[str], default: str, *args, **kwargs):
+    def __init__(self, doc: str, valid_values: List[str], default: str, *args, **kwargs):
         super().__init__(doc, default, *args, **kwargs)
         if not valid_values:
             raise ValueError("No valid values provided")
@@ -274,17 +285,95 @@ class Strings(ListSetting):
                 f"'{self.name}' must be a string."
 
 
+class KnownHubsList:
+
+    def __init__(self, config: 'Config' = None, file_name: str = 'known_hubs.yml'):
+        self.file_name = file_name
+        self.path = os.path.join(config.wallet_dir, self.file_name) if config else None
+        self.hubs: Dict[Tuple[str, int], Dict] = {}
+        if self.exists:
+            self.load()
+
+    @property
+    def exists(self):
+        return self.path and os.path.exists(self.path)
+
+    @property
+    def serialized(self) -> Dict[str, Dict]:
+        return {f"{host}:{port}": details for (host, port), details in self.hubs.items()}
+
+    def filter(self, match_none=False, **kwargs):
+        if not kwargs:
+            return self.hubs
+        result = {}
+        for hub, details in self.hubs.items():
+            for key, constraint in kwargs.items():
+                value = details.get(key)
+                if value == constraint or (match_none and value is None):
+                    result[hub] = details
+                    break
+        return result
+
+    def load(self):
+        if self.path:
+            with open(self.path, 'r') as known_hubs_file:
+                raw = known_hubs_file.read()
+                for hub, details in yaml.safe_load(raw).items():
+                    self.set(hub, details)
+
+    def save(self):
+        if self.path:
+            with open(self.path, 'w') as known_hubs_file:
+                known_hubs_file.write(yaml.safe_dump(self.serialized, default_flow_style=False))
+
+    def set(self, hub: str, details: Dict):
+        if hub and hub.count(':') == 1:
+            host, port = hub.split(':')
+            hub_parts = (host, int(port))
+            if hub_parts not in self.hubs:
+                self.hubs[hub_parts] = details
+                return hub
+
+    def add_hubs(self, hubs: List[str]):
+        added = False
+        for hub in hubs:
+            if self.set(hub, {}) is not None:
+                added = True
+        return added
+
+    def items(self):
+        return self.hubs.items()
+
+    def __bool__(self):
+        return len(self) > 0
+
+    def __len__(self):
+        return self.hubs.__len__()
+
+    def __iter__(self):
+        return iter(self.hubs)
+
+
 class EnvironmentAccess:
     PREFIX = 'LBRY_'
 
-    def __init__(self, environ: dict):
-        self.environ = environ
+    def __init__(self, config: 'BaseConfig', environ: dict):
+        self.configuration = config
+        self.data = {}
+        if environ:
+            self.load(environ)
+
+    def load(self, environ):
+        for setting in self.configuration.get_settings():
+            value = environ.get(f'{self.PREFIX}{setting.name.upper()}', NOT_SET)
+            if value != NOT_SET and not (isinstance(setting, ListSetting) and value is None):
+                self.data[setting.name] = setting.deserialize(value)
 
     def __contains__(self, item: str):
-        return f'{self.PREFIX}{item.upper()}' in self.environ
+        return item in self.data
 
     def __getitem__(self, item: str):
-        return self.environ[f'{self.PREFIX}{item.upper()}']
+        return self.data[item]
 
 
 class ArgumentAccess:
@@ -325,7 +414,7 @@ class ConfigFileAccess:
         cls = type(self.configuration)
         with open(self.path, 'r') as config_file:
             raw = config_file.read()
-        serialized = yaml.load(raw) or {}
+        serialized = yaml.safe_load(raw) or {}
         for key, value in serialized.items():
             attr = getattr(cls, key, None)
             if attr is None:
@@ -369,7 +458,7 @@ class ConfigFileAccess:
         del self.data[key]
 
 
-TBC = typing.TypeVar('TBC', bound='BaseConfig')
+TBC = TypeVar('TBC', bound='BaseConfig')
 
 
 class BaseConfig:
@@ -443,7 +532,7 @@ class BaseConfig:
         self.arguments = ArgumentAccess(self, args)
 
     def set_environment(self, environ=None):
-        self.environment = EnvironmentAccess(environ or os.environ)
+        self.environment = EnvironmentAccess(self, environ or os.environ)
 
     def set_persisted(self, config_file_path=None):
         if config_file_path is None:
@@ -464,15 +553,22 @@ class BaseConfig:
 
 class TranscodeConfig(BaseConfig):
 
-    ffmpeg_folder = String('The path to ffmpeg and ffprobe', '')
+    ffmpeg_path = String('A list of places to check for ffmpeg and ffprobe. '
+                         f'$data_dir/ffmpeg/bin and $PATH are checked afterward. Separator: {os.pathsep}',
+                         '', previous_names=['ffmpeg_folder'])
     video_encoder = String('FFmpeg codec and parameters for the video encoding. '
                            'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental',
-                           'libx264 -crf 18 -vf "format=yuv420p"')
+                           'libx264 -crf 24 -preset faster -pix_fmt yuv420p')
+    video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 5_000_000)
+    video_scaler = String('FFmpeg scaling parameters for reducing bitrate. '
+                          'Example: -vf "scale=-2:720,fps=24" -maxrate 5M -bufsize 3M',
+                          r'-vf "scale=if(gte(iw\,ih)\,min(1920\,iw)\,-2):if(lt(iw\,ih)\,min(1920\,ih)\,-2)" '
+                          r'-maxrate 5500K -bufsize 5000K')
     audio_encoder = String('FFmpeg codec and parameters for the audio encoding. '
                            'Example: libopus -b:a 128k',
-                           'aac -b:a 192k')
-    volume_filter = String('FFmpeg filter for audio normalization.', '-af loudnorm')
-    volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', '240')
+                           'aac -b:a 160k')
+    volume_filter = String('FFmpeg filter for audio normalization. Exmple: -af loudnorm', '')
+    volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', 240)
 
 
 class CLIConfig(TranscodeConfig):
@@ -493,6 +589,9 @@ class CLIConfig(TranscodeConfig):
 
 
 class Config(CLIConfig):
+
+    jurisdiction = String("Limit interactions to wallet server in this jurisdiction.")
+
     # directories
     data_dir = Path("Directory path to store blobs.", metavar='DIR')
     download_dir = Path(
@@ -514,7 +613,7 @@ class Config(CLIConfig):
         "ports or have firewall rules you likely want to disable this.", True
     )
     udp_port = Integer("UDP port for communicating on the LBRY DHT", 4444, previous_names=['dht_node_port'])
-    tcp_port = Integer("TCP port to listen for incoming blob requests", 3333, previous_names=['peer_port'])
+    tcp_port = Integer("TCP port to listen for incoming blob requests", 4444, previous_names=['peer_port'])
     prometheus_port = Integer("Port to expose prometheus metrics (off by default)", 0)
     network_interface = String("Interface to use for the DHT and blob exchange", '0.0.0.0')
 
@@ -523,17 +622,24 @@ class Config(CLIConfig):
         "Routing table bucket index below which we always split the bucket if given a new key to add to it and "
         "the bucket is full. As this value is raised the depth of the routing table (and number of peers in it) "
         "will increase. This setting is used by seed nodes, you probably don't want to change it during normal "
-        "use.", 1
+        "use.", 2
+    )
+    is_bootstrap_node = Toggle(
+        "When running as a bootstrap node, disable all logic related to balancing the routing table, so we can "
+        "add as many peers as possible and better help first-runs.", False
     )
 
     # protocol timeouts
     download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
     blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
+    hub_timeout = Float("Timeout when making a hub request", 30.0)
     peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0)
     node_rpc_timeout = Float("Timeout when making a DHT request", constants.RPC_TIMEOUT)
 
     # blob announcement and download
     save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True)
+    network_storage_limit = Integer("Disk space in MB to be allocated for helping the P2P network. 0 = disable", 0)
+    blob_storage_limit = Integer("Disk space in MB to be allocated for blob storage. 0 = no limit", 0)
     blob_lru_cache_size = Integer(
         "LRU cache size for decrypted downloaded blobs used to minimize re-downloading the same blobs when "
         "replying to a range request. Set to 0 to disable.", 32
@@ -550,6 +656,7 @@ class Config(CLIConfig):
         "Maximum number of peers to connect to while downloading a blob", 4,
         previous_names=['max_connections_per_stream']
     )
+    concurrent_hub_requests = Integer("Maximum number of concurrent hub requests", 32)
     fixed_peer_delay = Float(
         "Amount of seconds before adding the reflector servers as potential peers to download from in case dht"
         "peers are not found or are slow", 2.0
@@ -558,6 +665,7 @@ class Config(CLIConfig):
         "Don't download streams with fees exceeding this amount. When set to "
         "null, the amount is unbounded.", {'currency': 'USD', 'amount': 50.0}
     )
+    max_wallet_server_fee = String("Maximum daily LBC amount allowed as payment for wallet servers.", "0.0")
 
     # reflector settings
     reflect_streams = Toggle(
@@ -569,9 +677,22 @@ class Config(CLIConfig):
     )
 
     # servers
-    reflector_servers = Servers("Reflector re-hosting servers", [
+    reflector_servers = Servers("Reflector re-hosting servers for mirroring publishes", [
         ('reflector.lbry.com', 5566)
     ])
+
+    fixed_peers = Servers("Fixed peers to fall back to if none are found on P2P for a blob", [
+        ('cdn.reflector.lbry.com', 5567)
+    ])
+
+    tracker_servers = Servers("BitTorrent-compatible (BEP15) UDP trackers for helping P2P discovery", [
+        ('tracker.lbry.com', 9252),
+        ('tracker.lbry.grin.io', 9252),
+        ('tracker.lbry.pigg.es', 9252),
+        ('tracker.lizard.technology', 9252),
+        ('s1.lbry.network', 9252),
+    ])
+
     lbryum_servers = Servers("SPV wallet servers", [
         ('spv11.lbry.com', 50001),
         ('spv12.lbry.com', 50001),
@@ -582,29 +703,36 @@ class Config(CLIConfig):
         ('spv17.lbry.com', 50001),
         ('spv18.lbry.com', 50001),
         ('spv19.lbry.com', 50001),
+        ('hub.lbry.grin.io', 50001),
+        ('hub.lizard.technology', 50001),
+        ('s1.lbry.network', 50001),
     ])
     known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
+        ('dht.lbry.grin.io', 4444),  # Grin
+        ('dht.lbry.madiator.com', 4444),  # Madiator
+        ('dht.lbry.pigg.es', 4444), # Pigges
         ('lbrynet1.lbry.com', 4444),  # US EAST
         ('lbrynet2.lbry.com', 4444),  # US WEST
         ('lbrynet3.lbry.com', 4444),  # EU
-        ('lbrynet4.lbry.com', 4444)  # ASIA
+        ('lbrynet4.lbry.com', 4444),  # ASIA
+        ('dht.lizard.technology', 4444),  # Jack
+        ('s2.lbry.network', 4444),
     ])
 
-    comment_server = String("Comment server API URL", "https://comments.lbry.com/api")
-
     # blockchain
     blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
-    s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
-    cache_time = Integer("Time to cache resolved claims", 150)  # TODO: use this
 
     # daemon
-    save_files = Toggle("Save downloaded files when calling `get` by default", True)
+    save_files = Toggle("Save downloaded files when calling `get` by default", False)
     components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
     share_usage_data = Toggle(
-        "Whether to share usage stats and diagnostic info with LBRY.", True,
+        "Whether to share usage stats and diagnostic info with LBRY.", False,
         previous_names=['upload_log', 'upload_log', 'share_debug_info']
     )
     track_bandwidth = Toggle("Track bandwidth usage", True)
+    allowed_origin = String(
+        "Allowed `Origin` header value for API request (sent by browser), use * to allow "
+        "all hosts; default is to only allow API requests with no `Origin` value.", "")
 
     # media server
     streaming_server = String('Host name and port to serve streaming media over range requests',
@@ -614,8 +742,10 @@ class Config(CLIConfig):
 
     coin_selection_strategy = StringChoice(
         "Strategy to use when selecting UTXOs for a transaction",
-        STRATEGIES, "standard")
+        STRATEGIES, "prefer_confirmed"
+    )
 
+    transaction_cache_size = Integer("Transaction cache size", 2 ** 17)
     save_resolved_claims = Toggle(
         "Save content claims to the database when they are resolved to keep file_list up to date, "
         "only disable this if file_x commands are not needed", True
@@ -632,6 +762,7 @@ class Config(CLIConfig):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
         self.set_default_paths()
+        self.known_hubs = KnownHubsList(self)
 
     def set_default_paths(self):
         if 'darwin' in sys.platform.lower():
@@ -653,7 +784,7 @@ class Config(CLIConfig):
         return os.path.join(self.data_dir, 'lbrynet.log')
 
 
-def get_windows_directories() -> typing.Tuple[str, str, str]:
+def get_windows_directories() -> Tuple[str, str, str]:
     from lbry.winpaths import get_path, FOLDERID, UserHandle, \
         PathNotFoundException  # pylint: disable=import-outside-toplevel
 
@@ -675,19 +806,20 @@ def get_windows_directories() -> typing.Tuple[str, str, str]:
     return data_dir, lbryum_dir, download_dir
 
 
-def get_darwin_directories() -> typing.Tuple[str, str, str]:
+def get_darwin_directories() -> Tuple[str, str, str]:
     data_dir = user_data_dir('LBRY')
     lbryum_dir = os.path.expanduser('~/.lbryum')
     download_dir = os.path.expanduser('~/Downloads')
     return data_dir, lbryum_dir, download_dir
 
 
-def get_linux_directories() -> typing.Tuple[str, str, str]:
+def get_linux_directories() -> Tuple[str, str, str]:
     try:
         with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg:
-            down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()).group(1)
-        down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir)
-        download_dir = re.sub('\"', '', down_dir)
+            down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read())
+        if down_dir:
+            down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir.group(1))
+            download_dir = re.sub('\"', '', down_dir)
     except OSError:
         download_dir = os.getenv('XDG_DOWNLOAD_DIR')
     if not download_dir:

lbry/connection_manager.py

@@ -67,7 +67,7 @@ class ConnectionManager:
 
         while True:
             last = time.perf_counter()
-            await asyncio.sleep(0.1, loop=self.loop)
+            await asyncio.sleep(0.1)
             self._status['incoming_bps'].clear()
             self._status['outgoing_bps'].clear()
             now = time.perf_counter()

lbry/dht/blob_announcer.py

@@ -1,6 +1,9 @@
 import asyncio
 import typing
 import logging
+
+from prometheus_client import Counter, Gauge
+
 if typing.TYPE_CHECKING:
     from lbry.dht.node import Node
     from lbry.extras.daemon.storage import SQLiteStorage
@@ -9,45 +12,59 @@ log = logging.getLogger(__name__)
 
 
 class BlobAnnouncer:
+    announcements_sent_metric = Counter(
+        "announcements_sent", "Number of announcements sent and their respective status.", namespace="dht_node",
+        labelnames=("peers", "error"),
+    )
+    announcement_queue_size_metric = Gauge(
+        "announcement_queue_size", "Number of hashes waiting to be announced.", namespace="dht_node",
+        labelnames=("scope",)
+    )
+
     def __init__(self, loop: asyncio.AbstractEventLoop, node: 'Node', storage: 'SQLiteStorage'):
         self.loop = loop
         self.node = node
         self.storage = storage
         self.announce_task: asyncio.Task = None
         self.announce_queue: typing.List[str] = []
+        self._done = asyncio.Event()
+        self.announced = set()
 
-    async def _submit_announcement(self, blob_hash):
-        try:
-            peers = len(await self.node.announce_blob(blob_hash))
-            if peers > 4:
-                return blob_hash
-            else:
-                log.debug("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
-        except Exception as err:
-            if isinstance(err, asyncio.CancelledError):  # TODO: remove when updated to 3.8
-                raise err
-            log.warning("error announcing %s: %s", blob_hash[:8], str(err))
+    async def _run_consumer(self):
+        while self.announce_queue:
+            try:
+                blob_hash = self.announce_queue.pop()
+                peers = len(await self.node.announce_blob(blob_hash))
+                self.announcements_sent_metric.labels(peers=peers, error=False).inc()
+                if peers > 4:
+                    self.announced.add(blob_hash)
+                else:
+                    log.debug("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
+            except Exception as err:
+                self.announcements_sent_metric.labels(peers=0, error=True).inc()
+                log.warning("error announcing %s: %s", blob_hash[:8], str(err))
 
     async def _announce(self, batch_size: typing.Optional[int] = 10):
         while batch_size:
             if not self.node.joined.is_set():
                 await self.node.joined.wait()
-            await asyncio.sleep(60, loop=self.loop)
+            await asyncio.sleep(60)
             if not self.node.protocol.routing_table.get_peers():
                 log.warning("No peers in DHT, announce round skipped")
                 continue
             self.announce_queue.extend(await self.storage.get_blobs_to_announce())
+            self.announcement_queue_size_metric.labels(scope="global").set(len(self.announce_queue))
             log.debug("announcer task wake up, %d blobs to announce", len(self.announce_queue))
             while len(self.announce_queue) > 0:
                 log.info("%i blobs to announce", len(self.announce_queue))
-                announced = await asyncio.gather(*[
-                    self._submit_announcement(
-                        self.announce_queue.pop()) for _ in range(batch_size) if self.announce_queue
-                ], loop=self.loop)
-                announced = list(filter(None, announced))
+                await asyncio.gather(*[self._run_consumer() for _ in range(batch_size)])
+                announced = list(filter(None, self.announced))
                 if announced:
                     await self.storage.update_last_announced_blobs(announced)
                     log.info("announced %i blobs", len(announced))
+                    self.announced.clear()
+            self._done.set()
+            self._done.clear()
 
     def start(self, batch_size: typing.Optional[int] = 10):
         assert not self.announce_task or self.announce_task.done(), "already running"
@@ -56,3 +73,6 @@ class BlobAnnouncer:
     def stop(self):
         if self.announce_task and not self.announce_task.done():
             self.announce_task.cancel()
+
+    def wait(self):
+        return self._done.wait()

lbry/dht/constants.py

@@ -20,7 +20,6 @@ MAYBE_PING_DELAY = 300  # 5 minutes
 CHECK_REFRESH_INTERVAL = REFRESH_INTERVAL / 5
 RPC_ID_LENGTH = 20
 PROTOCOL_VERSION = 1
-BOTTOM_OUT_LIMIT = 3
 MSG_SIZE_LIMIT = 1400
 
 

lbry/dht/node.py

@@ -1,9 +1,11 @@
 import logging
 import asyncio
 import typing
-import binascii
 import socket
-from lbry.utils import resolve_host
+
+from prometheus_client import Gauge
+
+from lbry.utils import aclosing, resolve_host
 from lbry.dht import constants
 from lbry.dht.peer import make_kademlia_peer
 from lbry.dht.protocol.distance import Distance
@@ -18,20 +20,32 @@ log = logging.getLogger(__name__)
 
 
 class Node:
+    storing_peers_metric = Gauge(
+        "storing_peers", "Number of peers storing blobs announced to this node", namespace="dht_node",
+        labelnames=("scope",),
+    )
+    stored_blob_with_x_bytes_colliding = Gauge(
+        "stored_blobs_x_bytes_colliding", "Number of blobs with at least X bytes colliding with this node id prefix",
+        namespace="dht_node", labelnames=("amount",)
+    )
     def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int,
                  internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: float = constants.RPC_TIMEOUT,
-                 split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX,
+                 split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX, is_bootstrap_node: bool = False,
                  storage: typing.Optional['SQLiteStorage'] = None):
         self.loop = loop
         self.internal_udp_port = internal_udp_port
         self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port, rpc_timeout,
-                                         split_buckets_under_index)
+                                         split_buckets_under_index, is_bootstrap_node)
         self.listening_port: asyncio.DatagramTransport = None
-        self.joined = asyncio.Event(loop=self.loop)
+        self.joined = asyncio.Event()
         self._join_task: asyncio.Task = None
         self._refresh_task: asyncio.Task = None
         self._storage = storage
 
+    @property
+    def stored_blob_hashes(self):
+        return self.protocol.data_store.keys()
+
     async def refresh_node(self, force_once=False):
         while True:
             # remove peers with expired blob announcements from the datastore
@@ -41,17 +55,21 @@ class Node:
             # add all peers in the routing table
             total_peers.extend(self.protocol.routing_table.get_peers())
             # add all the peers who have announced blobs to us
-            total_peers.extend(self.protocol.data_store.get_storing_contacts())
+            storing_peers = self.protocol.data_store.get_storing_contacts()
+            self.storing_peers_metric.labels("global").set(len(storing_peers))
+            total_peers.extend(storing_peers)
+
+            counts = {0: 0, 1: 0, 2: 0}
+            node_id = self.protocol.node_id
+            for blob_hash in self.protocol.data_store.keys():
+                bytes_colliding = 0 if blob_hash[0] != node_id[0] else 2 if blob_hash[1] == node_id[1] else 1
+                counts[bytes_colliding] += 1
+            self.stored_blob_with_x_bytes_colliding.labels(amount=0).set(counts[0])
+            self.stored_blob_with_x_bytes_colliding.labels(amount=1).set(counts[1])
+            self.stored_blob_with_x_bytes_colliding.labels(amount=2).set(counts[2])
 
             # get ids falling in the midpoint of each bucket that hasn't been recently updated
             node_ids = self.protocol.routing_table.get_refresh_list(0, True)
-            # if we have 3 or fewer populated buckets get two random ids in the range of each to try and
-            # populate/split the buckets further
-            buckets_with_contacts = self.protocol.routing_table.buckets_with_contacts()
-            if buckets_with_contacts <= 3:
-                for i in range(buckets_with_contacts):
-                    node_ids.append(self.protocol.routing_table.random_id_in_bucket_range(i))
-                    node_ids.append(self.protocol.routing_table.random_id_in_bucket_range(i))
 
             if self.protocol.routing_table.get_peers():
                 # if we have node ids to look up, perform the iterative search until we have k results
@@ -61,7 +79,7 @@ class Node:
             else:
                 if force_once:
                     break
-                fut = asyncio.Future(loop=self.loop)
+                fut = asyncio.Future()
                 self.loop.call_later(constants.REFRESH_INTERVAL // 4, fut.set_result, None)
                 await fut
                 continue
@@ -75,12 +93,12 @@ class Node:
             if force_once:
                 break
 
-            fut = asyncio.Future(loop=self.loop)
+            fut = asyncio.Future()
             self.loop.call_later(constants.REFRESH_INTERVAL, fut.set_result, None)
             await fut
 
     async def announce_blob(self, blob_hash: str) -> typing.List[bytes]:
-        hash_value = binascii.unhexlify(blob_hash.encode())
+        hash_value = bytes.fromhex(blob_hash)
         assert len(hash_value) == constants.HASH_LENGTH
         peers = await self.peer_search(hash_value)
 
@@ -90,12 +108,12 @@ class Node:
         for peer in peers:
             log.debug("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
         stored_to_tup = await asyncio.gather(
-            *(self.protocol.store_to_peer(hash_value, peer) for peer in peers), loop=self.loop
+            *(self.protocol.store_to_peer(hash_value, peer) for peer in peers)
         )
         stored_to = [node_id for node_id, contacted in stored_to_tup if contacted]
         if stored_to:
             log.debug(
-                "Stored %s to %i of %i attempted peers", binascii.hexlify(hash_value).decode()[:8],
+                "Stored %s to %i of %i attempted peers", hash_value.hex()[:8],
                 len(stored_to), len(peers)
             )
         else:
@@ -164,39 +182,36 @@ class Node:
                             for address, udp_port in known_node_urls or []
                         ]))
                     except socket.gaierror:
-                        await asyncio.sleep(30, loop=self.loop)
+                        await asyncio.sleep(30)
                         continue
 
                 self.protocol.peer_manager.reset()
                 self.protocol.ping_queue.enqueue_maybe_ping(*seed_peers, delay=0.0)
                 await self.peer_search(self.protocol.node_id, shortlist=seed_peers, count=32)
 
-            await asyncio.sleep(1, loop=self.loop)
+            await asyncio.sleep(1)
 
     def start(self, interface: str, known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
         self._join_task = self.loop.create_task(self.join_network(interface, known_node_urls))
 
     def get_iterative_node_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
-                                  bottom_out_limit: int = constants.BOTTOM_OUT_LIMIT,
                                   max_results: int = constants.K) -> IterativeNodeFinder:
-
-        return IterativeNodeFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
-                                   key, bottom_out_limit, max_results, None, shortlist)
+        shortlist = shortlist or self.protocol.routing_table.find_close_peers(key)
+        return IterativeNodeFinder(self.loop, self.protocol, key, max_results, shortlist)
 
     def get_iterative_value_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
-                                   bottom_out_limit: int = 40,
                                    max_results: int = -1) -> IterativeValueFinder:
-
-        return IterativeValueFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
-                                    key, bottom_out_limit, max_results, None, shortlist)
+        shortlist = shortlist or self.protocol.routing_table.find_close_peers(key)
+        return IterativeValueFinder(self.loop, self.protocol, key, max_results, shortlist)
 
     async def peer_search(self, node_id: bytes, count=constants.K, max_results=constants.K * 2,
-                          bottom_out_limit=20, shortlist: typing.Optional[typing.List['KademliaPeer']] = None
+                          shortlist: typing.Optional[typing.List['KademliaPeer']] = None
                           ) -> typing.List['KademliaPeer']:
         peers = []
-        async for iteration_peers in self.get_iterative_node_finder(
-                node_id, shortlist=shortlist, bottom_out_limit=bottom_out_limit, max_results=max_results):
-            peers.extend(iteration_peers)
+        async with aclosing(self.get_iterative_node_finder(
+                node_id, shortlist=shortlist, max_results=max_results)) as node_finder:
+            async for iteration_peers in node_finder:
+                peers.extend(iteration_peers)
         distance = Distance(node_id)
         peers.sort(key=lambda peer: distance(peer.node_id))
         return peers[:count]
@@ -222,39 +237,46 @@ class Node:
 
         # prioritize peers who reply to a dht ping first
         # this minimizes attempting to make tcp connections that won't work later to dead or unreachable peers
-
-        async for results in self.get_iterative_value_finder(binascii.unhexlify(blob_hash.encode())):
-            to_put = []
-            for peer in results:
-                if peer.address == self.protocol.external_ip and self.protocol.peer_port == peer.tcp_port:
-                    continue
-                is_good = self.protocol.peer_manager.peer_is_good(peer)
-                if is_good:
-                    # the peer has replied recently over UDP, it can probably be reached on the TCP port
-                    to_put.append(peer)
-                elif is_good is None:
-                    if not peer.udp_port:
-                        # TODO: use the same port for TCP and UDP
-                        # the udp port must be guessed
-                        # default to the ports being the same. if the TCP port appears to be <=0.48.0 default,
-                        # including on a network with several nodes, then assume the udp port is proportionately
-                        # based on a starting port of 4444
-                        udp_port_to_try = peer.tcp_port
-                        if 3400 > peer.tcp_port > 3332:
-                            udp_port_to_try = (peer.tcp_port - 3333) + 4444
-                        self.loop.create_task(put_into_result_queue_after_pong(
-                            make_kademlia_peer(peer.node_id, peer.address, udp_port_to_try, peer.tcp_port)
-                        ))
+        async with aclosing(self.get_iterative_value_finder(bytes.fromhex(blob_hash))) as value_finder:
+            async for results in value_finder:
+                to_put = []
+                for peer in results:
+                    if peer.address == self.protocol.external_ip and self.protocol.peer_port == peer.tcp_port:
+                        continue
+                    is_good = self.protocol.peer_manager.peer_is_good(peer)
+                    if is_good:
+                        # the peer has replied recently over UDP, it can probably be reached on the TCP port
+                        to_put.append(peer)
+                    elif is_good is None:
+                        if not peer.udp_port:
+                            # TODO: use the same port for TCP and UDP
+                            # the udp port must be guessed
+                            # default to the ports being the same. if the TCP port appears to be <=0.48.0 default,
+                            # including on a network with several nodes, then assume the udp port is proportionately
+                            # based on a starting port of 4444
+                            udp_port_to_try = peer.tcp_port
+                            if 3400 > peer.tcp_port > 3332:
+                                udp_port_to_try = (peer.tcp_port - 3333) + 4444
+                            self.loop.create_task(put_into_result_queue_after_pong(
+                                make_kademlia_peer(peer.node_id, peer.address, udp_port_to_try, peer.tcp_port)
+                            ))
+                        else:
+                            self.loop.create_task(put_into_result_queue_after_pong(peer))
                     else:
-                        self.loop.create_task(put_into_result_queue_after_pong(peer))
-                else:
-                    # the peer is known to be bad/unreachable, skip trying to connect to it over TCP
-                    log.debug("skip bad peer %s:%i for %s", peer.address, peer.tcp_port, blob_hash)
-            if to_put:
-                result_queue.put_nowait(to_put)
+                        # the peer is known to be bad/unreachable, skip trying to connect to it over TCP
+                        log.debug("skip bad peer %s:%i for %s", peer.address, peer.tcp_port, blob_hash)
+                if to_put:
+                    result_queue.put_nowait(to_put)
 
     def accumulate_peers(self, search_queue: asyncio.Queue,
                          peer_queue: typing.Optional[asyncio.Queue] = None
                          ) -> typing.Tuple[asyncio.Queue, asyncio.Task]:
-        queue = peer_queue or asyncio.Queue(loop=self.loop)
+        queue = peer_queue or asyncio.Queue()
         return queue, self.loop.create_task(self._accumulate_peers_for_value(search_queue, queue))
+
+
+async def get_kademlia_peers_from_hosts(peer_list: typing.List[typing.Tuple[str, int]]) -> typing.List['KademliaPeer']:
+    peer_address_list = [(await resolve_host(url, port, proto='tcp'), port) for url, port in peer_list]
+    kademlia_peer_list = [make_kademlia_peer(None, address, None, tcp_port=port, allow_localhost=True)
+                          for address, port in peer_address_list]
+    return kademlia_peer_list

lbry/dht/peer.py

@@ -1,18 +1,21 @@
 import typing
 import asyncio
 import logging
-import ipaddress
-from binascii import hexlify
 from dataclasses import dataclass, field
 from functools import lru_cache
 
+from prometheus_client import Gauge
+
+from lbry.utils import is_valid_public_ipv4 as _is_valid_public_ipv4, LRUCache
 from lbry.dht import constants
 from lbry.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
 
+ALLOW_LOCALHOST = False
+CACHE_SIZE = 16384
 log = logging.getLogger(__name__)
 
 
-@lru_cache(1024)
+@lru_cache(CACHE_SIZE)
 def make_kademlia_peer(node_id: typing.Optional[bytes], address: typing.Optional[str],
                        udp_port: typing.Optional[int] = None,
                        tcp_port: typing.Optional[int] = None,
@@ -20,40 +23,32 @@ def make_kademlia_peer(node_id: typing.Optional[bytes], address: typing.Optional
     return KademliaPeer(address, node_id, udp_port, tcp_port=tcp_port, allow_localhost=allow_localhost)
 
 
-# the ipaddress module does not show these subnets as reserved
-CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
-IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
-
-ALLOW_LOCALHOST = False
-
-
 def is_valid_public_ipv4(address, allow_localhost: bool = False):
     allow_localhost = bool(allow_localhost or ALLOW_LOCALHOST)
-    try:
-        parsed_ip = ipaddress.ip_address(address)
-        if parsed_ip.is_loopback and allow_localhost:
-            return True
-        return not any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local,
-                        parsed_ip.is_loopback, parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private,
-                        parsed_ip.is_reserved,
-                        CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
-                        IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
-    except ipaddress.AddressValueError:
-        return False
+    return _is_valid_public_ipv4(address, allow_localhost)
 
 
 class PeerManager:
+    peer_manager_keys_metric = Gauge(
+        "peer_manager_keys", "Number of keys tracked by PeerManager dicts (sum)", namespace="dht_node",
+        labelnames=("scope",)
+    )
     def __init__(self, loop: asyncio.AbstractEventLoop):
         self._loop = loop
         self._rpc_failures: typing.Dict[
             typing.Tuple[str, int], typing.Tuple[typing.Optional[float], typing.Optional[float]]
-        ] = {}
-        self._last_replied: typing.Dict[typing.Tuple[str, int], float] = {}
-        self._last_sent: typing.Dict[typing.Tuple[str, int], float] = {}
-        self._last_requested: typing.Dict[typing.Tuple[str, int], float] = {}
-        self._node_id_mapping: typing.Dict[typing.Tuple[str, int], bytes] = {}
-        self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = {}
-        self._node_tokens: typing.Dict[bytes, (float, bytes)] = {}
+        ] = LRUCache(CACHE_SIZE)
+        self._last_replied: typing.Dict[typing.Tuple[str, int], float] = LRUCache(CACHE_SIZE)
+        self._last_sent: typing.Dict[typing.Tuple[str, int], float] = LRUCache(CACHE_SIZE)
+        self._last_requested: typing.Dict[typing.Tuple[str, int], float] = LRUCache(CACHE_SIZE)
+        self._node_id_mapping: typing.Dict[typing.Tuple[str, int], bytes] = LRUCache(CACHE_SIZE)
+        self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = LRUCache(CACHE_SIZE)
+        self._node_tokens: typing.Dict[bytes, (float, bytes)] = LRUCache(CACHE_SIZE)
+
+    def count_cache_keys(self):
+        return len(self._rpc_failures) + len(self._last_replied) + len(self._last_sent) + len(
+            self._last_requested) + len(self._node_id_mapping) + len(self._node_id_reverse_mapping) + len(
+            self._node_tokens)
 
     def reset(self):
         for statistic in (self._rpc_failures, self._last_replied, self._last_sent, self._last_requested):
@@ -103,6 +98,10 @@ class PeerManager:
             self._node_id_mapping.pop(self._node_id_reverse_mapping.pop(node_id))
         self._node_id_mapping[(address, udp_port)] = node_id
         self._node_id_reverse_mapping[node_id] = (address, udp_port)
+        self.peer_manager_keys_metric.labels("global").set(self.count_cache_keys())
+
+    def get_node_id_for_endpoint(self, address, port):
+        return self._node_id_mapping.get((address, port))
 
     def prune(self):  # TODO: periodically call this
         now = self._loop.time()
@@ -154,9 +153,10 @@ class PeerManager:
     def peer_is_good(self, peer: 'KademliaPeer'):
         return self.contact_triple_is_good(peer.node_id, peer.address, peer.udp_port)
 
-    def decode_tcp_peer_from_compact_address(self, compact_address: bytes) -> 'KademliaPeer':  # pylint: disable=no-self-use
-        node_id, address, tcp_port = decode_compact_address(compact_address)
-        return make_kademlia_peer(node_id, address, udp_port=None, tcp_port=tcp_port)
+
+def decode_tcp_peer_from_compact_address(compact_address: bytes) -> 'KademliaPeer':  # pylint: disable=no-self-use
+    node_id, address, tcp_port = decode_compact_address(compact_address)
+    return make_kademlia_peer(node_id, address, udp_port=None, tcp_port=tcp_port)
 
 
 @dataclass(unsafe_hash=True)
@@ -171,11 +171,11 @@ class KademliaPeer:
     def __post_init__(self):
         if self._node_id is not None:
             if not len(self._node_id) == constants.HASH_LENGTH:
-                raise ValueError("invalid node_id: {}".format(hexlify(self._node_id).decode()))
-        if self.udp_port is not None and not 1 <= self.udp_port <= 65535:
-            raise ValueError("invalid udp port")
-        if self.tcp_port is not None and not 1 <= self.tcp_port <= 65535:
-            raise ValueError("invalid tcp port")
+                raise ValueError("invalid node_id: {}".format(self._node_id.hex()))
+        if self.udp_port is not None and not 1024 <= self.udp_port <= 65535:
+            raise ValueError(f"invalid udp port: {self.address}:{self.udp_port}")
+        if self.tcp_port is not None and not 1024 <= self.tcp_port <= 65535:
+            raise ValueError(f"invalid tcp port: {self.address}:{self.tcp_port}")
         if not is_valid_public_ipv4(self.address, self.allow_localhost):
             raise ValueError(f"invalid ip address: '{self.address}'")
 
@@ -194,3 +194,6 @@ class KademliaPeer:
 
     def compact_ip(self):
         return make_compact_ip(self.address)
+
+    def __str__(self):
+        return f"{self.__class__.__name__}({self.node_id.hex()[:8]}@{self.address}:{self.udp_port}-{self.tcp_port})"

lbry/dht/protocol/data_store.py

@@ -16,6 +16,12 @@ class DictDataStore:
         self._peer_manager = peer_manager
         self.completed_blobs: typing.Set[str] = set()
 
+    def keys(self):
+        return self._data_store.keys()
+
+    def __len__(self):
+        return self._data_store.__len__()
+
     def removed_expired_peers(self):
         now = self.loop.time()
         keys = list(self._data_store.keys())

lbry/dht/protocol/iterative_find.py

@@ -1,18 +1,17 @@
 import asyncio
-from binascii import hexlify
 from itertools import chain
-from collections import defaultdict
+from collections import defaultdict, OrderedDict
+from collections.abc import AsyncIterator
 import typing
 import logging
 from typing import TYPE_CHECKING
 from lbry.dht import constants
 from lbry.dht.error import RemoteException, TransportNotConnected
 from lbry.dht.protocol.distance import Distance
-from lbry.dht.peer import make_kademlia_peer
+from lbry.dht.peer import make_kademlia_peer, decode_tcp_peer_from_compact_address
 from lbry.dht.serialization.datagram import PAGE_KEY
 
 if TYPE_CHECKING:
-    from lbry.dht.protocol.routing_table import TreeRoutingTable
     from lbry.dht.protocol.protocol import KademliaProtocol
     from lbry.dht.peer import PeerManager, KademliaPeer
 
@@ -27,6 +26,15 @@ class FindResponse:
     def get_close_triples(self) -> typing.List[typing.Tuple[bytes, str, int]]:
         raise NotImplementedError()
 
+    def get_close_kademlia_peers(self, peer_info) -> typing.Generator[typing.Iterator['KademliaPeer'], None, None]:
+        for contact_triple in self.get_close_triples():
+            node_id, address, udp_port = contact_triple
+            try:
+                yield make_kademlia_peer(node_id, address, udp_port)
+            except ValueError:
+                log.warning("misbehaving peer %s:%i returned peer with reserved ip %s:%i", peer_info.address,
+                            peer_info.udp_port, address, udp_port)
+
 
 class FindNodeResponse(FindResponse):
     def __init__(self, key: bytes, close_triples: typing.List[typing.Tuple[bytes, str, int]]):
@@ -57,57 +65,33 @@ class FindValueResponse(FindResponse):
         return [(node_id, address.decode(), port) for node_id, address, port in self.close_triples]
 
 
-def get_shortlist(routing_table: 'TreeRoutingTable', key: bytes,
-                  shortlist: typing.Optional[typing.List['KademliaPeer']]) -> typing.List['KademliaPeer']:
-    """
-    If not provided, initialize the shortlist of peers to probe to the (up to) k closest peers in the routing table
-
-    :param routing_table: a TreeRoutingTable
-    :param key: a 48 byte hash
-    :param shortlist: optional manually provided shortlist, this is done during bootstrapping when there are no
-                      peers in the routing table. During bootstrap the shortlist is set to be the seed nodes.
-    """
-    if len(key) != constants.HASH_LENGTH:
-        raise ValueError("invalid key length: %i" % len(key))
-    return shortlist or routing_table.find_close_peers(key)
-
-
-class IterativeFinder:
-    def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager',
-                 routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
-                 bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.K,
-                 exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
+class IterativeFinder(AsyncIterator):
+    def __init__(self, loop: asyncio.AbstractEventLoop,
+                 protocol: 'KademliaProtocol', key: bytes,
+                 max_results: typing.Optional[int] = constants.K,
                  shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
         if len(key) != constants.HASH_LENGTH:
             raise ValueError("invalid key length: %i" % len(key))
         self.loop = loop
-        self.peer_manager = peer_manager
-        self.routing_table = routing_table
+        self.peer_manager = protocol.peer_manager
         self.protocol = protocol
 
         self.key = key
-        self.bottom_out_limit = bottom_out_limit
-        self.max_results = max_results
-        self.exclude = exclude or []
+        self.max_results = max(constants.K, max_results)
 
-        self.active: typing.Set['KademliaPeer'] = set()
+        self.active: typing.Dict['KademliaPeer', int] = OrderedDict()  # peer: distance, sorted
         self.contacted: typing.Set['KademliaPeer'] = set()
         self.distance = Distance(key)
 
-        self.closest_peer: typing.Optional['KademliaPeer'] = None
-        self.prev_closest_peer: typing.Optional['KademliaPeer'] = None
+        self.iteration_queue = asyncio.Queue()
 
-        self.iteration_queue = asyncio.Queue(loop=self.loop)
-
-        self.running_probes: typing.Set[asyncio.Task] = set()
+        self.running_probes: typing.Dict['KademliaPeer', asyncio.Task] = {}
         self.iteration_count = 0
-        self.bottom_out_count = 0
         self.running = False
         self.tasks: typing.List[asyncio.Task] = []
-        self.delayed_calls: typing.List[asyncio.Handle] = []
-        for peer in get_shortlist(routing_table, key, shortlist):
+        for peer in shortlist:
             if peer.node_id:
-                self._add_active(peer)
+                self._add_active(peer, force=True)
             else:
                 # seed nodes
                 self._schedule_probe(peer)
@@ -139,66 +123,79 @@ class IterativeFinder:
         """
         return []
 
-    def _is_closer(self, peer: 'KademliaPeer') -> bool:
-        return not self.closest_peer or self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
-
-    def _add_active(self, peer):
+    def _add_active(self, peer, force=False):
+        if not force and self.peer_manager.peer_is_good(peer) is False:
+            return
+        if peer in self.contacted:
+            return
         if peer not in self.active and peer.node_id and peer.node_id != self.protocol.node_id:
-            self.active.add(peer)
-            if self._is_closer(peer):
-                self.prev_closest_peer = self.closest_peer
-                self.closest_peer = peer
+            self.active[peer] = self.distance(peer.node_id)
+            self.active = OrderedDict(sorted(self.active.items(), key=lambda item: item[1]))
 
     async def _handle_probe_result(self, peer: 'KademliaPeer', response: FindResponse):
         self._add_active(peer)
-        for contact_triple in response.get_close_triples():
-            node_id, address, udp_port = contact_triple
-            try:
-                self._add_active(make_kademlia_peer(node_id, address, udp_port))
-            except ValueError:
-                log.warning("misbehaving peer %s:%i returned peer with reserved ip %s:%i", peer.address,
-                            peer.udp_port, address, udp_port)
+        for new_peer in response.get_close_kademlia_peers(peer):
+            self._add_active(new_peer)
         self.check_result_ready(response)
+        self._log_state(reason="check result")
+
+    def _reset_closest(self, peer):
+        if peer in self.active:
+            del self.active[peer]
 
     async def _send_probe(self, peer: 'KademliaPeer'):
         try:
             response = await self.send_probe(peer)
         except asyncio.TimeoutError:
-            self.active.discard(peer)
+            self._reset_closest(peer)
             return
+        except asyncio.CancelledError:
+            log.debug("%s[%x] cancelled probe",
+                      type(self).__name__, id(self))
+            raise
         except ValueError as err:
             log.warning(str(err))
-            self.active.discard(peer)
+            self._reset_closest(peer)
             return
         except TransportNotConnected:
-            return self.aclose()
+            await self._aclose(reason="not connected")
+            return
         except RemoteException:
+            self._reset_closest(peer)
             return
         return await self._handle_probe_result(peer, response)
 
-    async def _search_round(self):
+    def _search_round(self):
         """
         Send up to constants.alpha (5) probes to closest active peers
         """
 
         added = 0
-        to_probe = list(self.active - self.contacted)
-        to_probe.sort(key=lambda peer: self.distance(self.key))
-        for peer in to_probe:
-            if added >= constants.ALPHA:
+        for index, peer in enumerate(self.active.keys()):
+            if index == 0:
+                log.debug("%s[%x] closest to probe: %s",
+                          type(self).__name__, id(self),
+                          peer.node_id.hex()[:8])
+            if peer in self.contacted:
+                continue
+            if len(self.running_probes) >= constants.ALPHA:
+                break
+            if index > (constants.K + len(self.running_probes)):
                 break
             origin_address = (peer.address, peer.udp_port)
-            if origin_address in self.exclude:
-                continue
             if peer.node_id == self.protocol.node_id:
                 continue
             if origin_address == (self.protocol.external_ip, self.protocol.udp_port):
                 continue
             self._schedule_probe(peer)
             added += 1
-        log.debug("running %d probes", len(self.running_probes))
+        log.debug("%s[%x] running %d probes for key %s",
+                  type(self).__name__, id(self),
+                  len(self.running_probes), self.key.hex()[:8])
         if not added and not self.running_probes:
-            log.debug("search for %s exhausted", hexlify(self.key)[:8])
+            log.debug("%s[%x] search for %s exhausted",
+                      type(self).__name__, id(self),
+                      self.key.hex()[:8])
             self.search_exhausted()
 
     def _schedule_probe(self, peer: 'KademliaPeer'):
@@ -207,33 +204,24 @@ class IterativeFinder:
         t = self.loop.create_task(self._send_probe(peer))
 
         def callback(_):
-            self.running_probes.difference_update({
-                probe for probe in self.running_probes if probe.done() or probe == t
-            })
-            if not self.running_probes:
-                self.tasks.append(self.loop.create_task(self._search_task(0.0)))
+            self.running_probes.pop(peer, None)
+            if self.running:
+                self._search_round()
 
         t.add_done_callback(callback)
-        self.running_probes.add(t)
+        self.running_probes[peer] = t
 
-    async def _search_task(self, delay: typing.Optional[float] = constants.ITERATIVE_LOOKUP_DELAY):
-        try:
-            if self.running:
-                await self._search_round()
-            if self.running:
-                self.delayed_calls.append(self.loop.call_later(delay, self._search))
-        except (asyncio.CancelledError, StopAsyncIteration, TransportNotConnected):
-            if self.running:
-                self.loop.call_soon(self.aclose)
-
-    def _search(self):
-        self.tasks.append(self.loop.create_task(self._search_task()))
+    def _log_state(self, reason="?"):
+        log.debug("%s[%x] [%s] %s: %i active nodes %i contacted %i produced %i queued",
+                  type(self).__name__, id(self), self.key.hex()[:8],
+                  reason, len(self.active), len(self.contacted),
+                  self.iteration_count, self.iteration_queue.qsize())
 
     def __aiter__(self):
         if self.running:
             raise Exception("already running")
         self.running = True
-        self._search()
+        self.loop.call_soon(self._search_round)
         return self
 
     async def __anext__(self) -> typing.List['KademliaPeer']:
@@ -246,47 +234,57 @@ class IterativeFinder:
                 raise StopAsyncIteration
             self.iteration_count += 1
             return result
-        except (asyncio.CancelledError, StopAsyncIteration):
-            self.loop.call_soon(self.aclose)
+        except asyncio.CancelledError:
+            await self._aclose(reason="cancelled")
+            raise
+        except StopAsyncIteration:
+            await self._aclose(reason="no more results")
             raise
 
-    def aclose(self):
+    async def _aclose(self, reason="?"):
+        log.debug("%s[%x] [%s] shutdown because %s: %i active nodes %i contacted %i produced %i queued",
+                  type(self).__name__, id(self), self.key.hex()[:8],
+                  reason, len(self.active), len(self.contacted),
+                  self.iteration_count, self.iteration_queue.qsize())
         self.running = False
         self.iteration_queue.put_nowait(None)
-        for task in chain(self.tasks, self.running_probes, self.delayed_calls):
+        for task in chain(self.tasks, self.running_probes.values()):
             task.cancel()
         self.tasks.clear()
         self.running_probes.clear()
-        self.delayed_calls.clear()
 
+    async def aclose(self):
+        if self.running:
+            await self._aclose(reason="aclose")
+        log.debug("%s[%x] [%s] async close completed",
+                  type(self).__name__, id(self), self.key.hex()[:8])
 
 class IterativeNodeFinder(IterativeFinder):
-    def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager',
-                 routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
-                 bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.K,
-                 exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
+    def __init__(self, loop: asyncio.AbstractEventLoop,
+                 protocol: 'KademliaProtocol', key: bytes,
+                 max_results: typing.Optional[int] = constants.K,
                  shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
-        super().__init__(loop, peer_manager, routing_table, protocol, key, bottom_out_limit, max_results, exclude,
-                         shortlist)
+        super().__init__(loop, protocol, key, max_results, shortlist)
         self.yielded_peers: typing.Set['KademliaPeer'] = set()
 
     async def send_probe(self, peer: 'KademliaPeer') -> FindNodeResponse:
-        log.debug("probing %s:%d %s", peer.address, peer.udp_port, hexlify(peer.node_id)[:8] if peer.node_id else '')
+        log.debug("probe %s:%d (%s) for NODE %s",
+                  peer.address, peer.udp_port, peer.node_id.hex()[:8] if peer.node_id else '', self.key.hex()[:8])
         response = await self.protocol.get_rpc_peer(peer).find_node(self.key)
         return FindNodeResponse(self.key, response)
 
     def search_exhausted(self):
-        self.put_result(self.active, finish=True)
+        self.put_result(self.active.keys(), finish=True)
 
     def put_result(self, from_iter: typing.Iterable['KademliaPeer'], finish=False):
         not_yet_yielded = [
             peer for peer in from_iter
             if peer not in self.yielded_peers
             and peer.node_id != self.protocol.node_id
-            and self.peer_manager.peer_is_good(peer) is not False
+            and self.peer_manager.peer_is_good(peer) is True  # return only peers who answered
         ]
         not_yet_yielded.sort(key=lambda peer: self.distance(peer.node_id))
-        to_yield = not_yet_yielded[:min(constants.K, len(not_yet_yielded))]
+        to_yield = not_yet_yielded[:max(constants.K, self.max_results)]
         if to_yield:
             self.yielded_peers.update(to_yield)
             self.iteration_queue.put_nowait(to_yield)
@@ -298,27 +296,15 @@ class IterativeNodeFinder(IterativeFinder):
 
         if found:
             log.debug("found")
-            return self.put_result(self.active, finish=True)
-        if self.prev_closest_peer and self.closest_peer and not self._is_closer(self.prev_closest_peer):
-            # log.info("improving, %i %i %i %i %i", len(self.shortlist), len(self.active), len(self.contacted),
-            #          self.bottom_out_count, self.iteration_count)
-            self.bottom_out_count = 0
-        elif self.prev_closest_peer and self.closest_peer:
-            self.bottom_out_count += 1
-            log.info("bottom out %i %i %i", len(self.active), len(self.contacted), self.bottom_out_count)
-        if self.bottom_out_count >= self.bottom_out_limit or self.iteration_count >= self.bottom_out_limit:
-            log.info("limit hit")
-            self.put_result(self.active, True)
+            return self.put_result(self.active.keys(), finish=True)
 
 
 class IterativeValueFinder(IterativeFinder):
-    def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager',
-                 routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
-                 bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.K,
-                 exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
+    def __init__(self, loop: asyncio.AbstractEventLoop,
+                 protocol: 'KademliaProtocol', key: bytes,
+                 max_results: typing.Optional[int] = constants.K,
                  shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
-        super().__init__(loop, peer_manager, routing_table, protocol, key, bottom_out_limit, max_results, exclude,
-                         shortlist)
+        super().__init__(loop, protocol, key, max_results, shortlist)
         self.blob_peers: typing.Set['KademliaPeer'] = set()
         # this tracks the index of the most recent page we requested from each peer
         self.peer_pages: typing.DefaultDict['KademliaPeer', int] = defaultdict(int)
@@ -326,6 +312,8 @@ class IterativeValueFinder(IterativeFinder):
         self.discovered_peers: typing.Dict['KademliaPeer', typing.Set['KademliaPeer']] = defaultdict(set)
 
     async def send_probe(self, peer: 'KademliaPeer') -> FindValueResponse:
+        log.debug("probe %s:%d (%s) for VALUE %s",
+                  peer.address, peer.udp_port, peer.node_id.hex()[:8], self.key.hex()[:8])
         page = self.peer_pages[peer]
         response = await self.protocol.get_rpc_peer(peer).find_value(self.key, page=page)
         parsed = FindValueResponse(self.key, response)
@@ -335,7 +323,7 @@ class IterativeValueFinder(IterativeFinder):
         decoded_peers = set()
         for compact_addr in parsed.found_compact_addresses:
             try:
-                decoded_peers.add(self.peer_manager.decode_tcp_peer_from_compact_address(compact_addr))
+                decoded_peers.add(decode_tcp_peer_from_compact_address(compact_addr))
             except ValueError:
                 log.warning("misbehaving peer %s:%i returned invalid peer for blob",
                             peer.address, peer.udp_port)
@@ -347,7 +335,6 @@ class IterativeValueFinder(IterativeFinder):
                   already_known + len(parsed.found_compact_addresses))
         if len(self.discovered_peers[peer]) != already_known + len(parsed.found_compact_addresses):
             log.warning("misbehaving peer %s:%i returned duplicate peers for blob", peer.address, peer.udp_port)
-            parsed.found_compact_addresses.clear()
         elif len(parsed.found_compact_addresses) >= constants.K and self.peer_pages[peer] < parsed.pages:
             # the peer returned a full page and indicates it has more
             self.peer_pages[peer] += 1
@@ -358,26 +345,15 @@ class IterativeValueFinder(IterativeFinder):
 
     def check_result_ready(self, response: FindValueResponse):
         if response.found:
-            blob_peers = [self.peer_manager.decode_tcp_peer_from_compact_address(compact_addr)
+            blob_peers = [decode_tcp_peer_from_compact_address(compact_addr)
                           for compact_addr in response.found_compact_addresses]
             to_yield = []
-            self.bottom_out_count = 0
             for blob_peer in blob_peers:
                 if blob_peer not in self.blob_peers:
                     self.blob_peers.add(blob_peer)
                     to_yield.append(blob_peer)
             if to_yield:
-                # log.info("found %i new peers for blob", len(to_yield))
                 self.iteration_queue.put_nowait(to_yield)
-                # if self.max_results and len(self.blob_peers) >= self.max_results:
-                #     log.info("enough blob peers found")
-                #     if not self.finished.is_set():
-                #         self.finished.set()
-        elif self.prev_closest_peer and self.closest_peer:
-            self.bottom_out_count += 1
-            if self.bottom_out_count >= self.bottom_out_limit:
-                log.info("blob peer search bottomed out")
-                self.iteration_queue.put_nowait(None)
 
     def get_initial_result(self) -> typing.List['KademliaPeer']:
         if self.protocol.data_store.has_peers_for_blob(self.key):

lbry/dht/protocol/protocol.py

@@ -3,13 +3,16 @@ import socket
 import functools
 import hashlib
 import asyncio
+import time
 import typing
-import binascii
 import random
 from asyncio.protocols import DatagramProtocol
 from asyncio.transports import DatagramTransport
 
+from prometheus_client import Gauge, Counter, Histogram
+
 from lbry.dht import constants
+from lbry.dht.serialization.bencoding import DecodeError
 from lbry.dht.serialization.datagram import decode_datagram, ErrorDatagram, ResponseDatagram, RequestDatagram
 from lbry.dht.serialization.datagram import RESPONSE_TYPE, ERROR_TYPE, PAGE_KEY
 from lbry.dht.error import RemoteException, TransportNotConnected
@@ -30,6 +33,11 @@ OLD_PROTOCOL_ERRORS = {
 
 
 class KademliaRPC:
+    stored_blob_metric = Gauge(
+        "stored_blobs", "Number of blobs announced by other peers", namespace="dht_node",
+        labelnames=("scope",),
+    )
+
     def __init__(self, protocol: 'KademliaProtocol', loop: asyncio.AbstractEventLoop, peer_port: int = 3333):
         self.protocol = protocol
         self.loop = loop
@@ -61,6 +69,7 @@ class KademliaRPC:
         self.protocol.data_store.add_peer_to_blob(
             rpc_contact, blob_hash
         )
+        self.stored_blob_metric.labels("global").set(len(self.protocol.data_store))
         return b'OK'
 
     def find_node(self, rpc_contact: 'KademliaPeer', key: bytes) -> typing.List[typing.Tuple[bytes, str, int]]:
@@ -96,7 +105,7 @@ class KademliaRPC:
             if not rpc_contact.tcp_port or peer.compact_address_tcp() != rpc_contact.compact_address_tcp()
         ]
         # if we don't have k storing peers to return and we have this hash locally, include our contact information
-        if len(peers) < constants.K and binascii.hexlify(key).decode() in self.protocol.data_store.completed_blobs:
+        if len(peers) < constants.K and key.hex() in self.protocol.data_store.completed_blobs:
             peers.append(self.compact_address())
         if not peers:
             response[PAGE_KEY] = 0
@@ -209,6 +218,10 @@ class PingQueue:
     def running(self):
         return self._running
 
+    @property
+    def busy(self):
+        return self._running and (any(self._running_pings) or any(self._pending_contacts))
+
     def enqueue_maybe_ping(self, *peers: 'KademliaPeer', delay: typing.Optional[float] = None):
         delay = delay if delay is not None else self._default_delay
         now = self._loop.time()
@@ -220,7 +233,7 @@ class PingQueue:
         async def ping_task():
             try:
                 if self._protocol.peer_manager.peer_is_good(peer):
-                    if peer not in self._protocol.routing_table.get_peers():
+                    if not self._protocol.routing_table.get_peer(peer.node_id):
                         self._protocol.add_peer(peer)
                     return
                 await self._protocol.get_rpc_peer(peer).ping()
@@ -240,7 +253,7 @@ class PingQueue:
                     del self._pending_contacts[peer]
                     self.maybe_ping(peer)
                     break
-            await asyncio.sleep(1, loop=self._loop)
+            await asyncio.sleep(1)
 
     def start(self):
         assert not self._running
@@ -259,9 +272,33 @@ class PingQueue:
 
 
 class KademliaProtocol(DatagramProtocol):
+    request_sent_metric = Counter(
+        "request_sent", "Number of requests send from DHT RPC protocol", namespace="dht_node",
+        labelnames=("method",),
+    )
+    request_success_metric = Counter(
+        "request_success", "Number of successful requests", namespace="dht_node",
+        labelnames=("method",),
+    )
+    request_error_metric = Counter(
+        "request_error", "Number of errors returned from request to other peers", namespace="dht_node",
+        labelnames=("method",),
+    )
+    HISTOGRAM_BUCKETS = (
+        .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 3.0, 3.5, 4.0, 4.50, 5.0, 5.50, 6.0, float('inf')
+    )
+    response_time_metric = Histogram(
+        "response_time", "Response times of DHT RPC requests", namespace="dht_node", buckets=HISTOGRAM_BUCKETS,
+        labelnames=("method",)
+    )
+    received_request_metric = Counter(
+        "received_request", "Number of received DHT RPC requests", namespace="dht_node",
+        labelnames=("method",),
+    )
+
     def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', node_id: bytes, external_ip: str,
                  udp_port: int, peer_port: int, rpc_timeout: float = constants.RPC_TIMEOUT,
-                 split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX):
+                 split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX, is_boostrap_node: bool = False):
         self.peer_manager = peer_manager
         self.loop = loop
         self.node_id = node_id
@@ -276,15 +313,16 @@ class KademliaProtocol(DatagramProtocol):
         self.transport: DatagramTransport = None
         self.old_token_secret = constants.generate_id()
         self.token_secret = constants.generate_id()
-        self.routing_table = TreeRoutingTable(self.loop, self.peer_manager, self.node_id, split_buckets_under_index)
+        self.routing_table = TreeRoutingTable(
+            self.loop, self.peer_manager, self.node_id, split_buckets_under_index, is_bootstrap_node=is_boostrap_node)
         self.data_store = DictDataStore(self.loop, self.peer_manager)
         self.ping_queue = PingQueue(self.loop, self)
         self.node_rpc = KademliaRPC(self, self.loop, self.peer_port)
         self.rpc_timeout = rpc_timeout
-        self._split_lock = asyncio.Lock(loop=self.loop)
+        self._split_lock = asyncio.Lock()
         self._to_remove: typing.Set['KademliaPeer'] = set()
         self._to_add: typing.Set['KademliaPeer'] = set()
-        self._wakeup_routing_task = asyncio.Event(loop=self.loop)
+        self._wakeup_routing_task = asyncio.Event()
         self.maintaing_routing_task: typing.Optional[asyncio.Task] = None
 
     @functools.lru_cache(128)
@@ -323,72 +361,10 @@ class KademliaProtocol(DatagramProtocol):
         return args, {}
 
     async def _add_peer(self, peer: 'KademliaPeer'):
-        if not peer.node_id:
-            log.warning("Tried adding a peer with no node id!")
-            return False
-        for my_peer in self.routing_table.get_peers():
-            if (my_peer.address, my_peer.udp_port) == (peer.address, peer.udp_port) and my_peer.node_id != peer.node_id:
-                self.routing_table.remove_peer(my_peer)
-                self.routing_table.join_buckets()
-        bucket_index = self.routing_table.kbucket_index(peer.node_id)
-        if self.routing_table.buckets[bucket_index].add_peer(peer):
-            return True
-
-        # The bucket is full; see if it can be split (by checking if its range includes the host node's node_id)
-        if self.routing_table.should_split(bucket_index, peer.node_id):
-            self.routing_table.split_bucket(bucket_index)
-            # Retry the insertion attempt
-            result = await self._add_peer(peer)
-            self.routing_table.join_buckets()
-            return result
-        else:
-            # We can't split the k-bucket
-            #
-            # The 13 page kademlia paper specifies that the least recently contacted node in the bucket
-            # shall be pinged. If it fails to reply it is replaced with the new contact. If the ping is successful
-            # the new contact is ignored and not added to the bucket (sections 2.2 and 2.4).
-            #
-            # A reasonable extension to this is BEP 0005, which extends the above:
-            #
-            #    Not all nodes that we learn about are equal. Some are "good" and some are not.
-            #    Many nodes using the DHT are able to send queries and receive responses,
-            #    but are not able to respond to queries from other nodes. It is important that
-            #    each node's routing table must contain only known good nodes. A good node is
-            #    a node has responded to one of our queries within the last 15 minutes. A node
-            #    is also good if it has ever responded to one of our queries and has sent us a
-            #    query within the last 15 minutes. After 15 minutes of inactivity, a node becomes
-            #    questionable. Nodes become bad when they fail to respond to multiple queries
-            #    in a row. Nodes that we know are good are given priority over nodes with unknown status.
-            #
-            # When there are bad or questionable nodes in the bucket, the least recent is selected for
-            # potential replacement (BEP 0005). When all nodes in the bucket are fresh, the head (least recent)
-            # contact is selected as described in section 2.2 of the kademlia paper. In both cases the new contact
-            # is ignored if the pinged node replies.
-
-            not_good_contacts = self.routing_table.buckets[bucket_index].get_bad_or_unknown_peers()
-            not_recently_replied = []
-            for my_peer in not_good_contacts:
-                last_replied = self.peer_manager.get_last_replied(my_peer.address, my_peer.udp_port)
-                if not last_replied or last_replied + 60 < self.loop.time():
-                    not_recently_replied.append(my_peer)
-            if not_recently_replied:
-                to_replace = not_recently_replied[0]
-            else:
-                to_replace = self.routing_table.buckets[bucket_index].peers[0]
-                last_replied = self.peer_manager.get_last_replied(to_replace.address, to_replace.udp_port)
-                if last_replied and last_replied + 60 > self.loop.time():
-                    return False
-            log.debug("pinging %s:%s", to_replace.address, to_replace.udp_port)
-            try:
-                to_replace_rpc = self.get_rpc_peer(to_replace)
-                await to_replace_rpc.ping()
-                return False
-            except asyncio.TimeoutError:
-                log.debug("Replacing dead contact in bucket %i: %s:%i with %s:%i ", bucket_index,
-                          to_replace.address, to_replace.udp_port, peer.address, peer.udp_port)
-                if to_replace in self.routing_table.buckets[bucket_index]:
-                    self.routing_table.buckets[bucket_index].remove_peer(to_replace)
-                return await self._add_peer(peer)
+        async def probe(some_peer: 'KademliaPeer'):
+            rpc_peer = self.get_rpc_peer(some_peer)
+            await rpc_peer.ping()
+        return await self.routing_table.add_peer(peer, probe)
 
     def add_peer(self, peer: 'KademliaPeer'):
         if peer.node_id == self.node_id:
@@ -406,16 +382,15 @@ class KademliaProtocol(DatagramProtocol):
                 async with self._split_lock:
                     peer = self._to_remove.pop()
                     self.routing_table.remove_peer(peer)
-                    self.routing_table.join_buckets()
             while self._to_add:
                 async with self._split_lock:
                     await self._add_peer(self._to_add.pop())
-            await asyncio.gather(self._wakeup_routing_task.wait(), asyncio.sleep(.1, loop=self.loop), loop=self.loop)
+            await asyncio.gather(self._wakeup_routing_task.wait(), asyncio.sleep(.1))
             self._wakeup_routing_task.clear()
 
     def _handle_rpc(self, sender_contact: 'KademliaPeer', message: RequestDatagram):
-        assert sender_contact.node_id != self.node_id, (binascii.hexlify(sender_contact.node_id)[:8].decode(),
-                                                        binascii.hexlify(self.node_id)[:8].decode())
+        assert sender_contact.node_id != self.node_id, (sender_contact.node_id.hex()[:8],
+                                                        self.node_id.hex()[:8])
         method = message.method
         if method not in [b'ping', b'store', b'findNode', b'findValue']:
             raise AttributeError('Invalid method: %s' % message.method.decode())
@@ -447,11 +422,15 @@ class KademliaProtocol(DatagramProtocol):
 
     def handle_request_datagram(self, address: typing.Tuple[str, int], request_datagram: RequestDatagram):
         # This is an RPC method request
+        self.received_request_metric.labels(method=request_datagram.method).inc()
         self.peer_manager.report_last_requested(address[0], address[1])
-        try:
-            peer = self.routing_table.get_peer(request_datagram.node_id)
-        except IndexError:
-            peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1])
+        peer = self.routing_table.get_peer(request_datagram.node_id)
+        if not peer:
+            try:
+                peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1])
+            except ValueError as err:
+                log.warning("error replying to %s: %s", address[0], str(err))
+                return
         try:
             self._handle_rpc(peer, request_datagram)
             # if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
@@ -551,12 +530,12 @@ class KademliaProtocol(DatagramProtocol):
                     address[0], address[1], OLD_PROTOCOL_ERRORS[error_datagram.response]
                 )
 
-    def datagram_received(self, datagram: bytes, address: typing.Tuple[str, int]) -> None:  # pylint: disable=arguments-differ
+    def datagram_received(self, datagram: bytes, address: typing.Tuple[str, int]) -> None:  # pylint: disable=arguments-renamed
         try:
             message = decode_datagram(datagram)
-        except (ValueError, TypeError):
+        except (ValueError, TypeError, DecodeError):
             self.peer_manager.report_failure(address[0], address[1])
-            log.warning("Couldn't decode dht datagram from %s: %s", address, binascii.hexlify(datagram).decode())
+            log.warning("Couldn't decode dht datagram from %s: %s", address, datagram.hex())
             return
 
         if isinstance(message, RequestDatagram):
@@ -571,14 +550,19 @@ class KademliaProtocol(DatagramProtocol):
         self._send(peer, request)
         response_fut = self.sent_messages[request.rpc_id][1]
         try:
+            self.request_sent_metric.labels(method=request.method).inc()
+            start = time.perf_counter()
             response = await asyncio.wait_for(response_fut, self.rpc_timeout)
+            self.response_time_metric.labels(method=request.method).observe(time.perf_counter() - start)
             self.peer_manager.report_last_replied(peer.address, peer.udp_port)
+            self.request_success_metric.labels(method=request.method).inc()
             return response
         except asyncio.CancelledError:
             if not response_fut.done():
                 response_fut.cancel()
             raise
         except (asyncio.TimeoutError, RemoteException):
+            self.request_error_metric.labels(method=request.method).inc()
             self.peer_manager.report_failure(peer.address, peer.udp_port)
             if self.peer_manager.peer_is_good(peer) is False:
                 self.remove_peer(peer)
@@ -598,7 +582,7 @@ class KademliaProtocol(DatagramProtocol):
         if len(data) > constants.MSG_SIZE_LIMIT:
             log.warning("cannot send datagram larger than %i bytes (packet is %i bytes)",
                         constants.MSG_SIZE_LIMIT, len(data))
-            log.debug("Packet is too large to send: %s", binascii.hexlify(data[:3500]).decode())
+            log.debug("Packet is too large to send: %s", data[:3500].hex())
             raise ValueError(
                 f"cannot send datagram larger than {constants.MSG_SIZE_LIMIT} bytes (packet is {len(data)} bytes)"
             )
@@ -658,13 +642,13 @@ class KademliaProtocol(DatagramProtocol):
             res = await self.get_rpc_peer(peer).store(hash_value)
             if res != b"OK":
                 raise ValueError(res)
-            log.debug("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
+            log.debug("Stored %s to %s", hash_value.hex()[:8], peer)
             return peer.node_id, True
 
         try:
             return await __store()
         except asyncio.TimeoutError:
-            log.debug("Timeout while storing blob_hash %s at %s", binascii.hexlify(hash_value).decode()[:8], peer)
+            log.debug("Timeout while storing blob_hash %s at %s", hash_value.hex()[:8], peer)
             return peer.node_id, False
         except ValueError as err:
             log.error("Unexpected response: %s", err)

lbry/dht/protocol/routing_table.py

@@ -4,7 +4,11 @@ import logging
 import typing
 import itertools
 
+from prometheus_client import Gauge
+
+from lbry import utils
 from lbry.dht import constants
+from lbry.dht.error import RemoteException
 from lbry.dht.protocol.distance import Distance
 if typing.TYPE_CHECKING:
     from lbry.dht.peer import KademliaPeer, PeerManager
@@ -13,10 +17,20 @@ log = logging.getLogger(__name__)
 
 
 class KBucket:
-    """ Description - later
     """
+    Kademlia K-bucket implementation.
+    """
+    peer_in_routing_table_metric = Gauge(
+        "peers_in_routing_table", "Number of peers on routing table", namespace="dht_node",
+        labelnames=("scope",)
+    )
+    peer_with_x_bit_colliding_metric = Gauge(
+        "peer_x_bit_colliding", "Number of peers with at least X bits colliding with this node id",
+        namespace="dht_node", labelnames=("amount",)
+    )
 
-    def __init__(self, peer_manager: 'PeerManager', range_min: int, range_max: int, node_id: bytes):
+    def __init__(self, peer_manager: 'PeerManager', range_min: int, range_max: int,
+                 node_id: bytes, capacity: int = constants.K):
         """
         @param range_min: The lower boundary for the range in the n-bit ID
                          space covered by this k-bucket
@@ -24,12 +38,12 @@ class KBucket:
                          covered by this k-bucket
         """
         self._peer_manager = peer_manager
-        self.last_accessed = 0
         self.range_min = range_min
         self.range_max = range_max
         self.peers: typing.List['KademliaPeer'] = []
         self._node_id = node_id
         self._distance_to_self = Distance(node_id)
+        self.capacity = capacity
 
     def add_peer(self, peer: 'KademliaPeer') -> bool:
         """ Add contact to _contact list in the right order. This will move the
@@ -50,24 +64,25 @@ class KBucket:
             self.peers.append(peer)
             return True
         else:
-            for i in range(len(self.peers)):
+            for i, _ in enumerate(self.peers):
                 local_peer = self.peers[i]
                 if local_peer.node_id == peer.node_id:
                     self.peers.remove(local_peer)
                     self.peers.append(peer)
                     return True
-        if len(self.peers) < constants.K:
+        if len(self.peers) < self.capacity:
             self.peers.append(peer)
+            self.peer_in_routing_table_metric.labels("global").inc()
+            bits_colliding = utils.get_colliding_prefix_bits(peer.node_id, self._node_id)
+            self.peer_with_x_bit_colliding_metric.labels(amount=bits_colliding).inc()
             return True
         else:
             return False
-            # raise BucketFull("No space in bucket to insert contact")
 
     def get_peer(self, node_id: bytes) -> 'KademliaPeer':
         for peer in self.peers:
             if peer.node_id == node_id:
                 return peer
-        raise IndexError(node_id)
 
     def get_peers(self, count=-1, exclude_contact=None, sort_distance_to=None) -> typing.List['KademliaPeer']:
         """ Returns a list containing up to the first count number of contacts
@@ -124,6 +139,9 @@ class KBucket:
 
     def remove_peer(self, peer: 'KademliaPeer') -> None:
         self.peers.remove(peer)
+        self.peer_in_routing_table_metric.labels("global").dec()
+        bits_colliding = utils.get_colliding_prefix_bits(peer.node_id, self._node_id)
+        self.peer_with_x_bit_colliding_metric.labels(amount=bits_colliding).dec()
 
     def key_in_range(self, key: bytes) -> bool:
         """ Tests whether the specified key (i.e. node ID) is in the range
@@ -161,24 +179,36 @@ class TreeRoutingTable:
     version of the Kademlia paper, in section 2.4. It does, however, use the
     ping RPC-based k-bucket eviction algorithm described in section 2.2 of
     that paper.
+
+    BOOTSTRAP MODE: if set to True, we always add all peers. This is so a
+    bootstrap node does not get a bias towards its own node id and replies are
+    the best it can provide (joining peer knows its neighbors immediately).
+    Over time, this will need to be optimized so we use the disk as holding
+    everything in memory won't be feasible anymore.
+    See: https://github.com/bittorrent/bootstrap-dht
     """
+    bucket_in_routing_table_metric = Gauge(
+        "buckets_in_routing_table", "Number of buckets on routing table", namespace="dht_node",
+        labelnames=("scope",)
+    )
 
     def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', parent_node_id: bytes,
-                 split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX):
+                 split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX, is_bootstrap_node: bool = False):
         self._loop = loop
         self._peer_manager = peer_manager
         self._parent_node_id = parent_node_id
         self._split_buckets_under_index = split_buckets_under_index
         self.buckets: typing.List[KBucket] = [
             KBucket(
-                self._peer_manager, range_min=0, range_max=2 ** constants.HASH_BITS, node_id=self._parent_node_id
+                self._peer_manager, range_min=0, range_max=2 ** constants.HASH_BITS, node_id=self._parent_node_id,
+                capacity=1 << 32 if is_bootstrap_node else constants.K
             )
         ]
 
     def get_peers(self) -> typing.List['KademliaPeer']:
         return list(itertools.chain.from_iterable(map(lambda bucket: bucket.peers, self.buckets)))
 
-    def should_split(self, bucket_index: int, to_add: bytes) -> bool:
+    def _should_split(self, bucket_index: int, to_add: bytes) -> bool:
         #  https://stackoverflow.com/questions/32129978/highly-unbalanced-kademlia-routing-table/32187456#32187456
         if bucket_index < self._split_buckets_under_index:
             return True
@@ -203,39 +233,32 @@ class TreeRoutingTable:
         return []
 
     def get_peer(self, contact_id: bytes) -> 'KademliaPeer':
-        """
-        @raise IndexError: No contact with the specified contact ID is known
-                           by this node
-        """
-        return self.buckets[self.kbucket_index(contact_id)].get_peer(contact_id)
+        return self.buckets[self._kbucket_index(contact_id)].get_peer(contact_id)
 
     def get_refresh_list(self, start_index: int = 0, force: bool = False) -> typing.List[bytes]:
-        bucket_index = start_index
         refresh_ids = []
-        now = int(self._loop.time())
-        for bucket in self.buckets[start_index:]:
-            if force or now - bucket.last_accessed >= constants.REFRESH_INTERVAL:
-                to_search = self.midpoint_id_in_bucket_range(bucket_index)
-                refresh_ids.append(to_search)
-            bucket_index += 1
+        for offset, _ in enumerate(self.buckets[start_index:]):
+            refresh_ids.append(self._midpoint_id_in_bucket_range(start_index + offset))
+        # if we have 3 or fewer populated buckets get two random ids in the range of each to try and
+        # populate/split the buckets further
+        buckets_with_contacts = self.buckets_with_contacts()
+        if buckets_with_contacts <= 3:
+            for i in range(buckets_with_contacts):
+                refresh_ids.append(self._random_id_in_bucket_range(i))
+                refresh_ids.append(self._random_id_in_bucket_range(i))
         return refresh_ids
 
     def remove_peer(self, peer: 'KademliaPeer') -> None:
         if not peer.node_id:
             return
-        bucket_index = self.kbucket_index(peer.node_id)
+        bucket_index = self._kbucket_index(peer.node_id)
         try:
             self.buckets[bucket_index].remove_peer(peer)
+            self._join_buckets()
         except ValueError:
             return
 
-    def touch_kbucket(self, key: bytes) -> None:
-        self.touch_kbucket_by_index(self.kbucket_index(key))
-
-    def touch_kbucket_by_index(self, bucket_index: int):
-        self.buckets[bucket_index].last_accessed = int(self._loop.time())
-
-    def kbucket_index(self, key: bytes) -> int:
+    def _kbucket_index(self, key: bytes) -> int:
         i = 0
         for bucket in self.buckets:
             if bucket.key_in_range(key):
@@ -244,19 +267,19 @@ class TreeRoutingTable:
                 i += 1
         return i
 
-    def random_id_in_bucket_range(self, bucket_index: int) -> bytes:
+    def _random_id_in_bucket_range(self, bucket_index: int) -> bytes:
         random_id = int(random.randrange(self.buckets[bucket_index].range_min, self.buckets[bucket_index].range_max))
         return Distance(
             self._parent_node_id
         )(random_id.to_bytes(constants.HASH_LENGTH, 'big')).to_bytes(constants.HASH_LENGTH, 'big')
 
-    def midpoint_id_in_bucket_range(self, bucket_index: int) -> bytes:
+    def _midpoint_id_in_bucket_range(self, bucket_index: int) -> bytes:
         half = int((self.buckets[bucket_index].range_max - self.buckets[bucket_index].range_min) // 2)
         return Distance(self._parent_node_id)(
             int(self.buckets[bucket_index].range_min + half).to_bytes(constants.HASH_LENGTH, 'big')
         ).to_bytes(constants.HASH_LENGTH, 'big')
 
-    def split_bucket(self, old_bucket_index: int) -> None:
+    def _split_bucket(self, old_bucket_index: int) -> None:
         """ Splits the specified k-bucket into two new buckets which together
         cover the same range in the key/ID space
 
@@ -279,8 +302,9 @@ class TreeRoutingTable:
         # ...and remove them from the old bucket
         for contact in new_bucket.peers:
             old_bucket.remove_peer(contact)
+        self.bucket_in_routing_table_metric.labels("global").set(len(self.buckets))
 
-    def join_buckets(self):
+    def _join_buckets(self):
         if len(self.buckets) == 1:
             return
         to_pop = [i for i, bucket in enumerate(self.buckets) if len(bucket) == 0]
@@ -302,14 +326,8 @@ class TreeRoutingTable:
         elif can_go_higher:
             self.buckets[bucket_index_to_pop + 1].range_min = bucket.range_min
         self.buckets.remove(bucket)
-        return self.join_buckets()
-
-    def contact_in_routing_table(self, address_tuple: typing.Tuple[str, int]) -> bool:
-        for bucket in self.buckets:
-            for contact in bucket.get_peers(sort_distance_to=False):
-                if address_tuple[0] == contact.address and address_tuple[1] == contact.udp_port:
-                    return True
-        return False
+        self.bucket_in_routing_table_metric.labels("global").set(len(self.buckets))
+        return self._join_buckets()
 
     def buckets_with_contacts(self) -> int:
         count = 0
@@ -317,3 +335,70 @@ class TreeRoutingTable:
             if len(bucket) > 0:
                 count += 1
         return count
+
+    async def add_peer(self, peer: 'KademliaPeer', probe: typing.Callable[['KademliaPeer'], typing.Awaitable]):
+        if not peer.node_id:
+            log.warning("Tried adding a peer with no node id!")
+            return False
+        for my_peer in self.get_peers():
+            if (my_peer.address, my_peer.udp_port) == (peer.address, peer.udp_port) and my_peer.node_id != peer.node_id:
+                self.remove_peer(my_peer)
+                self._join_buckets()
+        bucket_index = self._kbucket_index(peer.node_id)
+        if self.buckets[bucket_index].add_peer(peer):
+            return True
+
+        # The bucket is full; see if it can be split (by checking if its range includes the host node's node_id)
+        if self._should_split(bucket_index, peer.node_id):
+            self._split_bucket(bucket_index)
+            # Retry the insertion attempt
+            result = await self.add_peer(peer, probe)
+            self._join_buckets()
+            return result
+        else:
+            # We can't split the k-bucket
+            #
+            # The 13 page kademlia paper specifies that the least recently contacted node in the bucket
+            # shall be pinged. If it fails to reply it is replaced with the new contact. If the ping is successful
+            # the new contact is ignored and not added to the bucket (sections 2.2 and 2.4).
+            #
+            # A reasonable extension to this is BEP 0005, which extends the above:
+            #
+            #    Not all nodes that we learn about are equal. Some are "good" and some are not.
+            #    Many nodes using the DHT are able to send queries and receive responses,
+            #    but are not able to respond to queries from other nodes. It is important that
+            #    each node's routing table must contain only known good nodes. A good node is
+            #    a node has responded to one of our queries within the last 15 minutes. A node
+            #    is also good if it has ever responded to one of our queries and has sent us a
+            #    query within the last 15 minutes. After 15 minutes of inactivity, a node becomes
+            #    questionable. Nodes become bad when they fail to respond to multiple queries
+            #    in a row. Nodes that we know are good are given priority over nodes with unknown status.
+            #
+            # When there are bad or questionable nodes in the bucket, the least recent is selected for
+            # potential replacement (BEP 0005). When all nodes in the bucket are fresh, the head (least recent)
+            # contact is selected as described in section 2.2 of the kademlia paper. In both cases the new contact
+            # is ignored if the pinged node replies.
+
+            not_good_contacts = self.buckets[bucket_index].get_bad_or_unknown_peers()
+            not_recently_replied = []
+            for my_peer in not_good_contacts:
+                last_replied = self._peer_manager.get_last_replied(my_peer.address, my_peer.udp_port)
+                if not last_replied or last_replied + 60 < self._loop.time():
+                    not_recently_replied.append(my_peer)
+            if not_recently_replied:
+                to_replace = not_recently_replied[0]
+            else:
+                to_replace = self.buckets[bucket_index].peers[0]
+                last_replied = self._peer_manager.get_last_replied(to_replace.address, to_replace.udp_port)
+                if last_replied and last_replied + 60 > self._loop.time():
+                    return False
+            log.debug("pinging %s:%s", to_replace.address, to_replace.udp_port)
+            try:
+                await probe(to_replace)
+                return False
+            except (asyncio.TimeoutError, RemoteException):
+                log.debug("Replacing dead contact in bucket %i: %s:%i with %s:%i ", bucket_index,
+                          to_replace.address, to_replace.udp_port, peer.address, peer.udp_port)
+                if to_replace in self.buckets[bucket_index]:
+                    self.buckets[bucket_index].remove_peer(to_replace)
+                return await self.add_peer(peer, probe)

lbry/dht/serialization/datagram.py

@@ -144,7 +144,7 @@ class ErrorDatagram(KademliaDatagramBase):
         self.response = response.decode()
 
 
-def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]:
+def _decode_datagram(datagram: bytes):
     msg_types = {
         REQUEST_TYPE: RequestDatagram,
         RESPONSE_TYPE: ResponseDatagram,
@@ -152,26 +152,36 @@ def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDa
     }
 
     primitive: typing.Dict = bdecode(datagram)
-    if primitive[0] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]:  # pylint: disable=unsubscriptable-object
-        datagram_type = primitive[0]  # pylint: disable=unsubscriptable-object
+
+    converted = {
+        str(k).encode() if not isinstance(k, bytes) else k: v for k, v in primitive.items()
+    }
+
+    if converted[b'0'] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]:  # pylint: disable=unsubscriptable-object
+        datagram_type = converted[b'0']  # pylint: disable=unsubscriptable-object
     else:
         raise ValueError("invalid datagram type")
     datagram_class = msg_types[datagram_type]
     decoded = {
-        k: primitive[i]  # pylint: disable=unsubscriptable-object
+        k: converted[str(i).encode()]  # pylint: disable=unsubscriptable-object
         for i, k in enumerate(datagram_class.required_fields)
-        if i in primitive  # pylint: disable=unsupported-membership-test
+        if str(i).encode() in converted  # pylint: disable=unsupported-membership-test
     }
     for i, _ in enumerate(OPTIONAL_FIELDS):
-        if i + OPTIONAL_ARG_OFFSET in primitive:
-            decoded[i + OPTIONAL_ARG_OFFSET] = primitive[i + OPTIONAL_ARG_OFFSET]
+        if str(i + OPTIONAL_ARG_OFFSET).encode() in converted:
+            decoded[i + OPTIONAL_ARG_OFFSET] = converted[str(i + OPTIONAL_ARG_OFFSET).encode()]
+    return decoded, datagram_class
+
+
+def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]:
+    decoded, datagram_class = _decode_datagram(datagram)
     return datagram_class(**decoded)
 
 
 def make_compact_ip(address: str) -> bytearray:
     compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), address.split('.'), bytearray())
     if len(compact_ip) != 4:
-        raise ValueError(f"invalid IPv4 length")
+        raise ValueError("invalid IPv4 length")
     return compact_ip
 
 
@@ -180,7 +190,7 @@ def make_compact_address(node_id: bytes, address: str, port: int) -> bytearray:
     if not 0 < port < 65536:
         raise ValueError(f'Invalid port: {port}')
     if len(node_id) != constants.HASH_BITS // 8:
-        raise ValueError(f"invalid node node_id length")
+        raise ValueError("invalid node node_id length")
     return compact_ip + port.to_bytes(2, 'big') + node_id
 
 
@@ -191,5 +201,5 @@ def decode_compact_address(compact_address: bytes) -> typing.Tuple[bytes, str, i
     if not 0 < port < 65536:
         raise ValueError(f'Invalid port: {port}')
     if len(node_id) != constants.HASH_BITS // 8:
-        raise ValueError(f"invalid node node_id length")
+        raise ValueError("invalid node node_id length")
     return node_id, address, port

lbry/error/README.md

@@ -34,6 +34,11 @@ Code | Name | Message
 **11x** | InputValue(ValueError) | Invalid argument value provided to command.
 111 | GenericInputValue | The value '{value}' for argument '{argument}' is not valid.
 112 | InputValueIsNone | None or null is not valid value for argument '{argument}'.
+113 | ConflictingInputValue | Only '{first_argument}' or '{second_argument}' is allowed, not both.
+114 | InputStringIsBlank | {argument} cannot be blank.
+115 | EmptyPublishedFile | Cannot publish empty file: {file_path}
+116 | MissingPublishedFile | File does not exist: {file_path}
+117 | InvalidStreamURL | Invalid LBRY stream URL: '{url}' -- When an URL cannot be downloaded, such as '@Channel/' or a collection
 **2xx** | Configuration | Configuration errors.
 201 | ConfigWrite | Cannot write configuration file '{path}'. -- When writing the default config fails on startup, such as due to permission issues.
 202 | ConfigRead | Cannot find provided configuration file '{path}'. -- Can't open the config file user provided via command line args.
@@ -51,12 +56,22 @@ Code | Name | Message
 405 | ChannelKeyNotFound | Channel signing key not found.
 406 | ChannelKeyInvalid | Channel signing key is out of date. -- For example, channel was updated but you don't have the updated key.
 407 | DataDownload | Failed to download blob. *generic*
+408 | PrivateKeyNotFound | Couldn't find private key for {key} '{value}'.
 410 | Resolve | Failed to resolve '{url}'.
 411 | ResolveTimeout | Failed to resolve '{url}' within the timeout.
-411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{claim_id(censor_hash)}'.
+411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{censor_id}'.
 420 | KeyFeeAboveMaxAllowed | {message}
 421 | InvalidPassword | Password is invalid.
 422 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version.
+423 | TooManyClaimSearchParameters | {key} cant have more than {limit} items.
+424 | AlreadyPurchased | You already have a purchase for claim_id '{claim_id_hex}'. Use --allow-duplicate-purchase flag to override.
+431 | ServerPaymentInvalidAddress | Invalid address from wallet server: '{address}' - skipping payment round.
+432 | ServerPaymentWalletLocked | Cannot spend funds with locked wallet, skipping payment round.
+433 | ServerPaymentFeeAboveMaxAllowed | Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.
+434 | WalletNotLoaded | Wallet {wallet_id} is not loaded.
+435 | WalletAlreadyLoaded | Wallet {wallet_path} is already loaded.
+436 | WalletNotFound | Wallet not found at {wallet_path}.
+437 | WalletAlreadyExists | Wallet {wallet_path} already exists, use `wallet_add` to load it.
 **5xx** | Blob | **Blobs**
 500 | BlobNotFound | Blob not found.
 501 | BlobPermissionDenied | Permission denied to read blob.

lbry/error/__init__.py

@@ -76,6 +76,45 @@ class InputValueIsNoneError(InputValueError):
         super().__init__(f"None or null is not valid value for argument '{argument}'.")
 
 
+class ConflictingInputValueError(InputValueError):
+
+    def __init__(self, first_argument, second_argument):
+        self.first_argument = first_argument
+        self.second_argument = second_argument
+        super().__init__(f"Only '{first_argument}' or '{second_argument}' is allowed, not both.")
+
+
+class InputStringIsBlankError(InputValueError):
+
+    def __init__(self, argument):
+        self.argument = argument
+        super().__init__(f"{argument} cannot be blank.")
+
+
+class EmptyPublishedFileError(InputValueError):
+
+    def __init__(self, file_path):
+        self.file_path = file_path
+        super().__init__(f"Cannot publish empty file: {file_path}")
+
+
+class MissingPublishedFileError(InputValueError):
+
+    def __init__(self, file_path):
+        self.file_path = file_path
+        super().__init__(f"File does not exist: {file_path}")
+
+
+class InvalidStreamURLError(InputValueError):
+    """
+    When an URL cannot be downloaded, such as '@Channel/' or a collection
+    """
+
+    def __init__(self, url):
+        self.url = url
+        super().__init__(f"Invalid LBRY stream URL: '{url}'")
+
+
 class ConfigurationError(BaseError):
     """
     Configuration errors.
@@ -199,6 +238,14 @@ class DataDownloadError(WalletError):
         super().__init__("Failed to download blob. *generic*")
 
 
+class PrivateKeyNotFoundError(WalletError):
+
+    def __init__(self, key, value):
+        self.key = key
+        self.value = value
+        super().__init__(f"Couldn't find private key for {key} '{value}'.")
+
+
 class ResolveError(WalletError):
 
     def __init__(self, url):
@@ -215,10 +262,11 @@ class ResolveTimeoutError(WalletError):
 
 class ResolveCensoredError(WalletError):
 
-    def __init__(self, url, censor_hash):
+    def __init__(self, url, censor_id, censor_row):
         self.url = url
-        self.censor_hash = censor_hash
-        super().__init__(f"Resolve of '{url}' was censored by channel with claim id '{claim_id(censor_hash)}'.")
+        self.censor_id = censor_id
+        self.censor_row = censor_row
+        super().__init__(f"Resolve of '{url}' was censored by channel with claim id '{censor_id}'.")
 
 
 class KeyFeeAboveMaxAllowedError(WalletError):
@@ -242,6 +290,73 @@ class IncompatibleWalletServerError(WalletError):
         super().__init__(f"'{server}:{port}' has an incompatibly old version.")
 
 
+class TooManyClaimSearchParametersError(WalletError):
+
+    def __init__(self, key, limit):
+        self.key = key
+        self.limit = limit
+        super().__init__(f"{key} cant have more than {limit} items.")
+
+
+class AlreadyPurchasedError(WalletError):
+    """
+    allow-duplicate-purchase flag to override.
+    """
+
+    def __init__(self, claim_id_hex):
+        self.claim_id_hex = claim_id_hex
+        super().__init__(f"You already have a purchase for claim_id '{claim_id_hex}'. Use")
+
+
+class ServerPaymentInvalidAddressError(WalletError):
+
+    def __init__(self, address):
+        self.address = address
+        super().__init__(f"Invalid address from wallet server: '{address}' - skipping payment round.")
+
+
+class ServerPaymentWalletLockedError(WalletError):
+
+    def __init__(self):
+        super().__init__("Cannot spend funds with locked wallet, skipping payment round.")
+
+
+class ServerPaymentFeeAboveMaxAllowedError(WalletError):
+
+    def __init__(self, daily_fee, max_fee):
+        self.daily_fee = daily_fee
+        self.max_fee = max_fee
+        super().__init__(f"Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.")
+
+
+class WalletNotLoadedError(WalletError):
+
+    def __init__(self, wallet_id):
+        self.wallet_id = wallet_id
+        super().__init__(f"Wallet {wallet_id} is not loaded.")
+
+
+class WalletAlreadyLoadedError(WalletError):
+
+    def __init__(self, wallet_path):
+        self.wallet_path = wallet_path
+        super().__init__(f"Wallet {wallet_path} is already loaded.")
+
+
+class WalletNotFoundError(WalletError):
+
+    def __init__(self, wallet_path):
+        self.wallet_path = wallet_path
+        super().__init__(f"Wallet not found at {wallet_path}.")
+
+
+class WalletAlreadyExistsError(WalletError):
+
+    def __init__(self, wallet_path):
+        self.wallet_path = wallet_path
+        super().__init__(f"Wallet {wallet_path} already exists, use `wallet_add` to load it.")
+
+
 class BlobError(BaseError):
     """
     **Blobs**

lbry/error/generate.py

@@ -63,7 +63,7 @@ class ErrorClass:
     @staticmethod
     def get_fields(args):
         if len(args) > 1:
-            return f''.join(f'\n{INDENT*2}self.{field} = {field}' for field in args[1:])
+            return ''.join(f'\n{INDENT*2}self.{field} = {field}' for field in args[1:])
         return ''
 
     @staticmethod

lbry/extras/cli.py

@@ -3,7 +3,6 @@ import sys
 import shutil
 import signal
 import pathlib
-import platform
 import json
 import asyncio
 import argparse
@@ -15,7 +14,6 @@ from aiohttp.web import GracefulExit
 from docopt import docopt
 
 from lbry import __version__ as lbrynet_version
-from lbry.extras.daemon.loggly_handler import get_loggly_handler
 from lbry.extras.daemon.daemon import Daemon
 from lbry.conf import Config, CLIConfig
 
@@ -103,7 +101,7 @@ class ArgumentParser(argparse.ArgumentParser):
         self._optionals.title = 'Options'
         if group_name is None:
             self.epilog = (
-                f"Run 'lbrynet COMMAND --help' for more information on a command or group."
+                "Run 'lbrynet COMMAND --help' for more information on a command or group."
             )
         else:
             self.epilog = (
@@ -228,6 +226,9 @@ def get_argument_parser():
 def ensure_directory_exists(path: str):
     if not os.path.isdir(path):
         pathlib.Path(path).mkdir(parents=True, exist_ok=True)
+    use_effective_ids = os.access in os.supports_effective_ids
+    if not os.access(path, os.W_OK, effective_ids=use_effective_ids):
+        raise PermissionError(f"The following directory is not writable: {path}")
 
 
 LOG_MODULES = 'lbry', 'aioupnp'
@@ -256,16 +257,8 @@ def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config
         else:
             logger.getChild('lbry').setLevel(logging.DEBUG)
 
-    if conf.share_usage_data:
-        loggly_handler = get_loggly_handler()
-        loggly_handler.setLevel(logging.ERROR)
-        logger.getChild('lbry').addHandler(loggly_handler)
-
 
 def run_daemon(args: argparse.Namespace, conf: Config):
-    if sys.version_info < (3, 8) and platform.system() == "Windows":
-        # TODO: remove after we move to requiring Python 3.8
-        asyncio.set_event_loop(asyncio.ProactorEventLoop())
     loop = asyncio.get_event_loop()
     if args.verbose is not None:
         loop.set_debug(True)

lbry/extras/daemon/analytics.py

@@ -18,6 +18,7 @@ DOWNLOAD_STARTED = 'Download Started'
 DOWNLOAD_ERRORED = 'Download Errored'
 DOWNLOAD_FINISHED = 'Download Finished'
 HEARTBEAT = 'Heartbeat'
+DISK_SPACE = 'Disk Space'
 CLAIM_ACTION = 'Claim Action'  # publish/create/update/abandon
 NEW_CHANNEL = 'New Channel'
 CREDITS_SENT = 'Credits Sent'
@@ -66,7 +67,7 @@ def _download_properties(conf: Config, external_ip: str, resolve_duration: float
         "node_rpc_timeout": conf.node_rpc_timeout,
         "peer_connect_timeout": conf.peer_connect_timeout,
         "blob_download_timeout": conf.blob_download_timeout,
-        "use_fixed_peers": len(conf.reflector_servers) > 0,
+        "use_fixed_peers": len(conf.fixed_peers) > 0,
         "fixed_peer_delay": fixed_peer_delay,
         "added_fixed_peers": added_fixed_peers,
         "active_peer_count": active_peer_count,
@@ -110,7 +111,6 @@ class AnalyticsManager:
         self.cookies = {}
         self.url = ANALYTICS_ENDPOINT
         self._write_key = utils.deobfuscate(ANALYTICS_TOKEN)
-        self._enabled = conf.share_usage_data
         self._tracked_data = collections.defaultdict(list)
         self.context = _make_context(system_info.get_platform())
         self.installation_id = installation_id
@@ -118,20 +118,24 @@ class AnalyticsManager:
         self.task: typing.Optional[asyncio.Task] = None
         self.external_ip: typing.Optional[str] = None
 
+    @property
+    def enabled(self):
+        return self.conf.share_usage_data
+
     @property
     def is_started(self):
         return self.task is not None
 
     async def start(self):
-        if self._enabled and self.task is None:
-            self.external_ip = await utils.get_external_ip()
+        if self.task is None:
             self.task = asyncio.create_task(self.run())
 
     async def run(self):
         while True:
-            await self._send_heartbeat()
+            if self.enabled:
+                self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
+                await self._send_heartbeat()
             await asyncio.sleep(1800)
-            self.external_ip = await utils.get_external_ip()
 
     def stop(self):
         if self.task is not None and not self.task.done():
@@ -154,7 +158,7 @@ class AnalyticsManager:
 
     async def track(self, event: typing.Dict):
         """Send a single tracking event"""
-        if self._enabled:
+        if self.enabled:
             log.debug('Sending track event: %s', event)
             await self._post(event)
 
@@ -166,6 +170,15 @@ class AnalyticsManager:
             })
         )
 
+    async def send_disk_space_used(self, storage_used, storage_limit, is_from_network_quota):
+        await self.track(
+            self._event(DISK_SPACE, {
+                'used': storage_used,
+                'limit': storage_limit,
+                'from_network_quota': is_from_network_quota
+            })
+        )
+
     async def send_server_startup(self):
         await self.track(self._event(SERVER_STARTUP))
 

lbry/extras/daemon/client.py

@@ -1,5 +1,5 @@
-from lbry.conf import Config
 from lbry.extras.cli import execute_command
+from lbry.conf import Config
 
 
 def daemon_rpc(conf: Config, method: str, **kwargs):

lbry/extras/daemon/comment_client.py

@@ -1,66 +0,0 @@
-import logging
-import time
-import hashlib
-import binascii
-
-import ecdsa
-from lbry import utils
-from lbry.crypto.hash import sha256
-from lbry.wallet.transaction import Output
-
-log = logging.getLogger(__name__)
-
-
-def get_encoded_signature(signature):
-    signature = signature.encode() if isinstance(signature, str) else signature
-    r = int(signature[:int(len(signature) / 2)], 16)
-    s = int(signature[int(len(signature) / 2):], 16)
-    return ecdsa.util.sigencode_der(r, s, len(signature) * 4)
-
-
-def cid2hash(claim_id: str) -> bytes:
-    return binascii.unhexlify(claim_id.encode())[::-1]
-
-
-def is_comment_signed_by_channel(comment: dict, channel: Output, abandon=False):
-    if isinstance(channel, Output):
-        try:
-            signing_field = comment['comment_id'] if abandon else comment['comment']
-            pieces = [
-                comment['signing_ts'].encode(),
-                cid2hash(comment['channel_id']),
-                signing_field.encode()
-            ]
-            return Output.is_signature_valid(
-                get_encoded_signature(comment['signature']),
-                sha256(b''.join(pieces)),
-                channel.claim.channel.public_key_bytes
-            )
-        except KeyError:
-            pass
-    return False
-
-
-def sign_comment(comment: dict, channel: Output, abandon=False):
-    timestamp = str(int(time.time()))
-    signing_field = comment['comment_id'] if abandon else comment['comment']
-    pieces = [timestamp.encode(), channel.claim_hash, signing_field.encode()]
-    digest = sha256(b''.join(pieces))
-    signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
-    comment.update({
-        'signature': binascii.hexlify(signature).decode(),
-        'signing_ts': timestamp
-    })
-
-
-async def jsonrpc_post(url: str, method: str, params: dict = None, **kwargs) -> any:
-    params = params or {}
-    params.update(kwargs)
-    json_body = {'jsonrpc': '2.0', 'id': None, 'method': method, 'params': params}
-    async with utils.aiohttp_request('POST', url, json=json_body) as response:
-        try:
-            result = await response.json()
-            return result['result'] if 'result' in result else result
-        except Exception as cte:
-            log.exception('Unable to decode response from server: %s', cte)
-            return await response.text()

lbry/extras/daemon/component.py

@@ -37,7 +37,7 @@ class Component(metaclass=ComponentType):
     def running(self):
         return self._running
 
-    async def get_status(self):
+    async def get_status(self): # pylint: disable=no-self-use
         return
 
     async def start(self):

lbry/extras/daemon/componentmanager.py

@@ -42,7 +42,7 @@ class ComponentManager:
         self.analytics_manager = analytics_manager
         self.component_classes = {}
         self.components = set()
-        self.started = asyncio.Event(loop=self.loop)
+        self.started = asyncio.Event()
         self.peer_manager = peer_manager or PeerManager(asyncio.get_event_loop_policy().get_event_loop())
 
         for component_name, component_class in self.default_component_classes.items():
@@ -118,7 +118,7 @@ class ComponentManager:
                 component._setup() for component in stage if not component.running
             ]
             if needing_start:
-                await asyncio.wait(needing_start)
+                await asyncio.wait(map(asyncio.create_task, needing_start))
         self.started.set()
 
     async def stop(self):
@@ -131,7 +131,7 @@ class ComponentManager:
                 component._stop() for component in stage if component.running
             ]
             if needing_stop:
-                await asyncio.wait(needing_stop)
+                await asyncio.wait(map(asyncio.create_task, needing_stop))
 
     def all_components_running(self, *component_names):
         """
@@ -158,11 +158,14 @@ class ComponentManager:
             for component in self.components
         }
 
-    def get_component(self, component_name):
+    def get_actual_component(self, component_name):
         for component in self.components:
             if component.component_name == component_name:
-                return component.component
+                return component
         raise NameError(component_name)
 
+    def get_component(self, component_name):
+        return self.get_actual_component(component_name).component
+
     def has_component(self, component_name):
         return any(component for component in self.components if component_name == component.component_name)

lbry/extras/daemon/components.py

@@ -4,6 +4,7 @@ import asyncio
 import logging
 import binascii
 import typing
+
 import base58
 
 from aioupnp import __version__ as aioupnp_version
@@ -15,12 +16,19 @@ from lbry.dht.node import Node
 from lbry.dht.peer import is_valid_public_ipv4
 from lbry.dht.blob_announcer import BlobAnnouncer
 from lbry.blob.blob_manager import BlobManager
+from lbry.blob.disk_space_manager import DiskSpaceManager
 from lbry.blob_exchange.server import BlobServer
+from lbry.stream.background_downloader import BackgroundDownloader
 from lbry.stream.stream_manager import StreamManager
+from lbry.file.file_manager import FileManager
 from lbry.extras.daemon.component import Component
 from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
 from lbry.extras.daemon.storage import SQLiteStorage
+from lbry.torrent.torrent_manager import TorrentManager
 from lbry.wallet import WalletManager
+from lbry.wallet.usage_payment import WalletServerPayer
+from lbry.torrent.tracker import TrackerClient
+from lbry.torrent.session import TorrentSession
 
 log = logging.getLogger(__name__)
 
@@ -29,12 +37,17 @@ log = logging.getLogger(__name__)
 DATABASE_COMPONENT = "database"
 BLOB_COMPONENT = "blob_manager"
 WALLET_COMPONENT = "wallet"
+WALLET_SERVER_PAYMENTS_COMPONENT = "wallet_server_payments"
 DHT_COMPONENT = "dht"
 HASH_ANNOUNCER_COMPONENT = "hash_announcer"
-STREAM_MANAGER_COMPONENT = "stream_manager"
+FILE_MANAGER_COMPONENT = "file_manager"
+DISK_SPACE_COMPONENT = "disk_space"
+BACKGROUND_DOWNLOADER_COMPONENT = "background_downloader"
 PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
 UPNP_COMPONENT = "upnp"
 EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager"
+TRACKER_ANNOUNCER_COMPONENT = "tracker_announcer_component"
+LIBTORRENT_COMPONENT = "libtorrent_component"
 
 
 class DatabaseComponent(Component):
@@ -50,7 +63,7 @@ class DatabaseComponent(Component):
 
     @staticmethod
     def get_current_db_revision():
-        return 14
+        return 15
 
     @property
     def revision_filename(self):
@@ -110,15 +123,17 @@ class WalletComponent(Component):
     async def get_status(self):
         if self.wallet_manager is None:
             return
-        session_pool = self.wallet_manager.ledger.network.session_pool
-        sessions = session_pool.sessions
+        is_connected = self.wallet_manager.ledger.network.is_connected
+        sessions = []
         connected = None
-        if self.wallet_manager.ledger.network.client:
-            addr_and_port = self.wallet_manager.ledger.network.client.server_address_and_port
-            if addr_and_port:
-                connected = f"{addr_and_port[0]}:{addr_and_port[1]}"
+        if is_connected:
+            addr, port = self.wallet_manager.ledger.network.client.server
+            connected = f"{addr}:{port}"
+            sessions.append(self.wallet_manager.ledger.network.client)
+
         result = {
             'connected': connected,
+            'connected_features': self.wallet_manager.ledger.network.server_features,
             'servers': [
                 {
                     'host': session.server[0],
@@ -127,8 +142,8 @@ class WalletComponent(Component):
                     'availability': session.available,
                 } for session in sessions
             ],
-            'known_servers': len(sessions),
-            'available_servers': len(list(session_pool.available_sessions))
+            'known_servers': len(self.wallet_manager.ledger.network.known_hubs),
+            'available_servers': 1 if is_connected else 0
         }
 
         if self.wallet_manager.ledger.network.remote_height:
@@ -140,7 +155,7 @@ class WalletComponent(Component):
                 progress = min(max(math.ceil(float(download_height) / float(target_height) * 100), 0), 100)
             else:
                 progress = 100
-            best_hash = self.wallet_manager.get_best_blockhash()
+            best_hash = await self.wallet_manager.get_best_blockhash()
             result.update({
                 'headers_synchronization_progress': progress,
                 'blocks': max(local_height, 0),
@@ -160,6 +175,34 @@ class WalletComponent(Component):
         self.wallet_manager = None
 
 
+class WalletServerPaymentsComponent(Component):
+    component_name = WALLET_SERVER_PAYMENTS_COMPONENT
+    depends_on = [WALLET_COMPONENT]
+
+    def __init__(self, component_manager):
+        super().__init__(component_manager)
+        self.usage_payment_service = WalletServerPayer(
+            max_fee=self.conf.max_wallet_server_fee, analytics_manager=self.component_manager.analytics_manager,
+        )
+
+    @property
+    def component(self) -> typing.Optional[WalletServerPayer]:
+        return self.usage_payment_service
+
+    async def start(self):
+        wallet_manager = self.component_manager.get_component(WALLET_COMPONENT)
+        await self.usage_payment_service.start(wallet_manager.ledger, wallet_manager.default_wallet)
+
+    async def stop(self):
+        await self.usage_payment_service.stop()
+
+    async def get_status(self):
+        return {
+            'max_fee': self.usage_payment_service.max_fee,
+            'running': self.usage_payment_service.running
+        }
+
+
 class BlobComponent(Component):
     component_name = BLOB_COMPONENT
     depends_on = [DATABASE_COMPONENT]
@@ -236,7 +279,7 @@ class DHTComponent(Component):
         external_ip = upnp_component.external_ip
         storage = self.component_manager.get_component(DATABASE_COMPONENT)
         if not external_ip:
-            external_ip = await utils.get_external_ip()
+            external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
             if not external_ip:
                 log.warning("failed to get external ip")
 
@@ -250,6 +293,7 @@ class DHTComponent(Component):
             peer_port=self.external_peer_port,
             rpc_timeout=self.conf.node_rpc_timeout,
             split_buckets_under_index=self.conf.split_buckets_under_index,
+            is_bootstrap_node=self.conf.is_bootstrap_node,
             storage=storage
         )
         self.dht_node.start(self.conf.network_interface, self.conf.known_dht_nodes)
@@ -288,23 +332,23 @@ class HashAnnouncerComponent(Component):
         }
 
 
-class StreamManagerComponent(Component):
-    component_name = STREAM_MANAGER_COMPONENT
+class FileManagerComponent(Component):
+    component_name = FILE_MANAGER_COMPONENT
     depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT]
 
     def __init__(self, component_manager):
         super().__init__(component_manager)
-        self.stream_manager: typing.Optional[StreamManager] = None
+        self.file_manager: typing.Optional[FileManager] = None
 
     @property
-    def component(self) -> typing.Optional[StreamManager]:
-        return self.stream_manager
+    def component(self) -> typing.Optional[FileManager]:
+        return self.file_manager
 
     async def get_status(self):
-        if not self.stream_manager:
+        if not self.file_manager:
             return
         return {
-            'managed_files': len(self.stream_manager.streams),
+            'managed_files': len(self.file_manager.get_filtered()),
         }
 
     async def start(self):
@@ -315,14 +359,148 @@ class StreamManagerComponent(Component):
             if self.component_manager.has_component(DHT_COMPONENT) else None
         log.info('Starting the file manager')
         loop = asyncio.get_event_loop()
-        self.stream_manager = StreamManager(
-            loop, self.conf, blob_manager, wallet, storage, node, self.component_manager.analytics_manager
+        self.file_manager = FileManager(
+            loop, self.conf, wallet, storage, self.component_manager.analytics_manager
         )
-        await self.stream_manager.start()
+        self.file_manager.source_managers['stream'] = StreamManager(
+            loop, self.conf, blob_manager, wallet, storage, node,
+        )
+        if self.component_manager.has_component(LIBTORRENT_COMPONENT):
+            torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT)
+            self.file_manager.source_managers['torrent'] = TorrentManager(
+                loop, self.conf, torrent, storage, self.component_manager.analytics_manager
+            )
+        await self.file_manager.start()
         log.info('Done setting up file manager')
 
     async def stop(self):
-        self.stream_manager.stop()
+        await self.file_manager.stop()
+
+
+class BackgroundDownloaderComponent(Component):
+    MIN_PREFIX_COLLIDING_BITS = 8
+    component_name = BACKGROUND_DOWNLOADER_COMPONENT
+    depends_on = [DATABASE_COMPONENT, BLOB_COMPONENT, DISK_SPACE_COMPONENT]
+
+    def __init__(self, component_manager):
+        super().__init__(component_manager)
+        self.background_task: typing.Optional[asyncio.Task] = None
+        self.download_loop_delay_seconds = 60
+        self.ongoing_download: typing.Optional[asyncio.Task] = None
+        self.space_manager: typing.Optional[DiskSpaceManager] = None
+        self.blob_manager: typing.Optional[BlobManager] = None
+        self.background_downloader: typing.Optional[BackgroundDownloader] = None
+        self.dht_node: typing.Optional[Node] = None
+        self.space_available: typing.Optional[int] = None
+
+    @property
+    def is_busy(self):
+        return bool(self.ongoing_download and not self.ongoing_download.done())
+
+    @property
+    def component(self) -> 'BackgroundDownloaderComponent':
+        return self
+
+    async def get_status(self):
+        return {'running': self.background_task is not None and not self.background_task.done(),
+                'available_free_space_mb': self.space_available,
+                'ongoing_download': self.is_busy}
+
+    async def download_blobs_in_background(self):
+        while True:
+            self.space_available = await self.space_manager.get_free_space_mb(True)
+            if not self.is_busy and self.space_available > 10:
+                self._download_next_close_blob_hash()
+            await asyncio.sleep(self.download_loop_delay_seconds)
+
+    def _download_next_close_blob_hash(self):
+        node_id = self.dht_node.protocol.node_id
+        for blob_hash in self.dht_node.stored_blob_hashes:
+            if blob_hash.hex() in self.blob_manager.completed_blob_hashes:
+                continue
+            if utils.get_colliding_prefix_bits(node_id, blob_hash) >= self.MIN_PREFIX_COLLIDING_BITS:
+                self.ongoing_download = asyncio.create_task(self.background_downloader.download_blobs(blob_hash.hex()))
+                return
+
+    async def start(self):
+        self.space_manager: DiskSpaceManager = self.component_manager.get_component(DISK_SPACE_COMPONENT)
+        if not self.component_manager.has_component(DHT_COMPONENT):
+            return
+        self.dht_node = self.component_manager.get_component(DHT_COMPONENT)
+        self.blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
+        storage = self.component_manager.get_component(DATABASE_COMPONENT)
+        self.background_downloader = BackgroundDownloader(self.conf, storage, self.blob_manager, self.dht_node)
+        self.background_task = asyncio.create_task(self.download_blobs_in_background())
+
+    async def stop(self):
+        if self.ongoing_download and not self.ongoing_download.done():
+            self.ongoing_download.cancel()
+        if self.background_task:
+            self.background_task.cancel()
+
+
+class DiskSpaceComponent(Component):
+    component_name = DISK_SPACE_COMPONENT
+    depends_on = [DATABASE_COMPONENT, BLOB_COMPONENT]
+
+    def __init__(self, component_manager):
+        super().__init__(component_manager)
+        self.disk_space_manager: typing.Optional[DiskSpaceManager] = None
+
+    @property
+    def component(self) -> typing.Optional[DiskSpaceManager]:
+        return self.disk_space_manager
+
+    async def get_status(self):
+        if self.disk_space_manager:
+            space_used = await self.disk_space_manager.get_space_used_mb(cached=True)
+            return {
+                'total_used_mb': space_used['total'],
+                'published_blobs_storage_used_mb': space_used['private_storage'],
+                'content_blobs_storage_used_mb': space_used['content_storage'],
+                'seed_blobs_storage_used_mb': space_used['network_storage'],
+                'running': self.disk_space_manager.running,
+            }
+        return {'space_used': '0', 'network_seeding_space_used': '0', 'running': False}
+
+    async def start(self):
+        db = self.component_manager.get_component(DATABASE_COMPONENT)
+        blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
+        self.disk_space_manager = DiskSpaceManager(
+            self.conf, db, blob_manager,
+            analytics=self.component_manager.analytics_manager
+        )
+        await self.disk_space_manager.start()
+
+    async def stop(self):
+        await self.disk_space_manager.stop()
+
+
+class TorrentComponent(Component):
+    component_name = LIBTORRENT_COMPONENT
+
+    def __init__(self, component_manager):
+        super().__init__(component_manager)
+        self.torrent_session = None
+
+    @property
+    def component(self) -> typing.Optional[TorrentSession]:
+        return self.torrent_session
+
+    async def get_status(self):
+        if not self.torrent_session:
+            return
+        return {
+            'running': True,  # TODO: what to return here?
+        }
+
+    async def start(self):
+        self.torrent_session = TorrentSession(asyncio.get_event_loop(), None)
+        await self.torrent_session.bind()  # TODO: specify host/port
+
+    async def stop(self):
+        if self.torrent_session:
+            await self.torrent_session.pause()
 
 
 class PeerProtocolServerComponent(Component):
@@ -373,7 +551,7 @@ class UPnPComponent(Component):
         while True:
             if now:
                 await self._maintain_redirects()
-            await asyncio.sleep(360, loop=self.component_manager.loop)
+            await asyncio.sleep(360)
 
     async def _maintain_redirects(self):
         # setup the gateway if necessary
@@ -382,8 +560,6 @@ class UPnPComponent(Component):
                 self.upnp = await UPnP.discover(loop=self.component_manager.loop)
                 log.info("found upnp gateway: %s", self.upnp.gateway.manufacturer_string)
             except Exception as err:
-                if isinstance(err, asyncio.CancelledError):  # TODO: remove when updated to 3.8
-                    raise
                 log.warning("upnp discovery failed: %s", err)
                 self.upnp = None
 
@@ -398,11 +574,15 @@ class UPnPComponent(Component):
                 pass
         if external_ip and not is_valid_public_ipv4(external_ip):
             log.warning("UPnP returned a private/reserved ip - %s, checking lbry.com fallback", external_ip)
-            external_ip = await utils.get_external_ip()
+            external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
         if self.external_ip and self.external_ip != external_ip:
             log.info("external ip changed from %s to %s", self.external_ip, external_ip)
         if external_ip:
             self.external_ip = external_ip
+            dht_component = self.component_manager.get_component(DHT_COMPONENT)
+            if dht_component:
+                dht_node = dht_component.component
+                dht_node.protocol.external_ip = external_ip
         # assert self.external_ip is not None   # TODO: handle going/starting offline
 
         if not self.upnp_redirects and self.upnp:  # setup missing redirects
@@ -456,13 +636,15 @@ class UPnPComponent(Component):
     async def start(self):
         log.info("detecting external ip")
         if not self.use_upnp:
-            self.external_ip = await utils.get_external_ip()
+            self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
             return
         success = False
         await self._maintain_redirects()
         if self.upnp:
-            if not self.upnp_redirects and not all([x in self.component_manager.skip_components for x in
-                                                    (DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT)]):
+            if not self.upnp_redirects and not all(
+                    x in self.component_manager.skip_components
+                    for x in (DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT)
+            ):
                 log.error("failed to setup upnp")
             else:
                 success = True
@@ -471,9 +653,9 @@ class UPnPComponent(Component):
         else:
             log.error("failed to setup upnp")
         if not self.external_ip:
-            self.external_ip = await utils.get_external_ip()
+            self.external_ip, probed_url = await utils.get_external_ip(self.conf.lbryum_servers)
             if self.external_ip:
-                log.info("detected external ip using lbry.com fallback")
+                log.info("detected external ip using %s fallback", probed_url)
         if self.component_manager.analytics_manager:
             self.component_manager.loop.create_task(
                 self.component_manager.analytics_manager.send_upnp_setup_success_fail(
@@ -489,7 +671,7 @@ class UPnPComponent(Component):
             log.info("Removing upnp redirects: %s", self.upnp_redirects)
             await asyncio.wait([
                 self.upnp.delete_port_mapping(port, protocol) for protocol, port in self.upnp_redirects.items()
-            ], loop=self.component_manager.loop)
+            ])
         if self._maintain_redirects_task and not self._maintain_redirects_task.done():
             self._maintain_redirects_task.cancel()
 
@@ -520,3 +702,49 @@ class ExchangeRateManagerComponent(Component):
 
     async def stop(self):
         self.exchange_rate_manager.stop()
+
+
+class TrackerAnnouncerComponent(Component):
+    component_name = TRACKER_ANNOUNCER_COMPONENT
+    depends_on = [FILE_MANAGER_COMPONENT]
+
+    def __init__(self, component_manager):
+        super().__init__(component_manager)
+        self.file_manager = None
+        self.announce_task = None
+        self.tracker_client: typing.Optional[TrackerClient] = None
+
+    @property
+    def component(self):
+        return self.tracker_client
+
+    @property
+    def running(self):
+        return self._running and self.announce_task and not self.announce_task.done()
+
+    async def announce_forever(self):
+        while True:
+            sleep_seconds = 60.0
+            announce_sd_hashes = []
+            for file in self.file_manager.get_filtered():
+                if not file.downloader:
+                    continue
+                announce_sd_hashes.append(bytes.fromhex(file.sd_hash))
+            await self.tracker_client.announce_many(*announce_sd_hashes)
+            await asyncio.sleep(sleep_seconds)
+
+    async def start(self):
+        node = self.component_manager.get_component(DHT_COMPONENT) \
+            if self.component_manager.has_component(DHT_COMPONENT) else None
+        node_id = node.protocol.node_id if node else None
+        self.tracker_client = TrackerClient(node_id, self.conf.tcp_port, lambda: self.conf.tracker_servers)
+        await self.tracker_client.start()
+        self.file_manager = self.component_manager.get_component(FILE_MANAGER_COMPONENT)
+        self.announce_task = asyncio.create_task(self.announce_forever())
+
+    async def stop(self):
+        self.file_manager = None
+        if self.announce_task and not self.announce_task.done():
+            self.announce_task.cancel()
+        self.announce_task = None
+        self.tracker_client.stop()

lbry/extras/daemon/exchange_rate_manager.py

@@ -2,9 +2,10 @@ import json
 import time
 import asyncio
 import logging
+from statistics import median
 from decimal import Decimal
 from typing import Optional, Iterable, Type
-from aiohttp.client_exceptions import ContentTypeError
+from aiohttp.client_exceptions import ContentTypeError, ClientConnectionError
 from lbry.error import InvalidExchangeRateResponseError, CurrencyConversionError
 from lbry.utils import aiohttp_request
 from lbry.wallet.dewies import lbc_to_dewies
@@ -58,9 +59,12 @@ class MarketFeed:
         raise NotImplementedError()
 
     async def get_response(self):
-        async with aiohttp_request('get', self.url, params=self.params, timeout=self.request_timeout) as response:
+        async with aiohttp_request(
+                'get', self.url, params=self.params,
+                timeout=self.request_timeout, headers={"User-Agent": "lbrynet"}
+        ) as response:
             try:
-                self._last_response = await response.json()
+                self._last_response = await response.json(content_type=None)
             except ContentTypeError as e:
                 self._last_response = {}
                 log.warning("Could not parse exchange rate response from %s: %s", self.name, e.message)
@@ -75,18 +79,21 @@ class MarketFeed:
             log.debug("Saving rate update %f for %s from %s", rate, self.market, self.name)
             self.rate = ExchangeRate(self.market, rate, int(time.time()))
             self.last_check = time.time()
-            self.event.set()
             return self.rate
-        except asyncio.CancelledError:
-            raise
         except asyncio.TimeoutError:
             log.warning("Timed out fetching exchange rate from %s.", self.name)
         except json.JSONDecodeError as e:
-            log.warning("Could not parse exchange rate response from %s: %s", self.name, e.doc)
+            msg = e.doc if '<html>' not in e.doc else 'unexpected content type.'
+            log.warning("Could not parse exchange rate response from %s: %s", self.name, msg)
+            log.debug(e.doc)
         except InvalidExchangeRateResponseError as e:
             log.warning(str(e))
+        except ClientConnectionError as e:
+            log.warning("Error trying to connect to exchange rate %s: %s", self.name, str(e))
         except Exception as e:
             log.exception("Exchange rate error (%s from %s):", self.market, self.name)
+        finally:
+            self.event.set()
 
     async def keep_updated(self):
         while True:
@@ -104,70 +111,92 @@ class MarketFeed:
         self.event.clear()
 
 
-class BittrexFeed(MarketFeed):
+class BaseBittrexFeed(MarketFeed):
     name = "Bittrex"
-    market = "BTCLBC"
-    url = "https://bittrex.com/api/v1.1/public/getmarkethistory"
-    params = {'market': 'BTC-LBC', 'count': 50}
+    market = None
+    url = None
     fee = 0.0025
 
+    def get_rate_from_response(self, json_response):
+        if 'lastTradeRate' not in json_response:
+            raise InvalidExchangeRateResponseError(self.name, 'result not found')
+        return 1.0 / float(json_response['lastTradeRate'])
+
+
+class BittrexBTCFeed(BaseBittrexFeed):
+    market = "BTCLBC"
+    url = "https://api.bittrex.com/v3/markets/LBC-BTC/ticker"
+
+
+class BittrexUSDFeed(BaseBittrexFeed):
+    market = "USDLBC"
+    url = "https://api.bittrex.com/v3/markets/LBC-USD/ticker"
+
+
+class BaseCoinExFeed(MarketFeed):
+    name = "CoinEx"
+    market = None
+    url = None
+
+    def get_rate_from_response(self, json_response):
+        if 'data' not in json_response or \
+           'ticker' not in json_response['data'] or \
+           'last' not in json_response['data']['ticker']:
+            raise InvalidExchangeRateResponseError(self.name, 'result not found')
+        return 1.0 / float(json_response['data']['ticker']['last'])
+
+
+class CoinExBTCFeed(BaseCoinExFeed):
+    market = "BTCLBC"
+    url = "https://api.coinex.com/v1/market/ticker?market=LBCBTC"
+
+
+class CoinExUSDFeed(BaseCoinExFeed):
+    market = "USDLBC"
+    url = "https://api.coinex.com/v1/market/ticker?market=LBCUSDT"
+
+
+class BaseHotbitFeed(MarketFeed):
+    name = "hotbit"
+    market = None
+    url = "https://api.hotbit.io/api/v1/market.last"
+
     def get_rate_from_response(self, json_response):
         if 'result' not in json_response:
             raise InvalidExchangeRateResponseError(self.name, 'result not found')
-        trades = json_response['result']
-        if len(trades) == 0:
-            raise InvalidExchangeRateResponseError(self.name, 'trades not found')
-        totals = sum([i['Total'] for i in trades])
-        qtys = sum([i['Quantity'] for i in trades])
-        if totals <= 0 or qtys <= 0:
-            raise InvalidExchangeRateResponseError(self.name, 'quantities were not positive')
-        vwap = totals / qtys
-        return float(1.0 / vwap)
+        return 1.0 / float(json_response['result'])
 
 
-class LBRYFeed(MarketFeed):
-    name = "lbry.com"
+class HotbitBTCFeed(BaseHotbitFeed):
     market = "BTCLBC"
-    url = "https://api.lbry.com/lbc/exchange_rate"
-
-    def get_rate_from_response(self, json_response):
-        if 'data' not in json_response:
-            raise InvalidExchangeRateResponseError(self.name, 'result not found')
-        return 1.0 / json_response['data']['lbc_btc']
+    params = {"market": "LBC/BTC"}
 
 
-class LBRYBTCFeed(LBRYFeed):
-    market = "USDBTC"
-
-    def get_rate_from_response(self, json_response):
-        if 'data' not in json_response:
-            raise InvalidExchangeRateResponseError(self.name, 'result not found')
-        return 1.0 / json_response['data']['btc_usd']
+class HotbitUSDFeed(BaseHotbitFeed):
+    market = "USDLBC"
+    params = {"market": "LBC/USDT"}
 
 
-class CryptonatorFeed(MarketFeed):
-    name = "cryptonator.com"
+class UPbitBTCFeed(MarketFeed):
+    name = "UPbit"
     market = "BTCLBC"
-    url = "https://api.cryptonator.com/api/ticker/btc-lbc"
+    url = "https://api.upbit.com/v1/ticker"
+    params = {"markets": "BTC-LBC"}
 
     def get_rate_from_response(self, json_response):
-        if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \
-                'success' not in json_response or json_response['success'] is not True:
+        if "error" in json_response or len(json_response) != 1 or 'trade_price' not in json_response[0]:
             raise InvalidExchangeRateResponseError(self.name, 'result not found')
-        return float(json_response['ticker']['price'])
-
-
-class CryptonatorBTCFeed(CryptonatorFeed):
-    market = "USDBTC"
-    url = "https://api.cryptonator.com/api/ticker/usd-btc"
+        return 1.0 / float(json_response[0]['trade_price'])
 
 
 FEEDS: Iterable[Type[MarketFeed]] = (
-    LBRYFeed,
-    LBRYBTCFeed,
-    BittrexFeed,
-    # CryptonatorFeed,
-    # CryptonatorBTCFeed,
+    BittrexBTCFeed,
+    BittrexUSDFeed,
+    CoinExBTCFeed,
+    CoinExUSDFeed,
+#    HotbitBTCFeed,
+#    HotbitUSDFeed,
+#    UPbitBTCFeed,
 )
 
 
@@ -191,20 +220,23 @@ class ExchangeRateManager:
             source.stop()
 
     def convert_currency(self, from_currency, to_currency, amount):
-        rates = [market.rate for market in self.market_feeds]
-        log.debug("Converting %f %s to %s, rates: %s", amount, from_currency, to_currency, rates)
+        log.debug(
+            "Converting %f %s to %s, rates: %s",
+            amount, from_currency, to_currency,
+            [market.rate for market in self.market_feeds]
+        )
         if from_currency == to_currency:
             return round(amount, 8)
 
+        rates = []
         for market in self.market_feeds:
             if (market.has_rate and market.is_online and
                     market.rate.currency_pair == (from_currency, to_currency)):
-                return round(amount * Decimal(market.rate.spot), 8)
-        for market in self.market_feeds:
-            if (market.has_rate and market.is_online and
-                    market.rate.currency_pair[0] == from_currency):
-                return round(self.convert_currency(
-                    market.rate.currency_pair[1], to_currency, amount * Decimal(market.rate.spot)), 8)
+                rates.append(market.rate.spot)
+
+        if rates:
+            return round(amount * Decimal(median(rates)), 8)
+
         raise CurrencyConversionError(
             f'Unable to convert {amount} from {from_currency} to {to_currency}')
 

lbry/extras/daemon/json_response_encoder.py

@@ -7,8 +7,10 @@ from json import JSONEncoder
 from google.protobuf.message import DecodeError
 
 from lbry.schema.claim import Claim
+from lbry.schema.support import Support
+from lbry.torrent.torrent_manager import TorrentSource
 from lbry.wallet import Wallet, Ledger, Account, Transaction, Output
-from lbry.wallet.bip32 import PubKey
+from lbry.wallet.bip32 import PublicKey
 from lbry.wallet.dewies import dewies_to_lbc
 from lbry.stream.managed_stream import ManagedStream
 
@@ -25,6 +27,8 @@ def encode_txo_doc():
         'address': "address of who can spend the txo",
         'confirmations': "number of confirmed blocks",
         'is_change': "payment to change address, only available when it can be determined",
+        'is_received': "true if txo was sent from external account to this account",
+        'is_spent': "true if txo is spent",
         'is_mine': "payment to one of your accounts, only available when it can be determined",
         'type': "one of 'claim', 'support' or 'purchase'",
         'name': "when type is 'claim' or 'support', this is the claim name",
@@ -106,7 +110,9 @@ def encode_file_doc():
         'metadata': '(dict) None if claim is not found else the claim metadata',
         'channel_claim_id': '(str) None if claim is not found or not signed',
         'channel_name': '(str) None if claim is not found or not signed',
-        'claim_name': '(str) None if claim is not found else the claim name'
+        'claim_name': '(str) None if claim is not found else the claim name',
+        'reflector_progress': '(int) reflector upload progress, 0 to 100',
+        'uploading_to_reflector': '(bool) set to True when currently uploading to reflector'
     }
 
 
@@ -117,12 +123,12 @@ class JSONResponseEncoder(JSONEncoder):
         self.ledger = ledger
         self.include_protobuf = include_protobuf
 
-    def default(self, obj):  # pylint: disable=method-hidden,arguments-differ,too-many-return-statements
+    def default(self, obj):  # pylint: disable=method-hidden,arguments-renamed,too-many-return-statements
         if isinstance(obj, Account):
             return self.encode_account(obj)
         if isinstance(obj, Wallet):
             return self.encode_wallet(obj)
-        if isinstance(obj, ManagedStream):
+        if isinstance(obj, (ManagedStream, TorrentSource)):
             return self.encode_file(obj)
         if isinstance(obj, Transaction):
             return self.encode_transaction(obj)
@@ -130,7 +136,9 @@ class JSONResponseEncoder(JSONEncoder):
             return self.encode_output(obj)
         if isinstance(obj, Claim):
             return self.encode_claim(obj)
-        if isinstance(obj, PubKey):
+        if isinstance(obj, Support):
+            return obj.to_dict()
+        if isinstance(obj, PublicKey):
             return obj.extended_key_string()
         if isinstance(obj, datetime):
             return obj.strftime("%Y%m%dT%H:%M:%S")
@@ -164,12 +172,22 @@ class JSONResponseEncoder(JSONEncoder):
             'amount': dewies_to_lbc(txo.amount),
             'address': txo.get_address(self.ledger) if txo.has_address else None,
             'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
-            'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None
+            'timestamp': self.ledger.headers.estimated_timestamp(tx_height)
         }
-        if txo.is_change is not None:
-            output['is_change'] = txo.is_change
-        if txo.is_my_account is not None:
-            output['is_mine'] = txo.is_my_account
+        if txo.is_spent is not None:
+            output['is_spent'] = txo.is_spent
+        if txo.is_my_output is not None:
+            output['is_my_output'] = txo.is_my_output
+        if txo.is_my_input is not None:
+            output['is_my_input'] = txo.is_my_input
+        if txo.sent_supports is not None:
+            output['sent_supports'] = dewies_to_lbc(txo.sent_supports)
+        if txo.sent_tips is not None:
+            output['sent_tips'] = dewies_to_lbc(txo.sent_tips)
+        if txo.received_tips is not None:
+            output['received_tips'] = dewies_to_lbc(txo.received_tips)
+        if txo.is_internal_transfer is not None:
+            output['is_internal_transfer'] = txo.is_internal_transfer
 
         if txo.script.is_claim_name:
             output['type'] = 'claim'
@@ -205,25 +223,26 @@ class JSONResponseEncoder(JSONEncoder):
                 output['claims'] = [self.encode_output(o) for o in txo.claims]
             if txo.reposted_claim is not None:
                 output['reposted_claim'] = self.encode_output(txo.reposted_claim)
-            if txo.script.is_claim_name or txo.script.is_update_claim:
-                try:
-                    output['value'] = txo.claim
+        if txo.script.is_claim_name or txo.script.is_update_claim or txo.script.is_support_claim_data:
+            try:
+                output['value'] = txo.signable
+                if self.include_protobuf:
+                    output['protobuf'] = hexlify(txo.signable.to_bytes())
+                if txo.purchase_receipt is not None:
+                    output['purchase_receipt'] = self.encode_output(txo.purchase_receipt)
+                if txo.script.is_claim_name or txo.script.is_update_claim:
                     output['value_type'] = txo.claim.claim_type
-                    if self.include_protobuf:
-                        output['protobuf'] = hexlify(txo.claim.to_bytes())
-                    if txo.purchase_receipt is not None:
-                        output['purchase_receipt'] = self.encode_output(txo.purchase_receipt)
                     if txo.claim.is_channel:
                         output['has_signing_key'] = txo.has_private_key
-                    if check_signature and txo.claim.is_signed:
-                        if txo.channel is not None:
-                            output['signing_channel'] = self.encode_output(txo.channel)
-                            output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
-                        else:
-                            output['signing_channel'] = {'channel_id': txo.claim.signing_channel_id}
-                            output['is_channel_signature_valid'] = False
-                except DecodeError:
-                    pass
+                if check_signature and txo.signable.is_signed:
+                    if txo.channel is not None:
+                        output['signing_channel'] = self.encode_output(txo.channel)
+                        output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
+                    else:
+                        output['signing_channel'] = {'channel_id': txo.signable.signing_channel_id}
+                        output['is_channel_signature_valid'] = False
+            except DecodeError:
+                pass
         return output
 
     def encode_claim_meta(self, meta):
@@ -232,7 +251,7 @@ class JSONResponseEncoder(JSONEncoder):
                 if isinstance(value, int):
                     meta[key] = dewies_to_lbc(value)
         if 0 < meta.get('creation_height', 0) <= self.ledger.headers.height:
-            meta['creation_timestamp'] = self.ledger.headers[meta['creation_height']]['timestamp']
+            meta['creation_timestamp'] = self.ledger.headers.estimated_timestamp(meta['creation_height'])
         return meta
 
     def encode_input(self, txi):
@@ -259,26 +278,32 @@ class JSONResponseEncoder(JSONEncoder):
         output_exists = managed_stream.output_file_exists
         tx_height = managed_stream.stream_claim_info.height
         best_height = self.ledger.headers.height
-        return {
-            'streaming_url': managed_stream.stream_url,
+        is_stream = hasattr(managed_stream, 'stream_hash')
+        if is_stream:
+            total_bytes_lower_bound = managed_stream.descriptor.lower_bound_decrypted_length()
+            total_bytes = managed_stream.descriptor.upper_bound_decrypted_length()
+        else:
+            total_bytes_lower_bound = total_bytes = managed_stream.torrent_length
+        result = {
+            'streaming_url': None,
             'completed': managed_stream.completed,
-            'file_name': managed_stream.file_name if output_exists else None,
-            'download_directory': managed_stream.download_directory if output_exists else None,
-            'download_path': managed_stream.full_path if output_exists else None,
+            'file_name': None,
+            'download_directory': None,
+            'download_path': None,
             'points_paid': 0.0,
             'stopped': not managed_stream.running,
-            'stream_hash': managed_stream.stream_hash,
-            'stream_name': managed_stream.descriptor.stream_name,
-            'suggested_file_name': managed_stream.descriptor.suggested_file_name,
-            'sd_hash': managed_stream.descriptor.sd_hash,
-            'mime_type': managed_stream.mime_type,
-            'key': managed_stream.descriptor.key,
-            'total_bytes_lower_bound': managed_stream.descriptor.lower_bound_decrypted_length(),
-            'total_bytes': managed_stream.descriptor.upper_bound_decrypted_length(),
+            'stream_hash': None,
+            'stream_name': None,
+            'suggested_file_name': None,
+            'sd_hash': None,
+            'mime_type': None,
+            'key': None,
+            'total_bytes_lower_bound': total_bytes_lower_bound,
+            'total_bytes': total_bytes,
             'written_bytes': managed_stream.written_bytes,
-            'blobs_completed': managed_stream.blobs_completed,
-            'blobs_in_stream': managed_stream.blobs_in_stream,
-            'blobs_remaining': managed_stream.blobs_remaining,
+            'blobs_completed': None,
+            'blobs_in_stream': None,
+            'blobs_remaining': None,
             'status': managed_stream.status,
             'claim_id': managed_stream.claim_id,
             'txid': managed_stream.txid,
@@ -294,8 +319,38 @@ class JSONResponseEncoder(JSONEncoder):
             'added_on': managed_stream.added_on,
             'height': tx_height,
             'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height,
-            'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None
+            'timestamp': self.ledger.headers.estimated_timestamp(tx_height),
+            'is_fully_reflected': False,
+            'reflector_progress': False,
+            'uploading_to_reflector': False
         }
+        if is_stream:
+            result.update({
+                'streaming_url': managed_stream.stream_url,
+                'stream_hash': managed_stream.stream_hash,
+                'stream_name': managed_stream.stream_name,
+                'suggested_file_name': managed_stream.suggested_file_name,
+                'sd_hash': managed_stream.descriptor.sd_hash,
+                'mime_type': managed_stream.mime_type,
+                'key': managed_stream.descriptor.key,
+                'blobs_completed': managed_stream.blobs_completed,
+                'blobs_in_stream': managed_stream.blobs_in_stream,
+                'blobs_remaining': managed_stream.blobs_remaining,
+                'is_fully_reflected': managed_stream.is_fully_reflected,
+                'reflector_progress': managed_stream.reflector_progress,
+                'uploading_to_reflector': managed_stream.uploading_to_reflector
+            })
+        else:
+            result.update({
+                'streaming_url': f'file://{managed_stream.full_path}',
+            })
+        if output_exists:
+            result.update({
+                'file_name': managed_stream.file_name,
+                'download_directory': managed_stream.download_directory,
+                'download_path': managed_stream.full_path,
+            })
+        return result
 
     def encode_claim(self, claim):
         encoded = getattr(claim, claim.claim_type).to_dict()

lbry/extras/daemon/loggly_handler.py

@@ -1,89 +0,0 @@
-import asyncio
-import json
-import logging.handlers
-import traceback
-
-from aiohttp.client_exceptions import ClientError
-import aiohttp
-from lbry import utils, __version__
-
-
-LOGGLY_TOKEN = 'BQEzZmMzLJHgAGxkBF00LGD0YGuyATVgAmqxAQEuAQZ2BQH4'
-
-
-class JsonFormatter(logging.Formatter):
-    """Format log records using json serialization"""
-
-    def __init__(self, **kwargs):
-        super().__init__()
-        self.attributes = kwargs
-
-    def format(self, record):
-        data = {
-            'loggerName': record.name,
-            'asciTime': self.formatTime(record),
-            'fileName': record.filename,
-            'functionName': record.funcName,
-            'levelNo': record.levelno,
-            'lineNo': record.lineno,
-            'levelName': record.levelname,
-            'message': record.getMessage(),
-        }
-        data.update(self.attributes)
-        if record.exc_info:
-            data['exc_info'] = self.formatException(record.exc_info)
-        return json.dumps(data)
-
-
-class HTTPSLogglyHandler(logging.Handler):
-    def __init__(self, loggly_token: str, fqdn=False, localname=None, facility=None, cookies=None):
-        super().__init__()
-        self.fqdn = fqdn
-        self.localname = localname
-        self.facility = facility
-        self.cookies = cookies or {}
-        self.url = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}".format(
-            token=utils.deobfuscate(loggly_token), tag='lbrynet-' + __version__
-        )
-        self._loop = asyncio.get_event_loop()
-        self._session = aiohttp.ClientSession()
-
-    @staticmethod
-    def get_full_message(record):
-        if record.exc_info:
-            return '\n'.join(traceback.format_exception(*record.exc_info))
-        else:
-            return record.getMessage()
-
-    async def _emit(self, record, retry=True):
-        data = self.format(record).encode()
-        try:
-            async with self._session.post(self.url, data=data,
-                                          cookies=self.cookies) as response:
-                self.cookies.update(response.cookies)
-        except ClientError:
-            if self._loop.is_running() and retry:
-                await self._session.close()
-                self._session = aiohttp.ClientSession()
-                return await self._emit(record, retry=False)
-
-    def emit(self, record):
-        try:
-            asyncio.ensure_future(self._emit(record), loop=self._loop)
-        except RuntimeError:  # TODO: use a second loop
-            print(f"\nfailed to send traceback to loggly, please file an issue with the following traceback:\n"
-                  f"{self.format(record)}")
-
-    def close(self):
-        super().close()
-        try:
-            loop = asyncio.get_event_loop()
-            loop.run_until_complete(self._session.close())
-        except RuntimeError:
-            pass
-
-
-def get_loggly_handler():
-    handler = HTTPSLogglyHandler(LOGGLY_TOKEN)
-    handler.setFormatter(JsonFormatter())
-    return handler

lbry/extras/daemon/migrator/dbmigrator.py

@@ -35,6 +35,10 @@ def migrate_db(conf, start, end):
             from .migrate12to13 import do_migration
         elif current == 13:
             from .migrate13to14 import do_migration
+        elif current == 14:
+            from .migrate14to15 import do_migration
+        elif current == 15:
+            from .migrate15to16 import do_migration
         else:
             raise Exception(f"DB migration of version {current} to {current+1} is not available")
         try:

lbry/extras/daemon/migrator/migrate14to15.py

@@ -0,0 +1,16 @@
+import os
+import sqlite3
+
+
+def do_migration(conf):
+    db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
+    connection = sqlite3.connect(db_path)
+    cursor = connection.cursor()
+
+    cursor.executescript("""
+        alter table blob add column added_on integer not null default 0;
+        alter table blob add column is_mine integer not null default 1;
+    """)
+
+    connection.commit()
+    connection.close()

lbry/extras/daemon/migrator/migrate15to16.py

@@ -0,0 +1,17 @@
+import os
+import sqlite3
+
+
+def do_migration(conf):
+    db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
+    connection = sqlite3.connect(db_path)
+    cursor = connection.cursor()
+
+    cursor.executescript("""
+        update blob set should_announce=0
+        where should_announce=1 and 
+        blob.blob_hash in (select stream_blob.blob_hash from stream_blob where position=0);
+    """)
+
+    connection.commit()
+    connection.close()

lbry/extras/daemon/migrator/migrate8to9.py

@@ -20,7 +20,7 @@ def do_migration(conf):
                            "left outer join blob b ON b.blob_hash=s.blob_hash order by s.position").fetchall()
     blobs_by_stream = {}
     for stream_hash, position, iv, blob_hash, blob_length in blobs:
-        blobs_by_stream.setdefault(stream_hash, []).append(BlobInfo(position, blob_length or 0, iv, blob_hash))
+        blobs_by_stream.setdefault(stream_hash, []).append(BlobInfo(position, blob_length or 0, iv, 0, blob_hash))
 
     for stream_name, stream_key, suggested_filename, sd_hash, stream_hash in streams:
         sd = StreamDescriptor(None, blob_dir, stream_name, stream_key, suggested_filename,

lbry/extras/daemon/security.py

@@ -0,0 +1,31 @@
+import logging
+from aiohttp import web
+
+log = logging.getLogger(__name__)
+
+
+def ensure_request_allowed(request, conf):
+    if is_request_allowed(request, conf):
+        return
+    if conf.allowed_origin:
+        log.warning(
+            "API requests with Origin '%s' are not allowed, "
+            "configuration 'allowed_origin' limits requests to: '%s'",
+            request.headers.get('Origin'), conf.allowed_origin
+        )
+    else:
+        log.warning(
+            "API requests with Origin '%s' are not allowed, "
+            "update configuration 'allowed_origin' to enable this origin.",
+            request.headers.get('Origin')
+        )
+    raise web.HTTPForbidden()
+
+
+def is_request_allowed(request, conf) -> bool:
+    origin = request.headers.get('Origin')
+    return (
+        origin is None or
+        origin == conf.allowed_origin or
+        conf.allowed_origin == '*'
+    )

lbry/extras/daemon/storage.py

@@ -9,7 +9,7 @@ from typing import Optional
 from lbry.wallet import SQLiteMixin
 from lbry.conf import Config
 from lbry.wallet.dewies import dewies_to_lbc, lbc_to_dewies
-from lbry.wallet.transaction import Transaction
+from lbry.wallet.transaction import Transaction, Output
 from lbry.schema.claim import Claim
 from lbry.dht.constants import DATA_EXPIRATION
 from lbry.blob.blob_info import BlobInfo
@@ -112,7 +112,7 @@ def _batched_select(transaction, query, parameters, batch_size=900):
 
 def _get_lbry_file_stream_dict(rowid, added_on, stream_hash, file_name, download_dir, data_rate, status,
                                sd_hash, stream_key, stream_name, suggested_file_name, claim, saved_file,
-                               raw_content_fee):
+                               raw_content_fee, fully_reflected):
     return {
         "rowid": rowid,
         "added_on": added_on,
@@ -129,7 +129,8 @@ def _get_lbry_file_stream_dict(rowid, added_on, stream_hash, file_name, download
         "saved_file": bool(saved_file),
         "content_fee": None if not raw_content_fee else Transaction(
             binascii.unhexlify(raw_content_fee)
-        )
+        ),
+        "fully_reflected": fully_reflected
     }
 
 
@@ -138,11 +139,14 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
     signed_claims = {}
     for (rowid, stream_hash, _, file_name, download_dir, data_rate, status, saved_file, raw_content_fee,
          added_on, _, sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in transaction.execute(
-             "select file.rowid, file.*, stream.*, c.* "
+             "select file.rowid, file.*, stream.*, c.*, "
+             "  case when (SELECT 1 FROM reflected_stream r WHERE r.sd_hash=stream.sd_hash) "
+             "      is null then 0 else 1 end as fully_reflected "
              "from file inner join stream on file.stream_hash=stream.stream_hash "
              "inner join content_claim cc on file.stream_hash=cc.stream_hash "
              "inner join claim c on cc.claim_outpoint=c.claim_outpoint "
              "order by c.rowid desc").fetchall():
+        claim_args, fully_reflected = tuple(claim_args[:-1]), claim_args[-1]
         claim = StoredContentClaim(*claim_args)
         if claim.channel_claim_id:
             if claim.channel_claim_id not in signed_claims:
@@ -152,7 +156,7 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
             _get_lbry_file_stream_dict(
                 rowid, added_on, stream_hash, file_name, download_dir, data_rate, status,
                 sd_hash, stream_key, stream_name, suggested_file_name, claim, saved_file,
-                raw_content_fee
+                raw_content_fee, fully_reflected
             )
         )
     for claim_name, claim_id in _batched_select(
@@ -166,8 +170,8 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
 def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descriptor: 'StreamDescriptor'):
     # add all blobs, except the last one, which is empty
     transaction.executemany(
-        "insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
-        ((blob.blob_hash, blob.length, 0, 0, "pending", 0, 0)
+        "insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
+        ((blob.blob_hash, blob.length, 0, 0, "pending", 0, 0, blob.added_on, blob.is_mine)
          for blob in (descriptor.blobs[:-1] if len(descriptor.blobs) > 1 else descriptor.blobs) + [sd_blob])
     ).fetchall()
     # associate the blobs to the stream
@@ -183,8 +187,8 @@ def store_stream(transaction: sqlite3.Connection, sd_blob: 'BlobFile', descripto
     ).fetchall()
     # ensure should_announce is set regardless if insert was ignored
     transaction.execute(
-        "update blob set should_announce=1 where blob_hash in (?, ?)",
-        (sd_blob.blob_hash, descriptor.blobs[0].blob_hash,)
+        "update blob set should_announce=1 where blob_hash in (?)",
+        (sd_blob.blob_hash,)
     ).fetchall()
 
 
@@ -238,7 +242,9 @@ class SQLiteStorage(SQLiteMixin):
                 should_announce integer not null default 0,
                 status text not null,
                 last_announced_time integer,
-                single_announce integer
+                single_announce integer,
+                added_on integer not null,
+                is_mine integer not null default 0
             );
 
             create table if not exists stream (
@@ -331,6 +337,7 @@ class SQLiteStorage(SQLiteMixin):
                 tcp_port integer,
                 unique (address, udp_port)
             );
+            create index if not exists blob_data on blob(blob_hash, blob_length, is_mine);
     """
 
     def __init__(self, conf: Config, path, loop=None, time_getter: typing.Optional[typing.Callable[[], float]] = None):
@@ -352,19 +359,19 @@ class SQLiteStorage(SQLiteMixin):
 
     # # # # # # # # # blob functions # # # # # # # # #
 
-    async def add_blobs(self, *blob_hashes_and_lengths: typing.Tuple[str, int], finished=False):
+    async def add_blobs(self, *blob_hashes_and_lengths: typing.Tuple[str, int, int, int], finished=False):
         def _add_blobs(transaction: sqlite3.Connection):
             transaction.executemany(
-                "insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)",
+                "insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
                 (
-                    (blob_hash, length, 0, 0, "pending" if not finished else "finished", 0, 0)
-                    for blob_hash, length in blob_hashes_and_lengths
+                    (blob_hash, length, 0, 0, "pending" if not finished else "finished", 0, 0, added_on, is_mine)
+                    for blob_hash, length, added_on, is_mine in blob_hashes_and_lengths
                 )
             ).fetchall()
             if finished:
                 transaction.executemany(
                     "update blob set status='finished' where blob.blob_hash=?", (
-                        (blob_hash, ) for blob_hash, _ in blob_hashes_and_lengths
+                        (blob_hash, ) for blob_hash, _, _, _ in blob_hashes_and_lengths
                     )
                 ).fetchall()
         return await self.db.run(_add_blobs)
@@ -374,6 +381,11 @@ class SQLiteStorage(SQLiteMixin):
             "select status from blob where blob_hash=?", blob_hash
         )
 
+    def set_announce(self, *blob_hashes):
+        return self.db.execute_fetchall(
+            "update blob set should_announce=1 where blob_hash in (?, ?)", blob_hashes
+        )
+
     def update_last_announced_blobs(self, blob_hashes: typing.List[str]):
         def _update_last_announced_blobs(transaction: sqlite3.Connection):
             last_announced = self.time_getter()
@@ -431,6 +443,62 @@ class SQLiteStorage(SQLiteMixin):
     def get_all_blob_hashes(self):
         return self.run_and_return_list("select blob_hash from blob")
 
+    async def get_stored_blobs(self, is_mine: bool, is_network_blob=False):
+        is_mine = 1 if is_mine else 0
+        if is_network_blob:
+            return await self.db.execute_fetchall(
+                "select blob.blob_hash, blob.blob_length, blob.added_on "
+                "from blob left join stream_blob using (blob_hash) "
+                "where stream_blob.stream_hash is null and blob.is_mine=? and blob.status='finished'"
+                "order by blob.blob_length desc, blob.added_on asc",
+                (is_mine,)
+            )
+
+        sd_blobs = await self.db.execute_fetchall(
+            "select blob.blob_hash, blob.blob_length, blob.added_on "
+            "from blob join stream on blob.blob_hash=stream.sd_hash join file using (stream_hash) "
+            "where blob.is_mine=? order by blob.added_on asc",
+            (is_mine,)
+        )
+        content_blobs = await self.db.execute_fetchall(
+            "select blob.blob_hash, blob.blob_length, blob.added_on "
+            "from blob join stream_blob using (blob_hash) cross join stream using (stream_hash)"
+            "cross join file using (stream_hash)"
+            "where blob.is_mine=? and blob.status='finished' order by blob.added_on asc, blob.blob_length asc",
+            (is_mine,)
+        )
+        return content_blobs + sd_blobs
+
+    async def get_stored_blob_disk_usage(self):
+        total, network_size, content_size, private_size = await self.db.execute_fetchone("""
+        select coalesce(sum(blob_length), 0) as total,
+               coalesce(sum(case when
+                   stream_blob.stream_hash is null
+               then blob_length else 0 end), 0) as network_storage,
+               coalesce(sum(case when
+                   stream_blob.blob_hash is not null and is_mine=0
+               then blob_length else 0 end), 0) as content_storage,
+               coalesce(sum(case when
+                   is_mine=1
+               then blob_length else 0 end), 0) as private_storage
+        from blob left join stream_blob using (blob_hash)
+        where blob_hash not in (select sd_hash from stream) and blob.status="finished"
+        """)
+        return {
+            'network_storage': network_size,
+            'content_storage': content_size,
+            'private_storage': private_size,
+            'total': total
+        }
+
+    async def update_blob_ownership(self, sd_hash, is_mine: bool):
+        is_mine = 1 if is_mine else 0
+        await self.db.execute_fetchall(
+            "update blob set is_mine = ? where blob_hash in ("
+            "   select blob_hash from blob natural join stream_blob natural join stream where sd_hash = ?"
+            ") OR blob_hash = ?", (is_mine, sd_hash, sd_hash)
+        )
+
     def sync_missing_blobs(self, blob_files: typing.Set[str]) -> typing.Awaitable[typing.Set[str]]:
         def _sync_blobs(transaction: sqlite3.Connection) -> typing.Set[str]:
             finished_blob_hashes = tuple(
@@ -466,7 +534,8 @@ class SQLiteStorage(SQLiteMixin):
         def _get_blobs_for_stream(transaction):
             crypt_blob_infos = []
             stream_blobs = transaction.execute(
-                "select blob_hash, position, iv from stream_blob where stream_hash=? "
+                "select s.blob_hash, s.position, s.iv, b.added_on "
+                "from stream_blob s left outer join blob b on b.blob_hash=s.blob_hash where stream_hash=? "
                 "order by position asc", (stream_hash, )
             ).fetchall()
             if only_completed:
@@ -486,9 +555,10 @@ class SQLiteStorage(SQLiteMixin):
             for blob_hash, length in lengths:
                 blob_length_dict[blob_hash] = length
 
-            for blob_hash, position, iv in stream_blobs:
+            current_time = time.time()
+            for blob_hash, position, iv, added_on in stream_blobs:
                 blob_length = blob_length_dict.get(blob_hash, 0)
-                crypt_blob_infos.append(BlobInfo(position, blob_length, iv, blob_hash))
+                crypt_blob_infos.append(BlobInfo(position, blob_length, iv, added_on or current_time, blob_hash))
                 if not blob_hash:
                     break
             return crypt_blob_infos
@@ -566,6 +636,10 @@ class SQLiteStorage(SQLiteMixin):
         log.debug("update file status %s -> %s", stream_hash, new_status)
         return self.db.execute_fetchall("update file set status=? where stream_hash=?", (new_status, stream_hash))
 
+    def stop_all_files(self):
+        log.debug("stopping all files")
+        return self.db.execute_fetchall("update file set status=?", ("stopped",))
+
     async def change_file_download_dir_and_file_name(self, stream_hash: str, download_dir: typing.Optional[str],
                                                      file_name: typing.Optional[str]):
         if not file_name or not download_dir:
@@ -613,7 +687,7 @@ class SQLiteStorage(SQLiteMixin):
             ).fetchall()
             download_dir = binascii.hexlify(self.conf.download_dir.encode()).decode()
             transaction.executemany(
-                f"update file set download_directory=? where stream_hash=?",
+                "update file set download_directory=? where stream_hash=?",
                 ((download_dir, stream_hash) for stream_hash in stream_hashes)
             ).fetchall()
         await self.db.run_with_foreign_keys_disabled(_recover)
@@ -719,10 +793,23 @@ class SQLiteStorage(SQLiteMixin):
 
         await self.db.run(_save_claims)
         if update_file_callbacks:
-            await asyncio.wait(update_file_callbacks)
+            await asyncio.wait(map(asyncio.create_task, update_file_callbacks))
         if claim_id_to_supports:
             await self.save_supports(claim_id_to_supports)
 
+    def save_claim_from_output(self, ledger, *outputs: Output):
+        return self.save_claims([{
+            "claim_id": output.claim_id,
+            "name": output.claim_name,
+            "amount": dewies_to_lbc(output.amount),
+            "address": output.get_address(ledger),
+            "txid": output.tx_ref.id,
+            "nout": output.position,
+            "value": output.claim,
+            "height": output.tx_ref.height,
+            "claim_sequence": -1,
+        } for output in outputs])
+
     def save_claims_for_resolve(self, claim_infos):
         to_save = {}
         for info in claim_infos:
@@ -736,7 +823,8 @@ class SQLiteStorage(SQLiteMixin):
         return self.save_claims(to_save.values())
 
     @staticmethod
-    def _save_content_claim(transaction, claim_outpoint, stream_hash):
+    def _save_content_claim(transaction, claim_outpoint, stream_hash=None, bt_infohash=None):
+        assert stream_hash or bt_infohash
         # get the claim id and serialized metadata
         claim_info = transaction.execute(
             "select claim_id, serialized_metadata from claim where claim_outpoint=?", (claim_outpoint,)
@@ -784,6 +872,19 @@ class SQLiteStorage(SQLiteMixin):
         if stream_hash in self.content_claim_callbacks:
             await self.content_claim_callbacks[stream_hash]()
 
+    async def save_torrent_content_claim(self, bt_infohash, claim_outpoint, length, name):
+        def _save_torrent(transaction):
+            transaction.execute(
+                "insert or replace into torrent values (?, NULL, ?, ?)", (bt_infohash, length, name)
+            ).fetchall()
+            transaction.execute(
+                "insert or replace into content_claim values (NULL, ?, ?)", (bt_infohash, claim_outpoint)
+            ).fetchall()
+        await self.db.run(_save_torrent)
+        # update corresponding ManagedEncryptedFileDownloader object
+        if bt_infohash in self.content_claim_callbacks:
+            await self.content_claim_callbacks[bt_infohash]()
+
     async def get_content_claim(self, stream_hash: str, include_supports: typing.Optional[bool] = True) -> typing.Dict:
         claims = await self.db.run(get_claims_from_stream_hashes, [stream_hash])
         claim = None
@@ -795,6 +896,10 @@ class SQLiteStorage(SQLiteMixin):
                 claim['effective_amount'] = calculate_effective_amount(claim['amount'], supports)
         return claim
 
+    async def get_content_claim_for_torrent(self, bt_infohash):
+        claims = await self.db.run(get_claims_from_torrent_info_hashes, [bt_infohash])
+        return claims[bt_infohash].as_dict() if claims else None
+
     # # # # # # # # # reflector functions # # # # # # # # #
 
     def update_reflected_stream(self, sd_hash, reflector_address, success=True):
@@ -826,6 +931,6 @@ class SQLiteStorage(SQLiteMixin):
             transaction.execute('delete from peer').fetchall()
             transaction.executemany(
                 'insert into peer(node_id, address, udp_port, tcp_port) values (?, ?, ?, ?)',
-                tuple([(binascii.hexlify(p.node_id), p.address, p.udp_port, p.tcp_port) for p in peers])
+                ((binascii.hexlify(p.node_id), p.address, p.udp_port, p.tcp_port) for p in peers)
             ).fetchall()
         return await self.db.run(_save_kademlia_peers)

lbry/file/file_manager.py

@@ -0,0 +1,307 @@
+import asyncio
+import logging
+import typing
+from typing import Optional
+from aiohttp.web import Request
+from lbry.error import ResolveError, DownloadSDTimeoutError, InsufficientFundsError
+from lbry.error import ResolveTimeoutError, DownloadDataTimeoutError, KeyFeeAboveMaxAllowedError
+from lbry.error import InvalidStreamURLError
+from lbry.stream.managed_stream import ManagedStream
+from lbry.torrent.torrent_manager import TorrentSource
+from lbry.utils import cache_concurrent
+from lbry.schema.url import URL
+from lbry.wallet.dewies import dewies_to_lbc
+from lbry.file.source_manager import SourceManager
+from lbry.file.source import ManagedDownloadSource
+from lbry.extras.daemon.storage import StoredContentClaim
+if typing.TYPE_CHECKING:
+    from lbry.conf import Config
+    from lbry.extras.daemon.analytics import AnalyticsManager
+    from lbry.extras.daemon.storage import SQLiteStorage
+    from lbry.wallet import WalletManager
+    from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
+
+log = logging.getLogger(__name__)
+
+
+class FileManager:
+    def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', wallet_manager: 'WalletManager',
+                 storage: 'SQLiteStorage', analytics_manager: Optional['AnalyticsManager'] = None):
+        self.loop = loop
+        self.config = config
+        self.wallet_manager = wallet_manager
+        self.storage = storage
+        self.analytics_manager = analytics_manager
+        self.source_managers: typing.Dict[str, SourceManager] = {}
+        self.started = asyncio.Event()
+
+    @property
+    def streams(self):
+        return self.source_managers['stream']._sources
+
+    async def create_stream(self, file_path: str, key: Optional[bytes] = None, **kwargs) -> ManagedDownloadSource:
+        if 'stream' in self.source_managers:
+            return await self.source_managers['stream'].create(file_path, key, **kwargs)
+        raise NotImplementedError
+
+    async def start(self):
+        await asyncio.gather(*(source_manager.start() for source_manager in self.source_managers.values()))
+        for manager in self.source_managers.values():
+            await manager.started.wait()
+        self.started.set()
+
+    async def stop(self):
+        for manager in self.source_managers.values():
+            # fixme: pop or not?
+            await manager.stop()
+        self.started.clear()
+
+    @cache_concurrent
+    async def download_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
+                                timeout: Optional[float] = None, file_name: Optional[str] = None,
+                                download_directory: Optional[str] = None,
+                                save_file: Optional[bool] = None, resolve_timeout: float = 3.0,
+                                wallet: Optional['Wallet'] = None) -> ManagedDownloadSource:
+
+        wallet = wallet or self.wallet_manager.default_wallet
+        timeout = timeout or self.config.download_timeout
+        start_time = self.loop.time()
+        resolved_time = None
+        stream = None
+        claim = None
+        error = None
+        outpoint = None
+        if save_file is None:
+            save_file = self.config.save_files
+        if file_name and not save_file:
+            save_file = True
+        if save_file:
+            download_directory = download_directory or self.config.download_dir
+        else:
+            download_directory = None
+
+        payment = None
+        try:
+            # resolve the claim
+            try:
+                if not URL.parse(uri).has_stream:
+                    raise InvalidStreamURLError(uri)
+            except ValueError:
+                raise InvalidStreamURLError(uri)
+            try:
+                resolved_result = await asyncio.wait_for(
+                    self.wallet_manager.ledger.resolve(
+                        wallet.accounts, [uri],
+                        include_purchase_receipt=True,
+                        include_is_my_output=True
+                    ), resolve_timeout
+                )
+            except asyncio.TimeoutError:
+                raise ResolveTimeoutError(uri)
+            except Exception as err:
+                log.exception("Unexpected error resolving stream:")
+                raise ResolveError(f"Unexpected error resolving stream: {str(err)}")
+            if 'error' in resolved_result:
+                raise ResolveError(f"Unexpected error resolving uri for download: {resolved_result['error']}")
+            if not resolved_result or uri not in resolved_result:
+                raise ResolveError(f"Failed to resolve stream at '{uri}'")
+            txo = resolved_result[uri]
+            if isinstance(txo, dict):
+                raise ResolveError(f"Failed to resolve stream at '{uri}': {txo}")
+            claim = txo.claim
+            outpoint = f"{txo.tx_ref.id}:{txo.position}"
+            resolved_time = self.loop.time() - start_time
+            await self.storage.save_claim_from_output(self.wallet_manager.ledger, txo)
+
+            ####################
+            # update or replace
+            ####################
+
+            if claim.stream.source.bt_infohash:
+                source_manager = self.source_managers['torrent']
+                existing = source_manager.get_filtered(bt_infohash=claim.stream.source.bt_infohash)
+            elif claim.stream.source.sd_hash:
+                source_manager = self.source_managers['stream']
+                existing = source_manager.get_filtered(sd_hash=claim.stream.source.sd_hash)
+            else:
+                raise ResolveError(f"There is nothing to download at {uri} - Source is unknown or unset")
+
+            # resume or update an existing stream, if the stream changed: download it and delete the old one after
+            to_replace, updated_stream = None, None
+            if existing and existing[0].claim_id != txo.claim_id:
+                raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {txo.claim_id}")
+            if existing:
+                log.info("claim contains a metadata only update to a stream we have")
+                if claim.stream.source.bt_infohash:
+                    await self.storage.save_torrent_content_claim(
+                        existing[0].identifier, outpoint, existing[0].torrent_length, existing[0].torrent_name
+                    )
+                    claim_info = await self.storage.get_content_claim_for_torrent(existing[0].identifier)
+                    existing[0].set_claim(claim_info, claim)
+                else:
+                    await self.storage.save_content_claim(
+                        existing[0].stream_hash, outpoint
+                    )
+                    await source_manager._update_content_claim(existing[0])
+                updated_stream = existing[0]
+            else:
+                existing_for_claim_id = self.get_filtered(claim_id=txo.claim_id)
+                if existing_for_claim_id:
+                    log.info("claim contains an update to a stream we have, downloading it")
+                    if save_file and existing_for_claim_id[0].output_file_exists:
+                        save_file = False
+                    if not claim.stream.source.bt_infohash:
+                        existing_for_claim_id[0].downloader.node = source_manager.node
+                    await existing_for_claim_id[0].start(timeout=timeout, save_now=save_file)
+                    if not existing_for_claim_id[0].output_file_exists and (
+                            save_file or file_name or download_directory):
+                        await existing_for_claim_id[0].save_file(
+                            file_name=file_name, download_directory=download_directory
+                        )
+                    to_replace = existing_for_claim_id[0]
+
+            # resume or update an existing stream, if the stream changed: download it and delete the old one after
+            if updated_stream:
+                log.info("already have stream for %s", uri)
+                if save_file and updated_stream.output_file_exists:
+                    save_file = False
+                if not claim.stream.source.bt_infohash:
+                    updated_stream.downloader.node = source_manager.node
+                await updated_stream.start(timeout=timeout, save_now=save_file)
+                if not updated_stream.output_file_exists and (save_file or file_name or download_directory):
+                    await updated_stream.save_file(
+                        file_name=file_name, download_directory=download_directory
+                    )
+                return updated_stream
+
+            ####################
+            # pay fee
+            ####################
+
+            needs_purchasing = (
+                not to_replace and
+                not txo.is_my_output and
+                txo.has_price and
+                not txo.purchase_receipt
+            )
+
+            if needs_purchasing:
+                payment = await self.wallet_manager.create_purchase_transaction(
+                    wallet.accounts, txo, exchange_rate_manager
+                )
+
+            ####################
+            # make downloader and wait for start
+            ####################
+            # temporary with fields we know so downloader can start. Missing fields are populated later.
+            stored_claim = StoredContentClaim(outpoint=outpoint, claim_id=txo.claim_id, name=txo.claim_name,
+                                              amount=txo.amount, height=txo.tx_ref.height,
+                                              serialized=claim.to_bytes().hex())
+
+            if not claim.stream.source.bt_infohash:
+                # fixme: this shouldnt be here
+                stream = ManagedStream(
+                    self.loop, self.config, source_manager.blob_manager, claim.stream.source.sd_hash,
+                    download_directory, file_name, ManagedStream.STATUS_RUNNING, content_fee=payment,
+                    analytics_manager=self.analytics_manager, claim=stored_claim
+                )
+                stream.downloader.node = source_manager.node
+            else:
+                stream = TorrentSource(
+                    self.loop, self.config, self.storage, identifier=claim.stream.source.bt_infohash,
+                    file_name=file_name, download_directory=download_directory or self.config.download_dir,
+                    status=ManagedStream.STATUS_RUNNING, claim=stored_claim, analytics_manager=self.analytics_manager,
+                    torrent_session=source_manager.torrent_session
+                )
+            log.info("starting download for %s", uri)
+
+            before_download = self.loop.time()
+            await stream.start(timeout, save_file)
+
+            ####################
+            # success case: delete to_replace if applicable, broadcast fee payment
+            ####################
+
+            if to_replace:  # delete old stream now that the replacement has started downloading
+                await source_manager.delete(to_replace)
+
+            if payment is not None:
+                await self.wallet_manager.broadcast_or_release(payment)
+                payment = None  # to avoid releasing in `finally` later
+                log.info("paid fee of %s for %s", dewies_to_lbc(stream.content_fee.outputs[0].amount), uri)
+                await self.storage.save_content_fee(stream.stream_hash, stream.content_fee)
+
+            source_manager.add(stream)
+
+            if not claim.stream.source.bt_infohash:
+                await self.storage.save_content_claim(stream.stream_hash, outpoint)
+            else:
+                await self.storage.save_torrent_content_claim(
+                    stream.identifier, outpoint, stream.torrent_length, stream.torrent_name
+                )
+                claim_info = await self.storage.get_content_claim_for_torrent(stream.identifier)
+                stream.set_claim(claim_info, claim)
+            if save_file:
+                await asyncio.wait_for(stream.save_file(), timeout - (self.loop.time() - before_download))
+            return stream
+        except asyncio.TimeoutError:
+            error = DownloadDataTimeoutError(stream.sd_hash)
+            raise error
+        except (Exception, asyncio.CancelledError) as err:  # forgive data timeout, don't delete stream
+            expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError,
+                        KeyFeeAboveMaxAllowedError, ResolveError, InvalidStreamURLError)
+            if isinstance(err, expected):
+                log.warning("Failed to download %s: %s", uri, str(err))
+            elif isinstance(err, asyncio.CancelledError):
+                pass
+            else:
+                log.exception("Unexpected error downloading stream:")
+            error = err
+            raise
+        finally:
+            if payment is not None:
+                # payment is set to None after broadcasting, if we're here an exception probably happened
+                await self.wallet_manager.ledger.release_tx(payment)
+            if self.analytics_manager and claim and claim.stream.source.bt_infohash:
+                # TODO: analytics for torrents
+                pass
+            elif self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or
+                                                                   stream.downloader.time_to_first_bytes))):
+                server = self.wallet_manager.ledger.network.client.server
+                self.loop.create_task(
+                    self.analytics_manager.send_time_to_first_bytes(
+                        resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id,
+                        uri, outpoint,
+                        None if not stream else len(stream.downloader.blob_downloader.active_connections),
+                        None if not stream else len(stream.downloader.blob_downloader.scores),
+                        None if not stream else len(stream.downloader.blob_downloader.connection_failures),
+                        False if not stream else stream.downloader.added_fixed_peers,
+                        self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay,
+                        None if not stream else stream.sd_hash,
+                        None if not stream else stream.downloader.time_to_descriptor,
+                        None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
+                        None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
+                        None if not stream else stream.downloader.time_to_first_bytes,
+                        None if not error else error.__class__.__name__,
+                        None if not error else str(error),
+                        None if not server else f"{server[0]}:{server[1]}"
+                    )
+                )
+
+    async def stream_partial_content(self, request: Request, sd_hash: str):
+        return await self.source_managers['stream'].stream_partial_content(request, sd_hash)
+
+    def get_filtered(self, *args, **kwargs) -> typing.List[ManagedDownloadSource]:
+        """
+        Get a list of filtered and sorted ManagedStream objects
+
+        :param sort_by: field to sort by
+        :param reverse: reverse sorting
+        :param comparison: comparison operator used for filtering
+        :param search_by: fields and values to filter by
+        """
+        return sum((manager.get_filtered(*args, **kwargs) for manager in self.source_managers.values()), [])
+
+    async def delete(self, source: ManagedDownloadSource, delete_file=False):
+        for manager in self.source_managers.values():
+            await manager.delete(source, delete_file)

lbry/file/source.py

@@ -0,0 +1,162 @@
+import os
+import asyncio
+import typing
+import logging
+import binascii
+from typing import Optional
+from lbry.utils import generate_id
+from lbry.extras.daemon.storage import StoredContentClaim
+
+if typing.TYPE_CHECKING:
+    from lbry.conf import Config
+    from lbry.extras.daemon.analytics import AnalyticsManager
+    from lbry.wallet.transaction import Transaction
+    from lbry.extras.daemon.storage import SQLiteStorage
+
+log = logging.getLogger(__name__)
+
+
+class ManagedDownloadSource:
+    STATUS_RUNNING = "running"
+    STATUS_STOPPED = "stopped"
+    STATUS_FINISHED = "finished"
+
+    SAVING_ID = 1
+    STREAMING_ID = 2
+
+    def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage', identifier: str,
+                 file_name: Optional[str] = None, download_directory: Optional[str] = None,
+                 status: Optional[str] = STATUS_STOPPED, claim: Optional[StoredContentClaim] = None,
+                 download_id: Optional[str] = None, rowid: Optional[int] = None,
+                 content_fee: Optional['Transaction'] = None,
+                 analytics_manager: Optional['AnalyticsManager'] = None,
+                 added_on: Optional[int] = None):
+        self.loop = loop
+        self.storage = storage
+        self.config = config
+        self.identifier = identifier
+        self.download_directory = download_directory
+        self._file_name = file_name
+        self._status = status
+        self.stream_claim_info = claim
+        self.download_id = download_id or binascii.hexlify(generate_id()).decode()
+        self.rowid = rowid
+        self.content_fee = content_fee
+        self.purchase_receipt = None
+        self._added_on = added_on
+        self.analytics_manager = analytics_manager
+        self.downloader = None
+
+        self.saving = asyncio.Event()
+        self.finished_writing = asyncio.Event()
+        self.started_writing = asyncio.Event()
+        self.finished_write_attempt = asyncio.Event()
+
+    # @classmethod
+    # async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', file_path: str,
+    #                  key: Optional[bytes] = None,
+    #                  iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedDownloadSource':
+    #     raise NotImplementedError()
+
+    async def start(self, timeout: Optional[float] = None, save_now: Optional[bool] = False):
+        raise NotImplementedError()
+
+    async def stop(self, finished: bool = False):
+        raise NotImplementedError()
+
+    async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
+        raise NotImplementedError()
+
+    async def stop_tasks(self):
+        raise NotImplementedError()
+
+    def set_claim(self, claim_info: typing.Dict, claim: 'Claim'):
+        self.stream_claim_info = StoredContentClaim(
+            f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
+            claim_info['name'], claim_info['amount'], claim_info['height'],
+            binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'],
+            claim_info['claim_sequence'], claim_info.get('channel_name')
+        )
+
+    # async def update_content_claim(self, claim_info: Optional[typing.Dict] = None):
+    #     if not claim_info:
+    #         claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash)
+    #     self.set_claim(claim_info, claim_info['value'])
+
+    @property
+    def file_name(self) -> Optional[str]:
+        return self._file_name
+
+    @property
+    def added_on(self) -> Optional[int]:
+        return self._added_on
+
+    @property
+    def status(self) -> str:
+        return self._status
+
+    @property
+    def completed(self):
+        raise NotImplementedError()
+
+    # @property
+    # def stream_url(self):
+    #     return f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash}
+
+    @property
+    def finished(self) -> bool:
+        return self.status == self.STATUS_FINISHED
+
+    @property
+    def running(self) -> bool:
+        return self.status == self.STATUS_RUNNING
+
+    @property
+    def claim_id(self) -> Optional[str]:
+        return None if not self.stream_claim_info else self.stream_claim_info.claim_id
+
+    @property
+    def txid(self) -> Optional[str]:
+        return None if not self.stream_claim_info else self.stream_claim_info.txid
+
+    @property
+    def nout(self) -> Optional[int]:
+        return None if not self.stream_claim_info else self.stream_claim_info.nout
+
+    @property
+    def outpoint(self) -> Optional[str]:
+        return None if not self.stream_claim_info else self.stream_claim_info.outpoint
+
+    @property
+    def claim_height(self) -> Optional[int]:
+        return None if not self.stream_claim_info else self.stream_claim_info.height
+
+    @property
+    def channel_claim_id(self) -> Optional[str]:
+        return None if not self.stream_claim_info else self.stream_claim_info.channel_claim_id
+
+    @property
+    def channel_name(self) -> Optional[str]:
+        return None if not self.stream_claim_info else self.stream_claim_info.channel_name
+
+    @property
+    def claim_name(self) -> Optional[str]:
+        return None if not self.stream_claim_info else self.stream_claim_info.claim_name
+
+    @property
+    def metadata(self) -> Optional[typing.Dict]:
+        return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict()
+
+    @property
+    def metadata_protobuf(self) -> bytes:
+        if self.stream_claim_info:
+            return binascii.hexlify(self.stream_claim_info.claim.to_bytes())
+
+    @property
+    def full_path(self) -> Optional[str]:
+        return os.path.join(self.download_directory, os.path.basename(self.file_name)) \
+            if self.file_name and self.download_directory else None
+
+    @property
+    def output_file_exists(self):
+        return os.path.isfile(self.full_path) if self.full_path else False

lbry/file/source_manager.py

@@ -0,0 +1,138 @@
+import os
+import asyncio
+import logging
+import typing
+from typing import Optional
+from lbry.file.source import ManagedDownloadSource
+if typing.TYPE_CHECKING:
+    from lbry.conf import Config
+    from lbry.extras.daemon.analytics import AnalyticsManager
+    from lbry.extras.daemon.storage import SQLiteStorage
+
+log = logging.getLogger(__name__)
+
+COMPARISON_OPERATORS = {
+    'eq': lambda a, b: a == b,
+    'ne': lambda a, b: a != b,
+    'g': lambda a, b: a > b,
+    'l': lambda a, b: a < b,
+    'ge': lambda a, b: a >= b,
+    'le': lambda a, b: a <= b,
+}
+
+
+class SourceManager:
+    filter_fields = {
+        'rowid',
+        'status',
+        'file_name',
+        'added_on',
+        'download_path',
+        'claim_name',
+        'claim_height',
+        'claim_id',
+        'outpoint',
+        'txid',
+        'nout',
+        'channel_claim_id',
+        'channel_name',
+        'completed'
+    }
+
+    set_filter_fields = {
+        "claim_ids": "claim_id",
+        "channel_claim_ids": "channel_claim_id",
+        "outpoints": "outpoint"
+    }
+
+    source_class = ManagedDownloadSource
+
+    def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage',
+                 analytics_manager: Optional['AnalyticsManager'] = None):
+        self.loop = loop
+        self.config = config
+        self.storage = storage
+        self.analytics_manager = analytics_manager
+        self._sources: typing.Dict[str, ManagedDownloadSource] = {}
+        self.started = asyncio.Event()
+
+    def add(self, source: ManagedDownloadSource):
+        self._sources[source.identifier] = source
+
+    async def remove(self, source: ManagedDownloadSource):
+        if source.identifier not in self._sources:
+            return
+        self._sources.pop(source.identifier)
+        await source.stop_tasks()
+
+    async def initialize_from_database(self):
+        raise NotImplementedError()
+
+    async def start(self):
+        await self.initialize_from_database()
+        self.started.set()
+
+    async def stop(self):
+        while self._sources:
+            _, source = self._sources.popitem()
+            await source.stop_tasks()
+        self.started.clear()
+
+    async def create(self, file_path: str, key: Optional[bytes] = None,
+                     iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedDownloadSource:
+        raise NotImplementedError()
+
+    async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
+        await self.remove(source)
+        if delete_file and source.output_file_exists:
+            os.remove(source.full_path)
+
+    def get_filtered(self, sort_by: Optional[str] = None, reverse: Optional[bool] = False,
+                     comparison: Optional[str] = None, **search_by) -> typing.List[ManagedDownloadSource]:
+        """
+        Get a list of filtered and sorted ManagedStream objects
+
+        :param sort_by: field to sort by
+        :param reverse: reverse sorting
+        :param comparison: comparison operator used for filtering
+        :param search_by: fields and values to filter by
+        """
+        if sort_by and sort_by not in self.filter_fields:
+            raise ValueError(f"'{sort_by}' is not a valid field to sort by")
+        if comparison and comparison not in COMPARISON_OPERATORS:
+            raise ValueError(f"'{comparison}' is not a valid comparison")
+        if 'full_status' in search_by:
+            del search_by['full_status']
+
+        for search in search_by:
+            if search not in self.filter_fields:
+                raise ValueError(f"'{search}' is not a valid search operation")
+
+        compare_sets = {}
+        if isinstance(search_by.get('claim_id'), list):
+            compare_sets['claim_ids'] = search_by.pop('claim_id')
+        if isinstance(search_by.get('outpoint'), list):
+            compare_sets['outpoints'] = search_by.pop('outpoint')
+        if isinstance(search_by.get('channel_claim_id'), list):
+            compare_sets['channel_claim_ids'] = search_by.pop('channel_claim_id')
+
+        if search_by or compare_sets:
+            comparison = comparison or 'eq'
+            streams = []
+            for stream in self._sources.values():
+                if compare_sets and not all(
+                        getattr(stream, self.set_filter_fields[set_search]) in val
+                        for set_search, val in compare_sets.items()):
+                    continue
+                if search_by and not all(
+                        COMPARISON_OPERATORS[comparison](getattr(stream, search), val)
+                        for search, val in search_by.items()):
+                    continue
+                streams.append(stream)
+        else:
+            streams = list(self._sources.values())
+        if sort_by:
+            streams.sort(key=lambda s: getattr(s, sort_by) or "")
+            if reverse:
+                streams.reverse()
+        return streams

lbry/file_analysis.py

@@ -3,10 +3,14 @@ import json
 import logging
 import os
 import pathlib
+import platform
 import re
 import shlex
 import shutil
+import subprocess
+from math import ceil
 
+import lbry.utils
 from lbry.conf import TranscodeConfig
 
 log = logging.getLogger(__name__)
@@ -14,55 +18,104 @@ log = logging.getLogger(__name__)
 
 class VideoFileAnalyzer:
 
+    def _replace_or_pop_env(self, variable):
+        if variable + '_ORIG' in self._env_copy:
+            self._env_copy[variable] = self._env_copy[variable + '_ORIG']
+        else:
+            self._env_copy.pop(variable, None)
+
     def __init__(self, conf: TranscodeConfig):
         self._conf = conf
         self._available_encoders = ""
-        self._ffmpeg_installed = False
-        self._which = None
+        self._ffmpeg_installed = None
+        self._which_ffmpeg = None
+        self._which_ffprobe = None
+        self._env_copy = dict(os.environ)
+        self._checked_ffmpeg = False
+        if lbry.utils.is_running_from_bundle():
+            # handle the situation where PyInstaller overrides our runtime environment:
+            self._replace_or_pop_env('LD_LIBRARY_PATH')
 
-    async def _execute(self, command, arguments):
-        args = shlex.split(arguments)
-        process = await asyncio.create_subprocess_exec(self._conf.ffmpeg_folder + command, *args,
-                                                       stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
-        stdout, stderr = await process.communicate()  # returns when the streams are closed
-        return stdout.decode(errors='replace') + stderr.decode(errors='replace'), process.returncode
-
-    async def _verify_executable(self, name):
+    @staticmethod
+    def _execute(command, environment):
+        # log.debug("Executing: %s", command)
         try:
-            version, code = await self._execute(name, "-version")
+            with subprocess.Popen(
+                    shlex.split(command) if platform.system() != 'Windows' else command,
+                    stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=environment
+            ) as process:
+                (stdout, stderr) = process.communicate()  # blocks until the process exits
+                return stdout.decode(errors='replace') + stderr.decode(errors='replace'), process.returncode
+        except subprocess.SubprocessError as e:
+            return str(e), -1
+
+    # This create_subprocess_exec call is broken in Windows Python 3.7, but it's prettier than what's here.
+    # The recommended fix is switching to ProactorEventLoop, but that breaks UDP in Linux Python 3.7.
+    # We work around that issue here by using run_in_executor. Check it again in Python 3.8.
+    async def _execute_ffmpeg(self, arguments):
+        arguments = self._which_ffmpeg + " " + arguments
+        return await asyncio.get_event_loop().run_in_executor(None, self._execute, arguments, self._env_copy)
+
+    async def _execute_ffprobe(self, arguments):
+        arguments = self._which_ffprobe + " " + arguments
+        return await asyncio.get_event_loop().run_in_executor(None, self._execute, arguments, self._env_copy)
+
+    async def _verify_executables(self):
+        try:
+            await self._execute_ffprobe("-version")
+            version, code = await self._execute_ffmpeg("-version")
         except Exception as e:
-            log.warning("Unable to run %s, but it was requested. Message: %s", name, str(e))
             code = -1
-            version = ""
-        if code != 0 or not version.startswith(name):
-            raise FileNotFoundError(f"Unable to locate or run {name}. Please install FFmpeg "
-                                    f"and ensure that it is callable via PATH or conf.ffmpeg_folder")
+            version = str(e)
+        if code != 0 or not version.startswith("ffmpeg"):
+            log.warning("Unable to run ffmpeg, but it was requested. Code: %d; Message: %s", code, version)
+            raise FileNotFoundError("Unable to locate or run ffmpeg or ffprobe. Please install FFmpeg "
+                                    "and ensure that it is callable via PATH or conf.ffmpeg_path")
+        log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which_ffmpeg)
         return version
 
+    @staticmethod
+    def _which_ffmpeg_and_ffmprobe(path):
+        return shutil.which("ffmpeg", path=path), shutil.which("ffprobe", path=path)
+
     async def _verify_ffmpeg_installed(self):
         if self._ffmpeg_installed:
             return
-        await self._verify_executable("ffprobe")
-        version = await self._verify_executable("ffmpeg")
-        self._which = shutil.which(f"{self._conf.ffmpeg_folder}ffmpeg")
-        self._ffmpeg_installed = True
-        log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which)
+        self._ffmpeg_installed = False
+        path = self._conf.ffmpeg_path
+        if hasattr(self._conf, "data_dir"):
+            path += os.path.pathsep + os.path.join(getattr(self._conf, "data_dir"), "ffmpeg", "bin")
+        path += os.path.pathsep + self._env_copy.get("PATH", "")
+        self._which_ffmpeg, self._which_ffprobe = await asyncio.get_running_loop().run_in_executor(
+            None, self._which_ffmpeg_and_ffmprobe, path
+        )
+        if not self._which_ffmpeg:
+            log.warning("Unable to locate ffmpeg executable. Path: %s", path)
+            raise FileNotFoundError(f"Unable to locate ffmpeg executable. Path: {path}")
+        if not self._which_ffprobe:
+            log.warning("Unable to locate ffprobe executable. Path: %s", path)
+            raise FileNotFoundError(f"Unable to locate ffprobe executable. Path: {path}")
+        if os.path.dirname(self._which_ffmpeg) != os.path.dirname(self._which_ffprobe):
+            log.warning("ffmpeg and ffprobe are in different folders!")
 
-    async def status(self, reset=False):
+        await self._verify_executables()
+        self._ffmpeg_installed = True
+
+    async def status(self, reset=False, recheck=False):
         if reset:
             self._available_encoders = ""
-            self._ffmpeg_installed = False
-            self._which = None
-
-        installed = True
-        try:
-            await self._verify_ffmpeg_installed()
-        except FileNotFoundError:
-            installed = False
-
+            self._ffmpeg_installed = None
+        if self._checked_ffmpeg and not recheck:
+            pass
+        elif self._ffmpeg_installed is None:
+            try:
+                await self._verify_ffmpeg_installed()
+            except FileNotFoundError:
+                pass
+            self._checked_ffmpeg = True
         return {
-            "available": installed,
-            "which": self._which,
+            "available": self._ffmpeg_installed,
+            "which": self._which_ffmpeg,
             "analyze_audio_volume": int(self._conf.volume_analysis_time) > 0
         }
 
@@ -70,9 +123,24 @@ class VideoFileAnalyzer:
     def _verify_container(scan_data: json):
         container = scan_data["format"]["format_name"]
         log.debug("   Detected container is %s", container)
-        if not {"webm", "mp4", "3gp", "ogg"}.intersection(container.split(",")):
+        splits = container.split(",")
+        if not {"webm", "mp4", "3gp", "ogg"}.intersection(splits):
             return "Container format is not in the approved list of WebM, MP4. " \
                    f"Actual: {container} [{scan_data['format']['format_long_name']}]"
+
+        if "matroska" in splits:
+            for stream in scan_data["streams"]:
+                if stream["codec_type"] == "video":
+                    codec = stream["codec_name"]
+                    if not {"vp8", "vp9", "av1"}.intersection(codec.split(",")):
+                        return "WebM format requires VP8/9 or AV1 video. " \
+                               f"Actual: {codec} [{stream['codec_long_name']}]"
+                elif stream["codec_type"] == "audio":
+                    codec = stream["codec_name"]
+                    if not {"vorbis", "opus"}.intersection(codec.split(",")):
+                        return "WebM format requires Vorbis or Opus audio. " \
+                               f"Actual: {codec} [{stream['codec_long_name']}]"
+
         return ""
 
     @staticmethod
@@ -92,24 +160,21 @@ class VideoFileAnalyzer:
 
         return ""
 
-    @staticmethod
-    def _verify_bitrate(scan_data: json):
-        if "bit_rate" not in scan_data["format"]:
+    def _verify_bitrate(self, scan_data: json, file_path):
+        bit_rate_max = float(self._conf.video_bitrate_maximum)
+        if bit_rate_max <= 0:
             return ""
 
-        bit_rate = float(scan_data["format"]["bit_rate"])
-        log.debug("   Detected bitrate is %s Mbps", str(bit_rate / 1000000.0))
-        pixels = -1.0
-        for stream in scan_data["streams"]:
-            if stream["codec_type"] == "video":
-                pieces = stream["r_frame_rate"].split('/', 1)
-                frame_rate = float(pieces[0]) if len(pieces) == 1 \
-                    else float(pieces[0]) / float(pieces[1])
-                pixels = max(pixels, float(stream["height"]) * float(stream["width"]) * frame_rate)
+        if "bit_rate" in scan_data["format"]:
+            bit_rate = float(scan_data["format"]["bit_rate"])
+        else:
+            bit_rate = os.stat(file_path).st_size / float(scan_data["format"]["duration"])
+        log.debug("   Detected bitrate is %s Mbps. Allowed max: %s Mbps",
+                  str(bit_rate / 1000000.0), str(bit_rate_max / 1000000.0))
 
-        if pixels > 0.0 and pixels / bit_rate < 3.0:
-            return "Bits per second is excessive for this data; this may impact web streaming performance. " \
-                   f"Actual: {str(bit_rate / 1000000.0)} Mbps"
+        if bit_rate > bit_rate_max:
+            return "The bit rate is above the configured maximum. Actual: " \
+                   f"{bit_rate / 1000000.0} Mbps; Allowed max: {bit_rate_max / 1000000.0} Mbps"
 
         return ""
 
@@ -118,7 +183,7 @@ class VideoFileAnalyzer:
         if {"webm", "ogg"}.intersection(container.split(",")):
             return ""
 
-        result, _ = await self._execute("ffprobe", f'-v debug "{video_file}"')
+        result, _ = await self._execute_ffprobe(f'-v debug "{video_file}"')
         match = re.search(r"Before avformat_find_stream_info.+?\s+seeks:(\d+)\s+", result)
         if match and int(match.group(1)) != 0:
             return "Video stream descriptors are not at the start of the file (the faststart flag was not used)."
@@ -134,6 +199,8 @@ class VideoFileAnalyzer:
             if not {"aac", "mp3", "flac", "vorbis", "opus"}.intersection(codec.split(",")):
                 return "Audio codec is not in the approved list of AAC, FLAC, MP3, Vorbis, and Opus. " \
                        f"Actual: {codec} [{stream['codec_long_name']}]"
+            if int(stream['sample_rate']) > 48000:
+                return "Sample rate out of range"
 
         return ""
 
@@ -146,8 +213,8 @@ class VideoFileAnalyzer:
         if not validate_volume:
             return ""
 
-        result, _ = await self._execute("ffmpeg", f'-i "{video_file}" -t {seconds} '
-                                        f'-af volumedetect -vn -sn -dn -f null "{os.devnull}"')
+        result, _ = await self._execute_ffmpeg(f'-i "{video_file}" -t {seconds} '
+                                               f'-af volumedetect -vn -sn -dn -f null "{os.devnull}"')
         try:
             mean_volume = float(re.search(r"mean_volume:\s+([-+]?\d*\.\d+|\d+)", result).group(1))
             max_volume = float(re.search(r"max_volume:\s+([-+]?\d*\.\d+|\d+)", result).group(1))
@@ -173,13 +240,16 @@ class VideoFileAnalyzer:
         # https://developers.google.com/media/vp9/settings/vod/
         return int(-0.011 * height + 40)
 
+    def _get_video_scaler(self):
+        return self._conf.video_scaler
+
     async def _get_video_encoder(self, scan_data):
         # use what the user said if it's there:
         # if it's not there, use h264 if we can because it's way faster than the others
         # if we don't have h264 use vp9; it's fairly compatible even though it's slow
 
         if not self._available_encoders:
-            self._available_encoders, _ = await self._execute("ffmpeg", "-encoders -v quiet")
+            self._available_encoders, _ = await self._execute_ffmpeg("-encoders -v quiet")
 
         encoder = self._conf.video_encoder.split(" ", 1)[0]
         if re.search(fr"^\s*V..... {encoder} ", self._available_encoders, re.MULTILINE):
@@ -213,7 +283,7 @@ class VideoFileAnalyzer:
 
         wants_opus = extension != "mp4"
         if not self._available_encoders:
-            self._available_encoders, _ = await self._execute("ffmpeg", "-encoders -v quiet")
+            self._available_encoders, _ = await self._execute_ffmpeg("-encoders -v quiet")
 
         encoder = self._conf.audio_encoder.split(" ", 1)[0]
         if wants_opus and 'opus' in encoder:
@@ -236,9 +306,6 @@ class VideoFileAnalyzer:
 
         raise Exception(f"The audio encoder is not available. Requested: {encoder or 'aac'}")
 
-    async def _get_volume_filter(self):
-        return self._conf.volume_filter if self._conf.volume_filter else "-af loudnorm"
-
     @staticmethod
     def _get_best_container_extension(scan_data, video_encoder):
         # the container is chosen by the video format
@@ -246,61 +313,93 @@ class VideoFileAnalyzer:
         # if we are vp8/vp9/av1 we want webm
         # use mp4 for anything else
 
-        if not video_encoder:  # not re-encoding video
-            for stream in scan_data["streams"]:
-                if stream["codec_type"] != "video":
-                    continue
-                codec = stream["codec_name"].split(",")
-                if "theora" in codec:
-                    return "ogg"
-                if {"vp8", "vp9", "av1"}.intersection(codec):
-                    return "webm"
+        if video_encoder:  # not re-encoding video
+            if "theora" in video_encoder:
+                return "ogv"
+            if re.search(r"vp[89x]|av1", video_encoder.split(" ", 1)[0]):
+                return "webm"
+            return "mp4"
+
+        for stream in scan_data["streams"]:
+            if stream["codec_type"] != "video":
+                continue
+            codec = stream["codec_name"].split(",")
+            if "theora" in codec:
+                return "ogv"
+            if {"vp8", "vp9", "av1"}.intersection(codec):
+                return "webm"
 
-        if "theora" in video_encoder:
-            return "ogg"
-        elif re.search(r"vp[89x]|av1", video_encoder.split(" ", 1)[0]):
-            return "webm"
         return "mp4"
 
     async def _get_scan_data(self, validate, file_path):
-        result, _ = await self._execute("ffprobe",
-                                        f'-v quiet -print_format json -show_format -show_streams "{file_path}"')
+        arguments = f'-v quiet -print_format json -show_format -show_streams "{file_path}"'
+        result, _ = await self._execute_ffprobe(arguments)
         try:
             scan_data = json.loads(result)
         except Exception as e:
             log.debug("Failure in JSON parsing ffprobe results. Message: %s", str(e))
-            if validate:
-                raise Exception(f'Invalid video file: {file_path}')
-            log.info("Unable to optimize %s . FFmpeg output was unreadable.", file_path)
-            return
+            raise ValueError(f'Absent or unreadable video file: {file_path}')
 
-        if "format" not in scan_data:
-            if validate:
-                raise FileNotFoundError(f'Unexpected or absent video file contents at: {file_path}')
-            log.info("Unable to optimize %s . FFmpeg output is missing the format section.", file_path)
-            return
+        if "format" not in scan_data or "duration" not in scan_data["format"]:
+            log.debug("Format data is missing from ffprobe results for: %s", file_path)
+            raise ValueError(f'Media file does not appear to contain video content: {file_path}')
+
+        if float(scan_data["format"]["duration"]) < 0.1:
+            log.debug("Media file appears to be an image: %s", file_path)
+            raise ValueError(f'Assuming image file at: {file_path}')
 
         return scan_data
 
-    async def verify_or_repair(self, validate, repair, file_path):
+    @staticmethod
+    def _build_spec(scan_data):
+        assert scan_data
+
+        duration = ceil(float(scan_data["format"]["duration"]))  # existence verified when scan_data made
+        width = -1
+        height = -1
+        for stream in scan_data["streams"]:
+            if stream["codec_type"] != "video":
+                continue
+            width = max(width, int(stream["width"]))
+            height = max(height, int(stream["height"]))
+
+        log.debug("   Detected duration: %d sec. with resolution: %d x %d", duration, width, height)
+
+        spec = {"duration": duration}
+        if height >= 0:
+            spec["height"] = height
+        if width >= 0:
+            spec["width"] = width
+        return spec
+
+    async def verify_or_repair(self, validate, repair, file_path, ignore_non_video=False):
         if not validate and not repair:
-            return file_path
+            return file_path, {}
+
+        if ignore_non_video and not file_path:
+            return file_path, {}
 
         await self._verify_ffmpeg_installed()
-        scan_data = await self._get_scan_data(validate, file_path)
+        try:
+            scan_data = await self._get_scan_data(validate, file_path)
+        except ValueError:
+            if ignore_non_video:
+                return file_path, {}
+            raise
 
         fast_start_msg = await self._verify_fast_start(scan_data, file_path)
         log.debug("Analyzing %s:", file_path)
+        spec = self._build_spec(scan_data)
         log.debug("   Detected faststart is %s", "false" if fast_start_msg else "true")
         container_msg = self._verify_container(scan_data)
-        bitrate_msg = self._verify_bitrate(scan_data)
+        bitrate_msg = self._verify_bitrate(scan_data, file_path)
         video_msg = self._verify_video_encoding(scan_data)
         audio_msg = self._verify_audio_encoding(scan_data)
         volume_msg = await self._verify_audio_volume(self._conf.volume_analysis_time, file_path)
         messages = [container_msg, bitrate_msg, fast_start_msg, video_msg, audio_msg, volume_msg]
 
         if not any(messages):
-            return file_path
+            return file_path, spec
 
         if not repair:
             errors = ["Streamability verification failed:"]
@@ -310,7 +409,7 @@ class VideoFileAnalyzer:
         # the plan for transcoding:
         # we have to re-encode the video if it is in a nonstandard format
         # we also re-encode if we are h264 but not yuv420p (both errors caught in video_msg)
-        # we also re-encode if our bitrate is too high
+        # we also re-encode if our bitrate or sample rate is too high
 
         try:
             transcode_command = [f'-i "{file_path}" -y -c:s copy -c:d copy -c:v']
@@ -319,29 +418,32 @@ class VideoFileAnalyzer:
             if video_msg or bitrate_msg:
                 video_encoder = await self._get_video_encoder(scan_data)
                 transcode_command.append(video_encoder)
+                # could do the scaling only if bitrate_msg, but if we're going to the effort to re-encode anyway...
+                transcode_command.append(self._get_video_scaler())
             else:
                 transcode_command.append("copy")
 
             transcode_command.append("-movflags +faststart -c:a")
-            path = pathlib.Path(file_path)
             extension = self._get_best_container_extension(scan_data, video_encoder)
 
             if audio_msg or volume_msg:
                 audio_encoder = await self._get_audio_encoder(extension)
                 transcode_command.append(audio_encoder)
-                if volume_msg:
-                    volume_filter = await self._get_volume_filter()
-                    transcode_command.append(volume_filter)
+                if volume_msg and self._conf.volume_filter:
+                    transcode_command.append(self._conf.volume_filter)
+                if audio_msg == "Sample rate out of range":
+                    transcode_command.append(" -ar 48000 ")
             else:
                 transcode_command.append("copy")
 
             # TODO: put it in a temp folder and delete it after we upload?
+            path = pathlib.Path(file_path)
             output = path.parent / f"{path.stem}_fixed.{extension}"
             transcode_command.append(f'"{output}"')
 
             ffmpeg_command = " ".join(transcode_command)
             log.info("Proceeding on transcode via: ffmpeg %s", ffmpeg_command)
-            result, code = await self._execute("ffmpeg", ffmpeg_command)
+            result, code = await self._execute_ffmpeg(ffmpeg_command)
             if code != 0:
                 raise Exception(f"Failure to complete the transcode command. Output: {result}")
         except Exception as e:
@@ -349,6 +451,6 @@ class VideoFileAnalyzer:
                 raise
             log.info("Unable to transcode %s . Message: %s", file_path, str(e))
             # TODO: delete partial output file here if it exists?
-            return file_path
+            return file_path, spec
 
-        return str(output)
+        return str(output), spec

lbry/prometheus.py

@@ -0,0 +1,68 @@
+import time
+import logging
+import asyncio
+import asyncio.tasks
+from aiohttp import web
+from prometheus_client import generate_latest as prom_generate_latest
+from prometheus_client import Counter, Histogram, Gauge
+
+
+PROBES_IN_FLIGHT = Counter("probes_in_flight", "Number of loop probes in flight", namespace='asyncio')
+PROBES_FINISHED = Counter("probes_finished", "Number of finished loop probes", namespace='asyncio')
+PROBE_TIMES = Histogram("probe_times", "Loop probe times", namespace='asyncio')
+TASK_COUNT = Gauge("running_tasks", "Number of running tasks", namespace='asyncio')
+
+
+def get_loop_metrics(delay=1):
+    loop = asyncio.get_event_loop()
+
+    def callback(started):
+        PROBE_TIMES.observe(time.perf_counter() - started - delay)
+        PROBES_FINISHED.inc()
+
+    async def monitor_loop_responsiveness():
+        while True:
+            now = time.perf_counter()
+            loop.call_later(delay, callback, now)
+            PROBES_IN_FLIGHT.inc()
+            TASK_COUNT.set(len(asyncio.tasks._all_tasks))
+            await asyncio.sleep(delay)
+
+    return loop.create_task(monitor_loop_responsiveness())
+
+
+class PrometheusServer:
+    def __init__(self, logger=None):
+        self.runner = None
+        self.logger = logger or logging.getLogger(__name__)
+        self._monitor_loop_task = None
+
+    async def start(self, interface: str, port: int):
+        self.logger.info("start prometheus metrics")
+        prom_app = web.Application()
+        prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
+        self.runner = web.AppRunner(prom_app)
+        await self.runner.setup()
+
+        metrics_site = web.TCPSite(self.runner, interface, port, shutdown_timeout=.5)
+        await metrics_site.start()
+        self.logger.info(
+            'prometheus metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2]
+        )
+        self._monitor_loop_task = get_loop_metrics()
+
+    async def handle_metrics_get_request(self, request: web.Request):
+        try:
+            return web.Response(
+                text=prom_generate_latest().decode(),
+                content_type='text/plain; version=0.0.4'
+            )
+        except Exception:
+            self.logger.exception('could not generate prometheus data')
+            raise
+
+    async def stop(self):
+        if self._monitor_loop_task and not self._monitor_loop_task.done():
+            self._monitor_loop_task.cancel()
+        self._monitor_loop_task = None
+        await self.runner.cleanup()

lbry/schema/Makefile

@@ -2,4 +2,5 @@ build:
 	rm types/v2/* -rf
 	touch types/v2/__init__.py
 	cd types/v2/ && protoc --python_out=. -I ../../../../../types/v2/proto/ ../../../../../types/v2/proto/*.proto
+	cd types/v2/ && cp ../../../../../types/jsonschema/* ./
 	sed -e 's/^import\ \(.*\)_pb2\ /from . import\ \1_pb2\ /g' -i types/v2/*.py

lbry/schema/README.md

@@ -0,0 +1,24 @@
+Schema
+=====
+
+Those files are generated from the [types repo](https://github.com/lbryio/types). If you are modifying/adding a new type, make sure it is cloned in the same root folder as the SDK repo, like:
+
+```
+repos/
+    - lbry-sdk/
+    - types/
+```
+
+Then, [download protoc 3.2.0](https://github.com/protocolbuffers/protobuf/releases/tag/v3.2.0), add it to your PATH. On linux it is:
+
+```bash
+cd ~/.local/bin
+wget https://github.com/protocolbuffers/protobuf/releases/download/v3.2.0/protoc-3.2.0-linux-x86_64.zip
+unzip protoc-3.2.0-linux-x86_64.zip bin/protoc -d..
+```
+
+Finally, `make` should update everything in place.
+
+
+### Why protoc 3.2.0?
+Different/newer versions will generate larger diffs and we need to make sure they are good. In theory, we can just update to latest and it will all work, but it is a good practice to check blockchain data and retro compatibility before bumping versions (if you do, please update this section!).

lbry/schema/attrs.py

@@ -10,6 +10,7 @@ from google.protobuf.json_format import MessageToDict
 
 from lbry.crypto.base58 import Base58
 from lbry.constants import COIN
+from lbry.error import MissingPublishedFileError, EmptyPublishedFileError
 
 from lbry.schema.mime_types import guess_media_type
 from lbry.schema.base import Metadata, BaseMessageList
@@ -32,6 +33,17 @@ def calculate_sha384_file_hash(file_path):
     return sha384.digest()
 
 
+def country_int_to_str(country: int) -> str:
+    r = LocationMessage.Country.Name(country)
+    return r[1:] if r.startswith('R') else r
+
+
+def country_str_to_int(country: str) -> int:
+    if len(country) == 3:
+        country = 'R' + country
+    return LocationMessage.Country.Value(country)
+
+
 class Dimmensional(Metadata):
 
     __slots__ = ()
@@ -128,10 +140,10 @@ class Source(Metadata):
             self.name = os.path.basename(file_path)
             self.media_type, stream_type = guess_media_type(file_path)
             if not os.path.isfile(file_path):
-                raise Exception(f"File does not exist: {file_path}")
+                raise MissingPublishedFileError(file_path)
             self.size = os.path.getsize(file_path)
             if self.size == 0:
-                raise Exception(f"Cannot publish empty file: {file_path}")
+                raise EmptyPublishedFileError(file_path)
             self.file_hash_bytes = calculate_sha384_file_hash(file_path)
             return stream_type
 
@@ -423,14 +435,11 @@ class Language(Metadata):
     @property
     def region(self) -> str:
         if self.message.region:
-            r = LocationMessage.Country.Name(self.message.region)
-            return r[1:] if r.startswith('R') else r
+            return country_int_to_str(self.message.region)
 
     @region.setter
     def region(self, region: str):
-        if len(region) == 3:
-            region = 'R'+region
-        self.message.region = LocationMessage.Country.Value(region)
+        self.message.region = country_str_to_int(region)
 
 
 class LanguageList(BaseMessageList[Language]):

lbry/schema/claim.py

@@ -2,6 +2,9 @@ import logging
 from typing import List
 from binascii import hexlify, unhexlify
 
+from asn1crypto.keys import PublicKeyInfo
+from coincurve import PublicKey as cPublicKey
+
 from google.protobuf.json_format import MessageToDict
 from google.protobuf.message import DecodeError
 from hachoir.core.log import log as hachoir_log
@@ -30,14 +33,10 @@ class Claim(Signable):
     COLLECTION = 'collection'
     REPOST = 'repost'
 
-    __slots__ = 'version',
+    __slots__ = ()
 
     message_class = ClaimMessage
 
-    def __init__(self, message=None):
-        super().__init__(message)
-        self.version = 2
-
     @property
     def claim_type(self) -> str:
         return self.message.WhichOneof('type')
@@ -253,7 +252,7 @@ class Stream(BaseClaim):
         if stream_type in ('image', 'video', 'audio'):
             media = getattr(self, stream_type)
             media_args = {'file_metadata': None}
-            if file_path is not None:
+            if file_path is not None and not all((duration, width, height)):
                 try:
                     media_args['file_metadata'] = binary_file_metadata(binary_file_parser(file_path))
                 except:
@@ -307,6 +306,10 @@ class Stream(BaseClaim):
     def has_fee(self) -> bool:
         return self.message.HasField('fee')
 
+    @property
+    def has_source(self) -> bool:
+        return self.message.HasField('source')
+
     @property
     def source(self) -> Source:
         return Source(self.message.source)
@@ -346,7 +349,7 @@ class Channel(BaseClaim):
 
     @property
     def public_key(self) -> str:
-        return hexlify(self.message.public_key).decode()
+        return hexlify(self.public_key_bytes).decode()
 
     @public_key.setter
     def public_key(self, sd_public_key: str):
@@ -354,7 +357,11 @@ class Channel(BaseClaim):
 
     @property
     def public_key_bytes(self) -> bytes:
-        return self.message.public_key
+        if len(self.message.public_key) == 33:
+            return self.message.public_key
+        public_key_info = PublicKeyInfo.load(self.message.public_key)
+        public_key = cPublicKey(public_key_info.native['public_key'])
+        return public_key.format(compressed=True)
 
     @public_key_bytes.setter
     def public_key_bytes(self, public_key: bytes):
@@ -391,6 +398,12 @@ class Repost(BaseClaim):
 
     claim_type = Claim.REPOST
 
+    def to_dict(self):
+        claim = super().to_dict()
+        if claim.pop('claim_hash', None):
+            claim['claim_id'] = self.reference.claim_id
+        return claim
+
     @property
     def reference(self) -> ClaimReference:
         return ClaimReference(self.message)

lbry/schema/mime_types.py

@@ -1,4 +1,6 @@
 import os
+import filetype
+import logging
 
 types_map = {
     # http://www.iana.org/assignments/media-types
@@ -46,8 +48,8 @@ types_map = {
     '.ksh': ('text/plain', 'document'),
     '.latex': ('application/x-latex', 'binary'),
     '.m1v': ('video/mpeg', 'video'),
-    '.m3u': ('application/vnd.apple.mpegurl', 'audio'),
-    '.m3u8': ('application/vnd.apple.mpegurl', 'audio'),
+    '.m3u': ('application/x-mpegurl', 'audio'),
+    '.m3u8': ('application/x-mpegurl', 'video'),
     '.man': ('application/x-troff-man', 'document'),
     '.markdown': ('text/markdown', 'document'),
     '.md': ('text/markdown', 'document'),
@@ -61,10 +63,12 @@ types_map = {
     '.mp3': ('audio/mpeg', 'audio'),
     '.mp4': ('video/mp4', 'video'),
     '.mpa': ('video/mpeg', 'video'),
+    '.mpd': ('application/dash+xml', 'video'),
     '.mpe': ('video/mpeg', 'video'),
     '.mpeg': ('video/mpeg', 'video'),
     '.mpg': ('video/mpeg', 'video'),
     '.ms': ('application/x-troff-ms', 'binary'),
+    '.m4s': ('video/iso.segment', 'binary'),
     '.nc': ('application/x-netcdf', 'binary'),
     '.nws': ('message/rfc822', 'document'),
     '.o': ('application/octet-stream', 'binary'),
@@ -118,10 +122,12 @@ types_map = {
     '.tif': ('image/tiff', 'image'),
     '.tiff': ('image/tiff', 'image'),
     '.tr': ('application/x-troff', 'binary'),
+    '.ts': ('video/mp2t', 'video'),
     '.tsv': ('text/tab-separated-values', 'document'),
     '.txt': ('text/plain', 'document'),
     '.ustar': ('application/x-ustar', 'binary'),
     '.vcf': ('text/x-vcard', 'document'),
+    '.vtt': ('text/vtt', 'document'),
     '.wav': ('audio/x-wav', 'audio'),
     '.webm': ('video/webm', 'video'),
     '.wiz': ('application/msword', 'document'),
@@ -141,6 +147,7 @@ types_map = {
     '.cbz': ('application/vnd.comicbook+zip', 'document'),
     '.flac': ('audio/flac', 'audio'),
     '.lbry': ('application/x-ext-lbry', 'document'),
+    '.m4a': ('audio/mp4', 'audio'),
     '.m4v': ('video/m4v', 'video'),
     '.mid': ('audio/midi', 'audio'),
     '.midi': ('audio/midi', 'audio'),
@@ -148,6 +155,7 @@ types_map = {
     '.mobi': ('application/x-mobipocket-ebook', 'document'),
     '.oga': ('audio/ogg', 'audio'),
     '.ogv': ('video/ogg', 'video'),
+    '.ogg': ('video/ogg', 'video'),
     '.pct': ('image/pict', 'image'),
     '.pic': ('image/pict', 'image'),
     '.pict': ('image/pict', 'image'),
@@ -160,10 +168,38 @@ types_map = {
     '.wmv': ('video/x-ms-wmv', 'video')
 }
 
+# maps detected extensions to the possible analogs
+# i.e. .cbz file is actually a .zip
+synonyms_map = {
+    '.zip': ['.cbz'],
+    '.rar': ['.cbr'],
+    '.ar': ['.a']
+}
+
+log = logging.getLogger(__name__)
+
 
 def guess_media_type(path):
     _, ext = os.path.splitext(path)
     extension = ext.strip().lower()
+
+    try:
+        kind = filetype.guess(path)
+        if kind:
+            real_extension = f".{kind.extension}"
+
+            if extension != real_extension:
+                if extension:
+                    log.warning(f"file extension does not match it's contents: {path}, identified as {real_extension}")
+                else:
+                    log.debug(f"file {path} does not have extension, identified by it's contents as {real_extension}")
+
+                if extension not in synonyms_map.get(real_extension, []):
+                    extension = real_extension
+
+    except OSError as error:
+        pass
+
     if extension[1:]:
         if extension in types_map:
             return types_map[extension]

lbry/schema/result.py

@@ -1,6 +1,5 @@
 import base64
-import struct
-from typing import List
+from typing import List, Union, Optional, NamedTuple
 from binascii import hexlify
 from itertools import chain
 
@@ -16,45 +15,70 @@ BLOCKED = ErrorMessage.Code.Name(ErrorMessage.BLOCKED)
 def set_reference(reference, claim_hash, rows):
     if claim_hash:
         for txo in rows:
-            if claim_hash == txo['claim_hash']:
-                reference.tx_hash = txo['txo_hash'][:32]
-                reference.nout = struct.unpack('<I', txo['txo_hash'][32:])[0]
-                reference.height = txo['height']
+            if claim_hash == txo.claim_hash:
+                reference.tx_hash = txo.tx_hash
+                reference.nout = txo.position
+                reference.height = txo.height
                 return
 
 
+class ResolveResult(NamedTuple):
+    name: str
+    normalized_name: str
+    claim_hash: bytes
+    tx_num: int
+    position: int
+    tx_hash: bytes
+    height: int
+    amount: int
+    short_url: str
+    is_controlling: bool
+    canonical_url: str
+    creation_height: int
+    activation_height: int
+    expiration_height: int
+    effective_amount: int
+    support_amount: int
+    reposted: int
+    last_takeover_height: Optional[int]
+    claims_in_channel: Optional[int]
+    channel_hash: Optional[bytes]
+    reposted_claim_hash: Optional[bytes]
+    signature_valid: Optional[bool]
+
+
 class Censor:
 
-    __slots__ = 'streams', 'channels', 'censored', 'total'
+    NOT_CENSORED = 0
+    SEARCH = 1
+    RESOLVE = 2
 
-    def __init__(self, streams: dict = None, channels: dict = None):
-        self.streams = streams or {}
-        self.channels = channels or {}
+    __slots__ = 'censor_type', 'censored'
+
+    def __init__(self, censor_type):
+        self.censor_type = censor_type
         self.censored = {}
-        self.total = 0
 
-    def censor(self, row) -> bool:
-        was_censored = False
-        for claim_hash, lookup in (
-                (row['claim_hash'], self.streams),
-                (row['claim_hash'], self.channels),
-                (row['channel_hash'], self.channels)):
-            censoring_channel_hash = lookup.get(claim_hash)
-            if censoring_channel_hash:
-                was_censored = True
-                self.censored.setdefault(censoring_channel_hash, 0)
-                self.censored[censoring_channel_hash] += 1
-                break
-        if was_censored:
-            self.total += 1
-        return was_censored
+    def is_censored(self, row):
+        return (row.get('censor_type') or self.NOT_CENSORED) >= self.censor_type
 
-    def to_message(self, outputs: OutputsMessage, extra_txo_rows):
-        outputs.blocked_total = self.total
+    def apply(self, rows):
+        return [row for row in rows if not self.censor(row)]
+
+    def censor(self, row) -> Optional[bytes]:
+        if self.is_censored(row):
+            censoring_channel_hash = bytes.fromhex(row['censoring_channel_id'])[::-1]
+            self.censored.setdefault(censoring_channel_hash, set())
+            self.censored[censoring_channel_hash].add(row['tx_hash'])
+            return censoring_channel_hash
+        return None
+
+    def to_message(self, outputs: OutputsMessage, extra_txo_rows: dict):
         for censoring_channel_hash, count in self.censored.items():
             blocked = outputs.blocked.add()
-            blocked.count = count
+            blocked.count = len(count)
             set_reference(blocked.channel, censoring_channel_hash, extra_txo_rows)
+            outputs.blocked_total += len(count)
 
 
 class Outputs:
@@ -118,10 +142,10 @@ class Outputs:
                 'expiration_height': claim.expiration_height,
                 'effective_amount': claim.effective_amount,
                 'support_amount': claim.support_amount,
-                'trending_group': claim.trending_group,
-                'trending_mixed': claim.trending_mixed,
-                'trending_local': claim.trending_local,
-                'trending_global': claim.trending_global,
+                # 'trending_group': claim.trending_group,
+                # 'trending_mixed': claim.trending_mixed,
+                # 'trending_local': claim.trending_local,
+                # 'trending_global': claim.trending_global,
             }
             if claim.HasField('channel'):
                 txo.channel = tx_map[claim.channel.tx_hash].outputs[claim.channel.nout]
@@ -165,44 +189,54 @@ class Outputs:
             page.total = total
         if blocked is not None:
             blocked.to_message(page, extra_txo_rows)
-        for row in txo_rows:
-            cls.row_to_message(row, page.txos.add(), extra_txo_rows)
         for row in extra_txo_rows:
-            cls.row_to_message(row, page.extra_txos.add(), extra_txo_rows)
+            txo_message: 'OutputsMessage' = page.extra_txos.add()
+            if not isinstance(row, Exception):
+                if row.channel_hash:
+                    set_reference(txo_message.claim.channel, row.channel_hash, extra_txo_rows)
+                if row.reposted_claim_hash:
+                    set_reference(txo_message.claim.repost, row.reposted_claim_hash, extra_txo_rows)
+            cls.encode_txo(txo_message, row)
+
+        for row in txo_rows:
+            # cls.row_to_message(row, page.txos.add(), extra_txo_rows)
+            txo_message: 'OutputsMessage' = page.txos.add()
+            cls.encode_txo(txo_message, row)
+            if not isinstance(row, Exception):
+                if row.channel_hash:
+                    set_reference(txo_message.claim.channel, row.channel_hash, extra_txo_rows)
+                if row.reposted_claim_hash:
+                    set_reference(txo_message.claim.repost, row.reposted_claim_hash, extra_txo_rows)
+            elif isinstance(row, ResolveCensoredError):
+                set_reference(txo_message.error.blocked.channel, row.censor_id, extra_txo_rows)
         return page.SerializeToString()
 
     @classmethod
-    def row_to_message(cls, txo, txo_message, extra_txo_rows):
-        if isinstance(txo, Exception):
-            txo_message.error.text = txo.args[0]
-            if isinstance(txo, ValueError):
+    def encode_txo(cls, txo_message, resolve_result: Union['ResolveResult', Exception]):
+        if isinstance(resolve_result, Exception):
+            txo_message.error.text = resolve_result.args[0]
+            if isinstance(resolve_result, ValueError):
                 txo_message.error.code = ErrorMessage.INVALID
-            elif isinstance(txo, LookupError):
+            elif isinstance(resolve_result, LookupError):
                 txo_message.error.code = ErrorMessage.NOT_FOUND
-            elif isinstance(txo, ResolveCensoredError):
+            elif isinstance(resolve_result, ResolveCensoredError):
                 txo_message.error.code = ErrorMessage.BLOCKED
-                set_reference(txo_message.error.blocked.channel, txo.censor_hash, extra_txo_rows)
             return
-        txo_message.tx_hash = txo['txo_hash'][:32]
-        txo_message.nout, = struct.unpack('<I', txo['txo_hash'][32:])
-        txo_message.height = txo['height']
-        txo_message.claim.short_url = txo['short_url']
-        txo_message.claim.reposted = txo['reposted']
-        if txo['canonical_url'] is not None:
-            txo_message.claim.canonical_url = txo['canonical_url']
-        txo_message.claim.is_controlling = bool(txo['is_controlling'])
-        if txo['last_take_over_height'] is not None:
-            txo_message.claim.take_over_height = txo['last_take_over_height']
-        txo_message.claim.creation_height = txo['creation_height']
-        txo_message.claim.activation_height = txo['activation_height']
-        txo_message.claim.expiration_height = txo['expiration_height']
-        if txo['claims_in_channel'] is not None:
-            txo_message.claim.claims_in_channel = txo['claims_in_channel']
-        txo_message.claim.effective_amount = txo['effective_amount']
-        txo_message.claim.support_amount = txo['support_amount']
-        txo_message.claim.trending_group = txo['trending_group']
-        txo_message.claim.trending_mixed = txo['trending_mixed']
-        txo_message.claim.trending_local = txo['trending_local']
-        txo_message.claim.trending_global = txo['trending_global']
-        set_reference(txo_message.claim.channel, txo['channel_hash'], extra_txo_rows)
-        set_reference(txo_message.claim.repost, txo['reposted_claim_hash'], extra_txo_rows)
+        txo_message.tx_hash = resolve_result.tx_hash
+        txo_message.nout = resolve_result.position
+        txo_message.height = resolve_result.height
+        txo_message.claim.short_url = resolve_result.short_url
+        txo_message.claim.reposted = resolve_result.reposted
+        txo_message.claim.is_controlling = resolve_result.is_controlling
+        txo_message.claim.creation_height = resolve_result.creation_height
+        txo_message.claim.activation_height = resolve_result.activation_height
+        txo_message.claim.expiration_height = resolve_result.expiration_height
+        txo_message.claim.effective_amount = resolve_result.effective_amount
+        txo_message.claim.support_amount = resolve_result.support_amount
+
+        if resolve_result.canonical_url is not None:
+            txo_message.claim.canonical_url = resolve_result.canonical_url
+        if resolve_result.last_takeover_height is not None:
+            txo_message.claim.take_over_height = resolve_result.last_takeover_height
+        if resolve_result.claims_in_channel is not None:
+            txo_message.claim.claims_in_channel = resolve_result.claims_in_channel

lbry/schema/support.py

@@ -1,6 +1,23 @@
 from lbry.schema.base import Signable
+from lbry.schema.types.v2.support_pb2 import Support as SupportMessage
 
 
 class Support(Signable):
     __slots__ = ()
-    message_class = None  # TODO: add support protobufs
+    message_class = SupportMessage
+
+    @property
+    def emoji(self) -> str:
+        return self.message.emoji
+
+    @emoji.setter
+    def emoji(self, emoji: str):
+        self.message.emoji = emoji
+
+    @property
+    def comment(self) -> str:
+        return self.message.comment
+
+    @comment.setter
+    def comment(self, comment: str):
+        self.message.comment = comment

lbry/schema/types/v2/result_pb2.py

@@ -1,13 +1,11 @@
+# -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: result.proto
-
-import sys
-_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+"""Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import message as _message
 from google.protobuf import reflection as _reflection
 from google.protobuf import symbol_database as _symbol_database
-from google.protobuf import descriptor_pb2
 # @@protoc_insertion_point(imports)
 
 _sym_db = _symbol_database.Default()
@@ -19,9 +17,10 @@ DESCRIPTOR = _descriptor.FileDescriptor(
   name='result.proto',
   package='pb',
   syntax='proto3',
-  serialized_pb=_b('\n\x0cresult.proto\x12\x02pb\"\x97\x01\n\x07Outputs\x12\x18\n\x04txos\x18\x01 \x03(\x0b\x32\n.pb.Output\x12\x1e\n\nextra_txos\x18\x02 \x03(\x0b\x32\n.pb.Output\x12\r\n\x05total\x18\x03 \x01(\r\x12\x0e\n\x06offset\x18\x04 \x01(\r\x12\x1c\n\x07\x62locked\x18\x05 \x03(\x0b\x32\x0b.pb.Blocked\x12\x15\n\rblocked_total\x18\x06 \x01(\r\"{\n\x06Output\x12\x0f\n\x07tx_hash\x18\x01 \x01(\x0c\x12\x0c\n\x04nout\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\x1e\n\x05\x63laim\x18\x07 \x01(\x0b\x32\r.pb.ClaimMetaH\x00\x12\x1a\n\x05\x65rror\x18\x0f \x01(\x0b\x32\t.pb.ErrorH\x00\x42\x06\n\x04meta\"\xaf\x03\n\tClaimMeta\x12\x1b\n\x07\x63hannel\x18\x01 \x01(\x0b\x32\n.pb.Output\x12\x1a\n\x06repost\x18\x02 \x01(\x0b\x32\n.pb.Output\x12\x11\n\tshort_url\x18\x03 \x01(\t\x12\x15\n\rcanonical_url\x18\x04 \x01(\t\x12\x16\n\x0eis_controlling\x18\x05 \x01(\x08\x12\x18\n\x10take_over_height\x18\x06 \x01(\r\x12\x17\n\x0f\x63reation_height\x18\x07 \x01(\r\x12\x19\n\x11\x61\x63tivation_height\x18\x08 \x01(\r\x12\x19\n\x11\x65xpiration_height\x18\t \x01(\r\x12\x19\n\x11\x63laims_in_channel\x18\n \x01(\r\x12\x10\n\x08reposted\x18\x0b \x01(\r\x12\x18\n\x10\x65\x66\x66\x65\x63tive_amount\x18\x14 \x01(\x04\x12\x16\n\x0esupport_amount\x18\x15 \x01(\x04\x12\x16\n\x0etrending_group\x18\x16 \x01(\r\x12\x16\n\x0etrending_mixed\x18\x17 \x01(\x02\x12\x16\n\x0etrending_local\x18\x18 \x01(\x02\x12\x17\n\x0ftrending_global\x18\x19 \x01(\x02\"\x94\x01\n\x05\x45rror\x12\x1c\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x0e.pb.Error.Code\x12\x0c\n\x04text\x18\x02 \x01(\t\x12\x1c\n\x07\x62locked\x18\x03 \x01(\x0b\x32\x0b.pb.Blocked\"A\n\x04\x43ode\x12\x10\n\x0cUNKNOWN_CODE\x10\x00\x12\r\n\tNOT_FOUND\x10\x01\x12\x0b\n\x07INVALID\x10\x02\x12\x0b\n\x07\x42LOCKED\x10\x03\"5\n\x07\x42locked\x12\r\n\x05\x63ount\x18\x01 \x01(\r\x12\x1b\n\x07\x63hannel\x18\x02 \x01(\x0b\x32\n.pb.Outputb\x06proto3')
+  serialized_options=b'Z$github.com/lbryio/hub/protobuf/go/pb',
+  create_key=_descriptor._internal_create_key,
+  serialized_pb=b'\n\x0cresult.proto\x12\x02pb\"\x97\x01\n\x07Outputs\x12\x18\n\x04txos\x18\x01 \x03(\x0b\x32\n.pb.Output\x12\x1e\n\nextra_txos\x18\x02 \x03(\x0b\x32\n.pb.Output\x12\r\n\x05total\x18\x03 \x01(\r\x12\x0e\n\x06offset\x18\x04 \x01(\r\x12\x1c\n\x07\x62locked\x18\x05 \x03(\x0b\x32\x0b.pb.Blocked\x12\x15\n\rblocked_total\x18\x06 \x01(\r\"{\n\x06Output\x12\x0f\n\x07tx_hash\x18\x01 \x01(\x0c\x12\x0c\n\x04nout\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\x1e\n\x05\x63laim\x18\x07 \x01(\x0b\x32\r.pb.ClaimMetaH\x00\x12\x1a\n\x05\x65rror\x18\x0f \x01(\x0b\x32\t.pb.ErrorH\x00\x42\x06\n\x04meta\"\xe6\x02\n\tClaimMeta\x12\x1b\n\x07\x63hannel\x18\x01 \x01(\x0b\x32\n.pb.Output\x12\x1a\n\x06repost\x18\x02 \x01(\x0b\x32\n.pb.Output\x12\x11\n\tshort_url\x18\x03 \x01(\t\x12\x15\n\rcanonical_url\x18\x04 \x01(\t\x12\x16\n\x0eis_controlling\x18\x05 \x01(\x08\x12\x18\n\x10take_over_height\x18\x06 \x01(\r\x12\x17\n\x0f\x63reation_height\x18\x07 \x01(\r\x12\x19\n\x11\x61\x63tivation_height\x18\x08 \x01(\r\x12\x19\n\x11\x65xpiration_height\x18\t \x01(\r\x12\x19\n\x11\x63laims_in_channel\x18\n \x01(\r\x12\x10\n\x08reposted\x18\x0b \x01(\r\x12\x18\n\x10\x65\x66\x66\x65\x63tive_amount\x18\x14 \x01(\x04\x12\x16\n\x0esupport_amount\x18\x15 \x01(\x04\x12\x16\n\x0etrending_score\x18\x16 \x01(\x01\"\x94\x01\n\x05\x45rror\x12\x1c\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x0e.pb.Error.Code\x12\x0c\n\x04text\x18\x02 \x01(\t\x12\x1c\n\x07\x62locked\x18\x03 \x01(\x0b\x32\x0b.pb.Blocked\"A\n\x04\x43ode\x12\x10\n\x0cUNKNOWN_CODE\x10\x00\x12\r\n\tNOT_FOUND\x10\x01\x12\x0b\n\x07INVALID\x10\x02\x12\x0b\n\x07\x42LOCKED\x10\x03\"5\n\x07\x42locked\x12\r\n\x05\x63ount\x18\x01 \x01(\r\x12\x1b\n\x07\x63hannel\x18\x02 \x01(\x0b\x32\n.pb.OutputB&Z$github.com/lbryio/hub/protobuf/go/pbb\x06proto3'
 )
-_sym_db.RegisterFileDescriptor(DESCRIPTOR)
 
 
 
@@ -30,28 +29,33 @@ _ERROR_CODE = _descriptor.EnumDescriptor(
   full_name='pb.Error.Code',
   filename=None,
   file=DESCRIPTOR,
+  create_key=_descriptor._internal_create_key,
   values=[
     _descriptor.EnumValueDescriptor(
       name='UNKNOWN_CODE', index=0, number=0,
-      options=None,
-      type=None),
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
     _descriptor.EnumValueDescriptor(
       name='NOT_FOUND', index=1, number=1,
-      options=None,
-      type=None),
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
     _descriptor.EnumValueDescriptor(
       name='INVALID', index=2, number=2,
-      options=None,
-      type=None),
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
     _descriptor.EnumValueDescriptor(
       name='BLOCKED', index=3, number=3,
-      options=None,
-      type=None),
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
   ],
   containing_type=None,
-  options=None,
-  serialized_start=817,
-  serialized_end=882,
+  serialized_options=None,
+  serialized_start=744,
+  serialized_end=809,
 )
 _sym_db.RegisterEnumDescriptor(_ERROR_CODE)
 
@@ -62,6 +66,7 @@ _OUTPUTS = _descriptor.Descriptor(
   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
+  create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
       name='txos', full_name='pb.Outputs.txos', index=0,
@@ -69,49 +74,49 @@ _OUTPUTS = _descriptor.Descriptor(
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='extra_txos', full_name='pb.Outputs.extra_txos', index=1,
       number=2, type=11, cpp_type=10, label=3,
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='total', full_name='pb.Outputs.total', index=2,
       number=3, type=13, cpp_type=3, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='offset', full_name='pb.Outputs.offset', index=3,
       number=4, type=13, cpp_type=3, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='blocked', full_name='pb.Outputs.blocked', index=4,
       number=5, type=11, cpp_type=10, label=3,
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='blocked_total', full_name='pb.Outputs.blocked_total', index=5,
       number=6, type=13, cpp_type=3, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
   ],
   extensions=[
   ],
   nested_types=[],
   enum_types=[
   ],
-  options=None,
+  serialized_options=None,
   is_extendable=False,
   syntax='proto3',
   extension_ranges=[],
@@ -128,56 +133,59 @@ _OUTPUT = _descriptor.Descriptor(
   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
+  create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
       name='tx_hash', full_name='pb.Output.tx_hash', index=0,
       number=1, type=12, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b(""),
+      has_default_value=False, default_value=b"",
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='nout', full_name='pb.Output.nout', index=1,
       number=2, type=13, cpp_type=3, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='height', full_name='pb.Output.height', index=2,
       number=3, type=13, cpp_type=3, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='claim', full_name='pb.Output.claim', index=3,
       number=7, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='error', full_name='pb.Output.error', index=4,
       number=15, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
   ],
   extensions=[
   ],
   nested_types=[],
   enum_types=[
   ],
-  options=None,
+  serialized_options=None,
   is_extendable=False,
   syntax='proto3',
   extension_ranges=[],
   oneofs=[
     _descriptor.OneofDescriptor(
       name='meta', full_name='pb.Output.meta',
-      index=0, containing_type=None, fields=[]),
+      index=0, containing_type=None,
+      create_key=_descriptor._internal_create_key,
+    fields=[]),
   ],
   serialized_start=174,
   serialized_end=297,
@@ -190,6 +198,7 @@ _CLAIMMETA = _descriptor.Descriptor(
   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
+  create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
       name='channel', full_name='pb.ClaimMeta.channel', index=0,
@@ -197,133 +206,112 @@ _CLAIMMETA = _descriptor.Descriptor(
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='repost', full_name='pb.ClaimMeta.repost', index=1,
       number=2, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='short_url', full_name='pb.ClaimMeta.short_url', index=2,
       number=3, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
+      has_default_value=False, default_value=b"".decode('utf-8'),
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='canonical_url', full_name='pb.ClaimMeta.canonical_url', index=3,
       number=4, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
+      has_default_value=False, default_value=b"".decode('utf-8'),
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='is_controlling', full_name='pb.ClaimMeta.is_controlling', index=4,
       number=5, type=8, cpp_type=7, label=1,
       has_default_value=False, default_value=False,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='take_over_height', full_name='pb.ClaimMeta.take_over_height', index=5,
       number=6, type=13, cpp_type=3, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='creation_height', full_name='pb.ClaimMeta.creation_height', index=6,
       number=7, type=13, cpp_type=3, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='activation_height', full_name='pb.ClaimMeta.activation_height', index=7,
       number=8, type=13, cpp_type=3, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='expiration_height', full_name='pb.ClaimMeta.expiration_height', index=8,
       number=9, type=13, cpp_type=3, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='claims_in_channel', full_name='pb.ClaimMeta.claims_in_channel', index=9,
       number=10, type=13, cpp_type=3, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='reposted', full_name='pb.ClaimMeta.reposted', index=10,
       number=11, type=13, cpp_type=3, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='effective_amount', full_name='pb.ClaimMeta.effective_amount', index=11,
       number=20, type=4, cpp_type=4, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='support_amount', full_name='pb.ClaimMeta.support_amount', index=12,
       number=21, type=4, cpp_type=4, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
-      name='trending_group', full_name='pb.ClaimMeta.trending_group', index=13,
-      number=22, type=13, cpp_type=3, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='trending_mixed', full_name='pb.ClaimMeta.trending_mixed', index=14,
-      number=23, type=2, cpp_type=6, label=1,
+      name='trending_score', full_name='pb.ClaimMeta.trending_score', index=13,
+      number=22, type=1, cpp_type=5, label=1,
       has_default_value=False, default_value=float(0),
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='trending_local', full_name='pb.ClaimMeta.trending_local', index=15,
-      number=24, type=2, cpp_type=6, label=1,
-      has_default_value=False, default_value=float(0),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
-    _descriptor.FieldDescriptor(
-      name='trending_global', full_name='pb.ClaimMeta.trending_global', index=16,
-      number=25, type=2, cpp_type=6, label=1,
-      has_default_value=False, default_value=float(0),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
   ],
   extensions=[
   ],
   nested_types=[],
   enum_types=[
   ],
-  options=None,
+  serialized_options=None,
   is_extendable=False,
   syntax='proto3',
   extension_ranges=[],
   oneofs=[
   ],
   serialized_start=300,
-  serialized_end=731,
+  serialized_end=658,
 )
 
 
@@ -333,6 +321,7 @@ _ERROR = _descriptor.Descriptor(
   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
+  create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
       name='code', full_name='pb.Error.code', index=0,
@@ -340,21 +329,21 @@ _ERROR = _descriptor.Descriptor(
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='text', full_name='pb.Error.text', index=1,
       number=2, type=9, cpp_type=9, label=1,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
+      has_default_value=False, default_value=b"".decode('utf-8'),
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='blocked', full_name='pb.Error.blocked', index=2,
       number=3, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
   ],
   extensions=[
   ],
@@ -362,14 +351,14 @@ _ERROR = _descriptor.Descriptor(
   enum_types=[
     _ERROR_CODE,
   ],
-  options=None,
+  serialized_options=None,
   is_extendable=False,
   syntax='proto3',
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=734,
-  serialized_end=882,
+  serialized_start=661,
+  serialized_end=809,
 )
 
 
@@ -379,6 +368,7 @@ _BLOCKED = _descriptor.Descriptor(
   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
+  create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
       name='count', full_name='pb.Blocked.count', index=0,
@@ -386,28 +376,28 @@ _BLOCKED = _descriptor.Descriptor(
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='channel', full_name='pb.Blocked.channel', index=1,
       number=2, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      options=None),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
   ],
   extensions=[
   ],
   nested_types=[],
   enum_types=[
   ],
-  options=None,
+  serialized_options=None,
   is_extendable=False,
   syntax='proto3',
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=884,
-  serialized_end=937,
+  serialized_start=811,
+  serialized_end=864,
 )
 
 _OUTPUTS.fields_by_name['txos'].message_type = _OUTPUT
@@ -432,41 +422,43 @@ DESCRIPTOR.message_types_by_name['Output'] = _OUTPUT
 DESCRIPTOR.message_types_by_name['ClaimMeta'] = _CLAIMMETA
 DESCRIPTOR.message_types_by_name['Error'] = _ERROR
 DESCRIPTOR.message_types_by_name['Blocked'] = _BLOCKED
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
 
-Outputs = _reflection.GeneratedProtocolMessageType('Outputs', (_message.Message,), dict(
-  DESCRIPTOR = _OUTPUTS,
-  __module__ = 'result_pb2'
+Outputs = _reflection.GeneratedProtocolMessageType('Outputs', (_message.Message,), {
+  'DESCRIPTOR' : _OUTPUTS,
+  '__module__' : 'result_pb2'
   # @@protoc_insertion_point(class_scope:pb.Outputs)
-  ))
+  })
 _sym_db.RegisterMessage(Outputs)
 
-Output = _reflection.GeneratedProtocolMessageType('Output', (_message.Message,), dict(
-  DESCRIPTOR = _OUTPUT,
-  __module__ = 'result_pb2'
+Output = _reflection.GeneratedProtocolMessageType('Output', (_message.Message,), {
+  'DESCRIPTOR' : _OUTPUT,
+  '__module__' : 'result_pb2'
   # @@protoc_insertion_point(class_scope:pb.Output)
-  ))
+  })
 _sym_db.RegisterMessage(Output)
 
-ClaimMeta = _reflection.GeneratedProtocolMessageType('ClaimMeta', (_message.Message,), dict(
-  DESCRIPTOR = _CLAIMMETA,
-  __module__ = 'result_pb2'
+ClaimMeta = _reflection.GeneratedProtocolMessageType('ClaimMeta', (_message.Message,), {
+  'DESCRIPTOR' : _CLAIMMETA,
+  '__module__' : 'result_pb2'
   # @@protoc_insertion_point(class_scope:pb.ClaimMeta)
-  ))
+  })
 _sym_db.RegisterMessage(ClaimMeta)
 
-Error = _reflection.GeneratedProtocolMessageType('Error', (_message.Message,), dict(
-  DESCRIPTOR = _ERROR,
-  __module__ = 'result_pb2'
+Error = _reflection.GeneratedProtocolMessageType('Error', (_message.Message,), {
+  'DESCRIPTOR' : _ERROR,
+  '__module__' : 'result_pb2'
   # @@protoc_insertion_point(class_scope:pb.Error)
-  ))
+  })
 _sym_db.RegisterMessage(Error)
 
-Blocked = _reflection.GeneratedProtocolMessageType('Blocked', (_message.Message,), dict(
-  DESCRIPTOR = _BLOCKED,
-  __module__ = 'result_pb2'
+Blocked = _reflection.GeneratedProtocolMessageType('Blocked', (_message.Message,), {
+  'DESCRIPTOR' : _BLOCKED,
+  '__module__' : 'result_pb2'
   # @@protoc_insertion_point(class_scope:pb.Blocked)
-  ))
+  })
 _sym_db.RegisterMessage(Blocked)
 
 
+DESCRIPTOR._options = None
 # @@protoc_insertion_point(module_scope)

lbry/schema/types/v2/support_pb2.py

@@ -0,0 +1,76 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: support.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf import descriptor_pb2
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='support.proto',
+  package='pb',
+  syntax='proto3',
+  serialized_pb=_b('\n\rsupport.proto\x12\x02pb\")\n\x07Support\x12\r\n\x05\x65moji\x18\x01 \x01(\t\x12\x0f\n\x07\x63omment\x18\x02 \x01(\tb\x06proto3')
+)
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+
+
+
+_SUPPORT = _descriptor.Descriptor(
+  name='Support',
+  full_name='pb.Support',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='emoji', full_name='pb.Support.emoji', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='comment', full_name='pb.Support.comment', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=21,
+  serialized_end=62,
+)
+
+DESCRIPTOR.message_types_by_name['Support'] = _SUPPORT
+
+Support = _reflection.GeneratedProtocolMessageType('Support', (_message.Message,), dict(
+  DESCRIPTOR = _SUPPORT,
+  __module__ = 'support_pb2'
+  # @@protoc_insertion_point(class_scope:pb.Support)
+  ))
+_sym_db.RegisterMessage(Support)
+
+
+# @@protoc_insertion_point(module_scope)

lbry/schema/types/v2/wallet.json

@@ -0,0 +1,139 @@
+{
+  "title": "Wallet",
+  "description": "An LBC wallet",
+  "type": "object",
+  "required": ["name", "version", "accounts", "preferences"],
+  "additionalProperties": false,
+  "properties": {
+    "name": {
+      "description": "Human readable name for this wallet",
+      "type": "string"
+    },
+    "version": {
+      "description": "Wallet spec version",
+      "type": "integer",
+      "$comment": "Should this be a string? We may need some sort of decimal type if we want exact decimal versions."
+    },
+    "accounts": {
+      "description": "Accounts associated with this wallet",
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["address_generator", "certificates", "encrypted", "ledger", "modified_on", "name", "private_key", "public_key", "seed"],
+        "additionalProperties": false,
+        "properties": {
+          "address_generator": {
+            "description": "Higher level manager of either singular or deterministically generated addresses",
+            "type": "object",
+            "oneOf": [
+              {
+                "required": ["name", "change", "receiving"],
+                "additionalProperties": false,
+                "properties": {
+                  "name": {
+                    "description": "type of address generator: a deterministic chain of addresses",
+                    "enum": ["deterministic-chain"],
+                    "type": "string"
+                  },
+                  "change": {
+                    "$ref": "#/$defs/address_manager",
+                    "description": "Manager for deterministically generated change address (not used for single address)"
+                  },
+                  "receiving": {
+                    "$ref": "#/$defs/address_manager",
+                    "description": "Manager for deterministically generated receiving address (not used for single address)"
+                  }
+                }
+              }, {
+                "required": ["name"],
+                "additionalProperties": false,
+                "properties": {
+                  "name": {
+                    "description": "type of address generator: a single address",
+                    "enum": ["single-address"],
+                    "type": "string"
+                  }
+                }
+              }
+            ]
+          },
+          "certificates": {
+            "type": "object",
+            "description": "Channel keys. Mapping from public key address to pem-formatted private key.",
+            "additionalProperties": {"type": "string"}
+          },
+          "encrypted": {
+            "type": "boolean",
+            "description": "Whether private key and seed are encrypted with a password"
+          },
+          "ledger": {
+            "description": "Which network to use",
+            "type": "string",
+            "examples": [
+              "lbc_mainnet",
+              "lbc_testnet"
+            ]
+          },
+          "modified_on": {
+            "description": "last modified time in Unix Time",
+            "type": "integer"
+          },
+          "name": {
+            "description": "Name for account, possibly human readable",
+            "type": "string"
+          },
+          "private_key": {
+            "description": "Private key for address if `address_generator` is a single address. Root of chain of private keys for addresses if `address_generator` is a deterministic chain of addresses. Encrypted if `encrypted` is true.",
+            "type": "string"
+          },
+          "public_key": {
+            "description": "Public key for address if `address_generator` is a single address. Root of chain of public keys for addresses if `address_generator` is a deterministic chain of addresses.",
+            "type": "string"
+          },
+          "seed": {
+            "description": "Human readable representation of `private_key`. encrypted if `encrypted` is set to `true`",
+            "type": "string"
+          }
+        }
+      }
+    },
+    "preferences": {
+      "description": "Timestamped application-level preferences. Values can be objects or of a primitive type.",
+      "$comment": "enable-sync is seen in example wallet. encrypt-on-disk is seen in example wallet. they both have a boolean `value` field. Do we want them explicitly defined here? local and shared seem to have at least a similar structure (type, value [yes, again], version), value being the free-form part. Should we define those here? Or can there be any key under preferences, and `value` be literally be anything in any form?",
+      "type": "object",
+      "additionalProperties": {
+        "type": "object",
+        "required": ["ts", "value"],
+        "additionalProperties": false,
+        "properties": {
+          "ts": {
+            "type": "number",
+            "description": "When the item was set, in Unix time format.",
+            "$comment": "Do we want a string (decimal)?"
+          },
+          "value": {
+            "$comment": "Sometimes this has been an object, sometimes just a boolean. I don't want to prescribe anything."
+          }
+        }
+      }
+    }
+  },
+  "$defs": {
+    "address_manager": {
+      "description": "Manager for deterministically generated addresses",
+      "type": "object",
+      "required": ["gap", "maximum_uses_per_address"],
+      "additionalProperties": false,
+      "properties": {
+        "gap": {
+          "description": "Maximum allowed consecutive generated addresses with no transactions",
+          "type": "integer"
+        },
+        "maximum_uses_per_address": {
+          "description": "Maximum number of uses for each generated address",
+          "type": "integer"
+        }
+      }
+    }
+  }
+}

lbry/schema/url.py

@@ -22,8 +22,7 @@ def _create_url_regex():
         return _group(
             _named(name+"_name", prefix + invalid_names_regex) +
             _oneof(
-                _group('#' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
-                _group(':' + _named(name+"_sequence", '[1-9][0-9]*')),
+                _group('[:#]' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
                 _group(r'\$' + _named(name+"_amount_order", '[1-9][0-9]*'))
             ) + '?'
         )
@@ -50,28 +49,31 @@ def normalize_name(name):
 class PathSegment(NamedTuple):
     name: str
     claim_id: str = None
-    sequence: int = None
     amount_order: int = None
 
     @property
     def normalized(self):
         return normalize_name(self.name)
 
+    @property
+    def is_shortid(self):
+        return self.claim_id is not None and len(self.claim_id) < 40
+
+    @property
+    def is_fullid(self):
+        return self.claim_id is not None and len(self.claim_id) == 40
+
     def to_dict(self):
         q = {'name': self.name}
         if self.claim_id is not None:
             q['claim_id'] = self.claim_id
-        if self.sequence is not None:
-            q['sequence'] = self.sequence
         if self.amount_order is not None:
             q['amount_order'] = self.amount_order
         return q
 
     def __str__(self):
         if self.claim_id is not None:
-            return f"{self.name}#{self.claim_id}"
-        elif self.sequence is not None:
-            return f"{self.name}:{self.sequence}"
+            return f"{self.name}:{self.claim_id}"
         elif self.amount_order is not None:
             return f"{self.name}${self.amount_order}"
         return self.name
@@ -118,7 +120,6 @@ class URL(NamedTuple):
                 segments[segment] = PathSegment(
                     parts[f'{segment}_name'],
                     parts[f'{segment}_claim_id'],
-                    parts[f'{segment}_sequence'],
                     parts[f'{segment}_amount_order']
                 )
 

lbry/stream/background_downloader.py

@@ -0,0 +1,31 @@
+import asyncio
+import logging
+
+from lbry.stream.downloader import StreamDownloader
+
+
+log = logging.getLogger(__name__)
+
+
+class BackgroundDownloader:
+    def __init__(self, conf, storage, blob_manager, dht_node=None):
+        self.storage = storage
+        self.blob_manager = blob_manager
+        self.node = dht_node
+        self.conf = conf
+
+    async def download_blobs(self, sd_hash):
+        downloader = StreamDownloader(asyncio.get_running_loop(), self.conf, self.blob_manager, sd_hash)
+        try:
+            await downloader.start(self.node, save_stream=False)
+            for blob_info in downloader.descriptor.blobs[:-1]:
+                await downloader.download_stream_blob(blob_info)
+        except ValueError:
+            return
+        except asyncio.CancelledError:
+            log.debug("Cancelled background downloader")
+            raise
+        except Exception:
+            log.error("Unexpected download error on background downloader")
+        finally:
+            downloader.stop()

lbry/stream/descriptor.py

@@ -4,6 +4,7 @@ import binascii
 import logging
 import typing
 import asyncio
+import time
 import re
 from collections import OrderedDict
 from cryptography.hazmat.primitives.ciphers.algorithms import AES
@@ -44,18 +45,25 @@ def random_iv_generator() -> typing.Generator[bytes, None, None]:
         yield os.urandom(AES.block_size // 8)
 
 
-def file_reader(file_path: str):
+def read_bytes(file_path: str, offset: int, to_read: int):
+    with open(file_path, 'rb') as f:
+        f.seek(offset)
+        return f.read(to_read)
+
+
+async def file_reader(file_path: str):
     length = int(os.stat(file_path).st_size)
     offset = 0
 
-    with open(file_path, 'rb') as stream_file:
-        while offset < length:
-            bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1)
-            if not bytes_to_read:
-                break
-            blob_bytes = stream_file.read(bytes_to_read)
-            yield blob_bytes
-            offset += bytes_to_read
+    while offset < length:
+        bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1)
+        if not bytes_to_read:
+            break
+        blob_bytes = await asyncio.get_event_loop().run_in_executor(
+            None, read_bytes, file_path, offset, bytes_to_read
+        )
+        yield blob_bytes
+        offset += bytes_to_read
 
 
 def sanitize_file_name(dirty_name: str, default_file_name: str = 'lbry_download'):
@@ -145,15 +153,19 @@ class StreamDescriptor:
         h.update(self.old_sort_json())
         return h.hexdigest()
 
-    async def make_sd_blob(self, blob_file_obj: typing.Optional[AbstractBlob] = None,
-                           old_sort: typing.Optional[bool] = False,
-                           blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None):
+    async def make_sd_blob(
+            self, blob_file_obj: typing.Optional[AbstractBlob] = None, old_sort: typing.Optional[bool] = False,
+            blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None,
+            added_on: float = None, is_mine: bool = False
+        ):
         sd_hash = self.calculate_sd_hash() if not old_sort else self.calculate_old_sort_sd_hash()
         if not old_sort:
             sd_data = self.as_json()
         else:
             sd_data = self.old_sort_json()
-        sd_blob = blob_file_obj or BlobFile(self.loop, sd_hash, len(sd_data), blob_completed_callback, self.blob_dir)
+        sd_blob = blob_file_obj or BlobFile(
+            self.loop, sd_hash, len(sd_data), blob_completed_callback, self.blob_dir, added_on, is_mine
+        )
         if blob_file_obj:
             blob_file_obj.set_length(len(sd_data))
         if not sd_blob.get_is_verified():
@@ -176,18 +188,19 @@ class StreamDescriptor:
             raise InvalidStreamDescriptorError("Does not decode as valid JSON")
         if decoded['blobs'][-1]['length'] != 0:
             raise InvalidStreamDescriptorError("Does not end with a zero-length blob.")
-        if any([blob_info['length'] == 0 for blob_info in decoded['blobs'][:-1]]):
+        if any(blob_info['length'] == 0 for blob_info in decoded['blobs'][:-1]):
             raise InvalidStreamDescriptorError("Contains zero-length data blob")
         if 'blob_hash' in decoded['blobs'][-1]:
             raise InvalidStreamDescriptorError("Stream terminator blob should not have a hash")
-        if any([i != blob_info['blob_num'] for i, blob_info in enumerate(decoded['blobs'])]):
+        if any(i != blob_info['blob_num'] for i, blob_info in enumerate(decoded['blobs'])):
             raise InvalidStreamDescriptorError("Stream contains out of order or skipped blobs")
+        added_on = time.time()
         descriptor = cls(
             loop, blob_dir,
             binascii.unhexlify(decoded['stream_name']).decode(),
             decoded['key'],
             binascii.unhexlify(decoded['suggested_file_name']).decode(),
-            [BlobInfo(info['blob_num'], info['length'], info['iv'], info.get('blob_hash'))
+            [BlobInfo(info['blob_num'], info['length'], info['iv'], added_on, info.get('blob_hash'))
              for info in decoded['blobs']],
             decoded['stream_hash'],
             blob.blob_hash
@@ -245,20 +258,25 @@ class StreamDescriptor:
         iv_generator = iv_generator or random_iv_generator()
         key = key or os.urandom(AES.block_size // 8)
         blob_num = -1
-        for blob_bytes in file_reader(file_path):
+        added_on = time.time()
+        async for blob_bytes in file_reader(file_path):
             blob_num += 1
             blob_info = await BlobFile.create_from_unencrypted(
-                loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num, blob_completed_callback
+                loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num, added_on, True, blob_completed_callback
             )
             blobs.append(blob_info)
         blobs.append(
-            BlobInfo(len(blobs), 0, binascii.hexlify(next(iv_generator)).decode()))  # add the stream terminator
+            # add the stream terminator
+            BlobInfo(len(blobs), 0, binascii.hexlify(next(iv_generator)).decode(), added_on, None, True)
+        )
         file_name = os.path.basename(file_path)
         suggested_file_name = sanitize_file_name(file_name)
         descriptor = cls(
             loop, blob_dir, file_name, binascii.hexlify(key).decode(), suggested_file_name, blobs
         )
-        sd_blob = await descriptor.make_sd_blob(old_sort=old_sort, blob_completed_callback=blob_completed_callback)
+        sd_blob = await descriptor.make_sd_blob(
+            old_sort=old_sort, blob_completed_callback=blob_completed_callback, added_on=added_on, is_mine=True
+        )
         descriptor.sd_hash = sd_blob.blob_hash
         return descriptor
 

lbry/stream/downloader.py

@@ -3,11 +3,13 @@ import typing
 import logging
 import binascii
 
-from lbry.dht.peer import make_kademlia_peer
+from lbry.dht.node import get_kademlia_peers_from_hosts
 from lbry.error import DownloadSDTimeoutError
-from lbry.utils import resolve_host, lru_cache_concurrent
+from lbry.utils import lru_cache_concurrent
 from lbry.stream.descriptor import StreamDescriptor
 from lbry.blob_exchange.downloader import BlobDownloader
+from lbry.torrent.tracker import enqueue_tracker_search
+
 if typing.TYPE_CHECKING:
     from lbry.conf import Config
     from lbry.dht.node import Node
@@ -25,8 +27,8 @@ class StreamDownloader:
         self.config = config
         self.blob_manager = blob_manager
         self.sd_hash = sd_hash
-        self.search_queue = asyncio.Queue(loop=loop)     # blob hashes to feed into the iterative finder
-        self.peer_queue = asyncio.Queue(loop=loop)       # new peers to try
+        self.search_queue = asyncio.Queue()     # blob hashes to feed into the iterative finder
+        self.peer_queue = asyncio.Queue()       # new peers to try
         self.blob_downloader = BlobDownloader(self.loop, self.config, self.blob_manager, self.peer_queue)
         self.descriptor: typing.Optional[StreamDescriptor] = descriptor
         self.node: typing.Optional['Node'] = None
@@ -40,7 +42,7 @@ class StreamDownloader:
         async def cached_read_blob(blob_info: 'BlobInfo') -> bytes:
             return await self.read_blob(blob_info, 2)
 
-        if self.blob_manager.decrypted_blob_lru_cache:
+        if self.blob_manager.decrypted_blob_lru_cache is not None:
             cached_read_blob = lru_cache_concurrent(override_lru_cache=self.blob_manager.decrypted_blob_lru_cache)(
                 cached_read_blob
             )
@@ -48,26 +50,19 @@ class StreamDownloader:
         self.cached_read_blob = cached_read_blob
 
     async def add_fixed_peers(self):
-        def _delayed_add_fixed_peers():
+        def _add_fixed_peers(fixed_peers):
+            self.peer_queue.put_nowait(fixed_peers)
             self.added_fixed_peers = True
-            self.peer_queue.put_nowait([
-                make_kademlia_peer(None, address, None, tcp_port=port + 1, allow_localhost=True)
-                for address, port in addresses
-            ])
 
-        if not self.config.reflector_servers:
+        if not self.config.fixed_peers:
             return
-        addresses = [
-            (await resolve_host(url, port + 1, proto='tcp'), port)
-            for url, port in self.config.reflector_servers
-        ]
         if 'dht' in self.config.components_to_skip or not self.node or not \
                 len(self.node.protocol.routing_table.get_peers()) > 0:
             self.fixed_peers_delay = 0.0
         else:
             self.fixed_peers_delay = self.config.fixed_peer_delay
-
-        self.fixed_peers_handle = self.loop.call_later(self.fixed_peers_delay, _delayed_add_fixed_peers)
+        fixed_peers = await get_kademlia_peers_from_hosts(self.config.fixed_peers)
+        self.fixed_peers_handle = self.loop.call_later(self.fixed_peers_delay, _add_fixed_peers, fixed_peers)
 
     async def load_descriptor(self, connection_id: int = 0):
         # download or get the sd blob
@@ -77,7 +72,7 @@ class StreamDownloader:
                 now = self.loop.time()
                 sd_blob = await asyncio.wait_for(
                     self.blob_downloader.download_blob(self.sd_hash, connection_id),
-                    self.config.blob_download_timeout, loop=self.loop
+                    self.config.blob_download_timeout
                 )
                 log.info("downloaded sd blob %s", self.sd_hash)
                 self.time_to_descriptor = self.loop.time() - now
@@ -90,14 +85,15 @@ class StreamDownloader:
         )
         log.info("loaded stream manifest %s", self.sd_hash)
 
-    async def start(self, node: typing.Optional['Node'] = None, connection_id: int = 0):
+    async def start(self, node: typing.Optional['Node'] = None, connection_id: int = 0, save_stream=True):
         # set up peer accumulation
-        if node:
-            self.node = node
+        self.node = node or self.node  # fixme: this shouldnt be set here!
+        if self.node:
             if self.accumulate_task and not self.accumulate_task.done():
                 self.accumulate_task.cancel()
             _, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue)
         await self.add_fixed_peers()
+        enqueue_tracker_search(bytes.fromhex(self.sd_hash), self.peer_queue)
         # start searching for peers for the sd hash
         self.search_queue.put_nowait(self.sd_hash)
         log.info("searching for peers for stream %s", self.sd_hash)
@@ -105,11 +101,7 @@ class StreamDownloader:
         if not self.descriptor:
             await self.load_descriptor(connection_id)
 
-        # add the head blob to the peer search
-        self.search_queue.put_nowait(self.descriptor.blobs[0].blob_hash)
-        log.info("added head blob to peer search for stream %s", self.sd_hash)
-
-        if not await self.blob_manager.storage.stream_exists(self.sd_hash):
+        if not await self.blob_manager.storage.stream_exists(self.sd_hash) and save_stream:
             await self.blob_manager.storage.store_stream(
                 self.blob_manager.get_blob(self.sd_hash, length=self.descriptor.length), self.descriptor
             )
@@ -119,7 +111,7 @@ class StreamDownloader:
             raise ValueError(f"blob {blob_info.blob_hash} is not part of stream with sd hash {self.sd_hash}")
         blob = await asyncio.wait_for(
             self.blob_downloader.download_blob(blob_info.blob_hash, blob_info.length, connection_id),
-            self.config.blob_download_timeout * 10, loop=self.loop
+            self.config.blob_download_timeout * 10
         )
         return blob
 

lbry/stream/managed_stream.py

@@ -3,9 +3,8 @@ import asyncio
 import time
 import typing
 import logging
-import binascii
+from typing import Optional
 from aiohttp.web import Request, StreamResponse, HTTPRequestRangeNotSatisfiable
-from lbry.utils import generate_id
 from lbry.error import DownloadSDTimeoutError
 from lbry.schema.mime_types import guess_media_type
 from lbry.stream.downloader import StreamDownloader
@@ -13,13 +12,12 @@ from lbry.stream.descriptor import StreamDescriptor, sanitize_file_name
 from lbry.stream.reflector.client import StreamReflectorClient
 from lbry.extras.daemon.storage import StoredContentClaim
 from lbry.blob import MAX_BLOB_SIZE
+from lbry.file.source import ManagedDownloadSource
 
 if typing.TYPE_CHECKING:
     from lbry.conf import Config
-    from lbry.schema.claim import Claim
     from lbry.blob.blob_manager import BlobManager
     from lbry.blob.blob_info import BlobInfo
-    from lbry.dht.node import Node
     from lbry.extras.daemon.analytics import AnalyticsManager
     from lbry.wallet.transaction import Transaction
 
@@ -40,76 +38,39 @@ async def get_next_available_file_name(loop: asyncio.AbstractEventLoop, download
     return await loop.run_in_executor(None, _get_next_available_file_name, download_directory, file_name)
 
 
-class ManagedStream:
-    STATUS_RUNNING = "running"
-    STATUS_STOPPED = "stopped"
-    STATUS_FINISHED = "finished"
-
-    SAVING_ID = 1
-    STREAMING_ID = 2
-
-    __slots__ = [
-        'loop',
-        'config',
-        'blob_manager',
-        'sd_hash',
-        'download_directory',
-        '_file_name',
-        '_added_on',
-        '_status',
-        'stream_claim_info',
-        'download_id',
-        'rowid',
-        'content_fee',
-        'purchase_receipt',
-        'downloader',
-        'analytics_manager',
-        'fully_reflected',
-        'file_output_task',
-        'delayed_stop_task',
-        'streaming_responses',
-        'streaming',
-        '_running',
-        'saving',
-        'finished_writing',
-        'started_writing',
-        'finished_write_attempt'
-    ]
-
+class ManagedStream(ManagedDownloadSource):
     def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
-                 sd_hash: str, download_directory: typing.Optional[str] = None, file_name: typing.Optional[str] = None,
-                 status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional[StoredContentClaim] = None,
-                 download_id: typing.Optional[str] = None, rowid: typing.Optional[int] = None,
-                 descriptor: typing.Optional[StreamDescriptor] = None,
-                 content_fee: typing.Optional['Transaction'] = None,
-                 analytics_manager: typing.Optional['AnalyticsManager'] = None,
-                 added_on: typing.Optional[int] = None):
-        self.loop = loop
-        self.config = config
+                 sd_hash: str, download_directory: Optional[str] = None, file_name: Optional[str] = None,
+                 status: Optional[str] = ManagedDownloadSource.STATUS_STOPPED,
+                 claim: Optional[StoredContentClaim] = None,
+                 download_id: Optional[str] = None, rowid: Optional[int] = None,
+                 descriptor: Optional[StreamDescriptor] = None,
+                 content_fee: Optional['Transaction'] = None,
+                 analytics_manager: Optional['AnalyticsManager'] = None,
+                 added_on: Optional[int] = None):
+        super().__init__(loop, config, blob_manager.storage, sd_hash, file_name, download_directory, status, claim,
+                         download_id, rowid, content_fee, analytics_manager, added_on)
         self.blob_manager = blob_manager
-        self.sd_hash = sd_hash
-        self.download_directory = download_directory
-        self._file_name = file_name
-        self._status = status
-        self.stream_claim_info = claim
-        self.download_id = download_id or binascii.hexlify(generate_id()).decode()
-        self.rowid = rowid
-        self.content_fee = content_fee
         self.purchase_receipt = None
-        self._added_on = added_on
         self.downloader = StreamDownloader(self.loop, self.config, self.blob_manager, sd_hash, descriptor)
         self.analytics_manager = analytics_manager
 
-        self.fully_reflected = asyncio.Event(loop=self.loop)
+        self.reflector_progress = 0
+        self.uploading_to_reflector = False
         self.file_output_task: typing.Optional[asyncio.Task] = None
         self.delayed_stop_task: typing.Optional[asyncio.Task] = None
         self.streaming_responses: typing.List[typing.Tuple[Request, StreamResponse]] = []
-        self.streaming = asyncio.Event(loop=self.loop)
-        self._running = asyncio.Event(loop=self.loop)
-        self.saving = asyncio.Event(loop=self.loop)
-        self.finished_writing = asyncio.Event(loop=self.loop)
-        self.started_writing = asyncio.Event(loop=self.loop)
-        self.finished_write_attempt = asyncio.Event(loop=self.loop)
+        self.fully_reflected = asyncio.Event()
+        self.streaming = asyncio.Event()
+        self._running = asyncio.Event()
+
+    @property
+    def sd_hash(self) -> str:
+        return self.identifier
+
+    @property
+    def is_fully_reflected(self) -> bool:
+        return self.fully_reflected.is_set()
 
     @property
     def descriptor(self) -> StreamDescriptor:
@@ -120,16 +81,20 @@ class ManagedStream:
         return self.descriptor.stream_hash
 
     @property
-    def file_name(self) -> typing.Optional[str]:
-        return self._file_name or (self.descriptor.suggested_file_name if self.descriptor else None)
+    def file_name(self) -> Optional[str]:
+        return self._file_name or self.suggested_file_name
 
     @property
-    def added_on(self) -> typing.Optional[int]:
-        return self._added_on
+    def suggested_file_name(self) -> Optional[str]:
+        first_option = ((self.descriptor and self.descriptor.suggested_file_name) or '').strip()
+        return sanitize_file_name(first_option or (self.stream_claim_info and self.stream_claim_info.claim and
+                                                   self.stream_claim_info.claim.stream.source.name))
 
     @property
-    def status(self) -> str:
-        return self._status
+    def stream_name(self) -> Optional[str]:
+        first_option = ((self.descriptor and self.descriptor.stream_name) or '').strip()
+        return first_option or (self.stream_claim_info and self.stream_claim_info.claim and
+                                self.stream_claim_info.claim.stream.source.name)
 
     @property
     def written_bytes(self) -> int:
@@ -148,55 +113,6 @@ class ManagedStream:
         self._status = status
         await self.blob_manager.storage.change_file_status(self.stream_hash, status)
 
-    @property
-    def finished(self) -> bool:
-        return self.status == self.STATUS_FINISHED
-
-    @property
-    def running(self) -> bool:
-        return self.status == self.STATUS_RUNNING
-
-    @property
-    def claim_id(self) -> typing.Optional[str]:
-        return None if not self.stream_claim_info else self.stream_claim_info.claim_id
-
-    @property
-    def txid(self) -> typing.Optional[str]:
-        return None if not self.stream_claim_info else self.stream_claim_info.txid
-
-    @property
-    def nout(self) -> typing.Optional[int]:
-        return None if not self.stream_claim_info else self.stream_claim_info.nout
-
-    @property
-    def outpoint(self) -> typing.Optional[str]:
-        return None if not self.stream_claim_info else self.stream_claim_info.outpoint
-
-    @property
-    def claim_height(self) -> typing.Optional[int]:
-        return None if not self.stream_claim_info else self.stream_claim_info.height
-
-    @property
-    def channel_claim_id(self) -> typing.Optional[str]:
-        return None if not self.stream_claim_info else self.stream_claim_info.channel_claim_id
-
-    @property
-    def channel_name(self) -> typing.Optional[str]:
-        return None if not self.stream_claim_info else self.stream_claim_info.channel_name
-
-    @property
-    def claim_name(self) -> typing.Optional[str]:
-        return None if not self.stream_claim_info else self.stream_claim_info.claim_name
-
-    @property
-    def metadata(self) -> typing.Optional[typing.Dict]:
-        return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict()
-
-    @property
-    def metadata_protobuf(self) -> bytes:
-        if self.stream_claim_info:
-            return binascii.hexlify(self.stream_claim_info.claim.to_bytes())
-
     @property
     def blobs_completed(self) -> int:
         return sum([1 if b.blob_hash in self.blob_manager.completed_blob_hashes else 0
@@ -210,39 +126,34 @@ class ManagedStream:
     def blobs_remaining(self) -> int:
         return self.blobs_in_stream - self.blobs_completed
 
-    @property
-    def full_path(self) -> typing.Optional[str]:
-        return os.path.join(self.download_directory, os.path.basename(self.file_name)) \
-            if self.file_name and self.download_directory else None
-
-    @property
-    def output_file_exists(self):
-        return os.path.isfile(self.full_path) if self.full_path else False
-
     @property
     def mime_type(self):
-        return guess_media_type(os.path.basename(self.descriptor.suggested_file_name))[0]
+        return guess_media_type(os.path.basename(self.suggested_file_name))[0]
 
-    @classmethod
-    async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
-                     file_path: str, key: typing.Optional[bytes] = None,
-                     iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedStream':
-        """
-        Generate a stream from a file and save it to the db
-        """
-        descriptor = await StreamDescriptor.create_stream(
-            loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
-            blob_completed_callback=blob_manager.blob_completed
-        )
-        await blob_manager.storage.store_stream(
-            blob_manager.get_blob(descriptor.sd_hash), descriptor
-        )
-        row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path),
-                                                                os.path.dirname(file_path), 0)
-        return cls(loop, config, blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
-                   os.path.basename(file_path), status=cls.STATUS_FINISHED, rowid=row_id, descriptor=descriptor)
+    @property
+    def download_path(self):
+        return f"{self.download_directory}/{self._file_name}" if self.download_directory and self._file_name else None
 
-    async def start(self, node: typing.Optional['Node'] = None, timeout: typing.Optional[float] = None,
+    # @classmethod
+    # async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config',
+    #                  file_path: str, key: Optional[bytes] = None,
+    #                  iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedDownloadSource':
+    #     """
+    #     Generate a stream from a file and save it to the db
+    #     """
+    #     descriptor = await StreamDescriptor.create_stream(
+    #         loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
+    #         blob_completed_callback=blob_manager.blob_completed
+    #     )
+    #     await blob_manager.storage.store_stream(
+    #         blob_manager.get_blob(descriptor.sd_hash), descriptor
+    #     )
+    #     row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path),
+    #                                                             os.path.dirname(file_path), 0)
+    #     return cls(loop, config, blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
+    #                os.path.basename(file_path), status=cls.STATUS_FINISHED, rowid=row_id, descriptor=descriptor)
+
+    async def start(self, timeout: Optional[float] = None,
                     save_now: bool = False):
         timeout = timeout or self.config.download_timeout
         if self._running.is_set():
@@ -250,7 +161,7 @@ class ManagedStream:
         log.info("start downloader for stream (sd hash: %s)", self.sd_hash)
         self._running.set()
         try:
-            await asyncio.wait_for(self.downloader.start(node), timeout, loop=self.loop)
+            await asyncio.wait_for(self.downloader.start(), timeout)
         except asyncio.TimeoutError:
             self._running.clear()
             raise DownloadSDTimeoutError(self.sd_hash)
@@ -260,6 +171,11 @@ class ManagedStream:
         self.delayed_stop_task = self.loop.create_task(self._delayed_stop())
         if not await self.blob_manager.storage.file_exists(self.sd_hash):
             if save_now:
+                if not self._file_name:
+                    self._file_name = await get_next_available_file_name(
+                        self.loop, self.download_directory,
+                        self._file_name or sanitize_file_name(self.suggested_file_name)
+                    )
                 file_name, download_dir = self._file_name, self.download_directory
             else:
                 file_name, download_dir = None, None
@@ -275,11 +191,11 @@ class ManagedStream:
         Stop any running save/stream tasks as well as the downloader and update the status in the database
         """
 
-        self.stop_tasks()
+        await self.stop_tasks()
         if (finished and self.status != self.STATUS_FINISHED) or self.status == self.STATUS_RUNNING:
             await self.update_status(self.STATUS_FINISHED if finished else self.STATUS_STOPPED)
 
-    async def _aiter_read_stream(self, start_blob_num: typing.Optional[int] = 0, connection_id: int = 0)\
+    async def _aiter_read_stream(self, start_blob_num: Optional[int] = 0, connection_id: int = 0)\
             -> typing.AsyncIterator[typing.Tuple['BlobInfo', bytes]]:
         if start_blob_num >= len(self.descriptor.blobs[:-1]):
             raise IndexError(start_blob_num)
@@ -291,13 +207,13 @@ class ManagedStream:
                 decrypted = await self.downloader.read_blob(blob_info, connection_id)
             yield (blob_info, decrypted)
 
-    async def stream_file(self, request: Request, node: typing.Optional['Node'] = None) -> StreamResponse:
+    async def stream_file(self, request: Request) -> StreamResponse:
         log.info("stream file to browser for lbry://%s#%s (sd hash %s...)", self.claim_name, self.claim_id,
                  self.sd_hash[:6])
         headers, size, skip_blobs, first_blob_start_offset = self._prepare_range_response_headers(
             request.headers.get('range', 'bytes=0-')
         )
-        await self.start(node)
+        await self.start()
         response = StreamResponse(
             status=206,
             headers=headers
@@ -335,9 +251,10 @@ class ManagedStream:
                 self.streaming.clear()
 
     @staticmethod
-    def _write_decrypted_blob(handle: typing.IO, data: bytes):
-        handle.write(data)
-        handle.flush()
+    def _write_decrypted_blob(output_path: str, data: bytes):
+        with open(output_path, 'ab') as handle:
+            handle.write(data)
+            handle.flush()
 
     async def _save_file(self, output_path: str):
         log.info("save file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id, self.sd_hash[:6],
@@ -347,12 +264,12 @@ class ManagedStream:
         self.finished_writing.clear()
         self.started_writing.clear()
         try:
-            with open(output_path, 'wb') as file_write_handle:
-                async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
-                    log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
-                    await self.loop.run_in_executor(None, self._write_decrypted_blob, file_write_handle, decrypted)
-                    if not self.started_writing.is_set():
-                        self.started_writing.set()
+            open(output_path, 'wb').close()  # pylint: disable=consider-using-with
+            async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
+                log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
+                await self.loop.run_in_executor(None, self._write_decrypted_blob, output_path, decrypted)
+                if not self.started_writing.is_set():
+                    self.started_writing.set()
             await self.update_status(ManagedStream.STATUS_FINISHED)
             if self.analytics_manager:
                 self.loop.create_task(self.analytics_manager.send_download_finished(
@@ -362,7 +279,7 @@ class ManagedStream:
             log.info("finished saving file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id,
                      self.sd_hash[:6], self.full_path)
             await self.blob_manager.storage.set_saved_file(self.stream_hash)
-        except Exception as err:
+        except (Exception, asyncio.CancelledError) as err:
             if os.path.isfile(output_path):
                 log.warning("removing incomplete download %s for %s", output_path, self.sd_hash)
                 os.remove(output_path)
@@ -382,22 +299,21 @@ class ManagedStream:
             self.saving.clear()
             self.finished_write_attempt.set()
 
-    async def save_file(self, file_name: typing.Optional[str] = None, download_directory: typing.Optional[str] = None,
-                        node: typing.Optional['Node'] = None):
-        await self.start(node)
+    async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
+        await self.start()
         if self.file_output_task and not self.file_output_task.done():  # cancel an already running save task
             self.file_output_task.cancel()
         self.download_directory = download_directory or self.download_directory or self.config.download_dir
         if not self.download_directory:
             raise ValueError("no directory to download to")
-        if not (file_name or self._file_name or self.descriptor.suggested_file_name):
+        if not (file_name or self._file_name or self.suggested_file_name):
             raise ValueError("no file name to download to")
         if not os.path.isdir(self.download_directory):
             log.warning("download directory '%s' does not exist, attempting to make it", self.download_directory)
             os.mkdir(self.download_directory)
         self._file_name = await get_next_available_file_name(
             self.loop, self.download_directory,
-            file_name or self._file_name or sanitize_file_name(self.descriptor.suggested_file_name)
+            file_name or self._file_name or sanitize_file_name(self.suggested_file_name)
         )
         await self.blob_manager.storage.change_file_download_dir_and_file_name(
             self.stream_hash, self.download_directory, self.file_name
@@ -405,15 +321,16 @@ class ManagedStream:
         await self.update_status(ManagedStream.STATUS_RUNNING)
         self.file_output_task = self.loop.create_task(self._save_file(self.full_path))
         try:
-            await asyncio.wait_for(self.started_writing.wait(), self.config.download_timeout, loop=self.loop)
+            await asyncio.wait_for(self.started_writing.wait(), self.config.download_timeout)
         except asyncio.TimeoutError:
             log.warning("timeout starting to write data for lbry://%s#%s", self.claim_name, self.claim_id)
-            self.stop_tasks()
+            await self.stop_tasks()
             await self.update_status(ManagedStream.STATUS_STOPPED)
 
-    def stop_tasks(self):
+    async def stop_tasks(self):
         if self.file_output_task and not self.file_output_task.done():
             self.file_output_task.cancel()
+            await asyncio.gather(self.file_output_task, return_exceptions=True)
         self.file_output_task = None
         while self.streaming_responses:
             req, response = self.streaming_responses.pop()
@@ -426,12 +343,13 @@ class ManagedStream:
         sent = []
         protocol = StreamReflectorClient(self.blob_manager, self.descriptor)
         try:
+            self.uploading_to_reflector = True
             await self.loop.create_connection(lambda: protocol, host, port)
             await protocol.send_handshake()
             sent_sd, needed = await protocol.send_descriptor()
-            if sent_sd:
+            if sent_sd:  # reflector needed the sd blob
                 sent.append(self.sd_hash)
-            if not sent_sd and not needed:
+            if not sent_sd and not needed:  # reflector already has the stream
                 if not self.fully_reflected.is_set():
                     self.fully_reflected.set()
                     await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
@@ -441,30 +359,33 @@ class ManagedStream:
             ]
             log.info("we have %i/%i needed blobs needed by reflector for lbry://%s#%s", len(we_have), len(needed),
                      self.claim_name, self.claim_id)
-            for blob_hash in we_have:
+            for i, blob_hash in enumerate(we_have):
                 await protocol.send_blob(blob_hash)
                 sent.append(blob_hash)
+                self.reflector_progress = int((i + 1) / len(we_have) * 100)
         except (asyncio.TimeoutError, ValueError):
             return sent
-        except ConnectionRefusedError:
+        except ConnectionError:
+            return sent
+        except (OSError, Exception, asyncio.CancelledError) as err:
+            if isinstance(err, asyncio.CancelledError):
+                log.warning("stopped uploading %s#%s to reflector", self.claim_name, self.claim_id)
+            elif isinstance(err, OSError):
+                log.warning(
+                    "stopped uploading %s#%s to reflector because blobs were deleted or moved", self.claim_name,
+                    self.claim_id
+                )
+            else:
+                log.exception("unexpected error reflecting %s#%s", self.claim_name, self.claim_id)
             return sent
         finally:
             if protocol.transport:
                 protocol.transport.close()
-        if not self.fully_reflected.is_set():
-            self.fully_reflected.set()
-            await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
+            self.uploading_to_reflector = False
+
         return sent
 
-    def set_claim(self, claim_info: typing.Dict, claim: 'Claim'):
-        self.stream_claim_info = StoredContentClaim(
-            f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
-            claim_info['name'], claim_info['amount'], claim_info['height'],
-            binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'],
-            claim_info['claim_sequence'], claim_info.get('channel_name')
-        )
-
-    async def update_content_claim(self, claim_info: typing.Optional[typing.Dict] = None):
+    async def update_content_claim(self, claim_info: Optional[typing.Dict] = None):
         if not claim_info:
             claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash)
         self.set_claim(claim_info, claim_info['value'])
@@ -481,7 +402,7 @@ class ManagedStream:
                          self.sd_hash[:6])
                 await self.stop()
                 return
-            await asyncio.sleep(1, loop=self.loop)
+            await asyncio.sleep(1)
 
     def _prepare_range_response_headers(self, get_range: str) -> typing.Tuple[typing.Dict[str, str], int, int, int]:
         if '=' in get_range:

lbry/stream/reflector/client.py

@@ -35,6 +35,8 @@ class StreamReflectorClient(asyncio.Protocol):
     def connection_lost(self, exc: typing.Optional[Exception]):
         self.transport = None
         self.connected.clear()
+        if self.pending_request:
+            self.pending_request.cancel()
         if self.reflected_blobs:
             log.info("Finished sending reflector %i blobs", len(self.reflected_blobs))
 
@@ -56,12 +58,18 @@ class StreamReflectorClient(asyncio.Protocol):
                 self.response_buff = b''
             return
 
-    async def send_request(self, request_dict: typing.Dict):
-        msg = json.dumps(request_dict)
-        self.transport.write(msg.encode())
+    async def send_request(self, request_dict: typing.Dict, timeout: int = 180):
+        msg = json.dumps(request_dict, sort_keys=True)
         try:
-            self.pending_request = self.loop.create_task(self.response_queue.get())
+            self.transport.write(msg.encode())
+            self.pending_request = self.loop.create_task(asyncio.wait_for(self.response_queue.get(), timeout))
             return await self.pending_request
+        except (AttributeError, asyncio.CancelledError) as err:
+            # attribute error happens when we transport.write after disconnect
+            # cancelled error happens when the pending_request task is cancelled by a disconnect
+            if self.transport:
+                self.transport.close()
+            raise err if isinstance(err, asyncio.CancelledError) else asyncio.CancelledError()
         finally:
             self.pending_request = None
 
@@ -86,8 +94,16 @@ class StreamReflectorClient(asyncio.Protocol):
         needed = response.get('needed_blobs', [])
         sent_sd = False
         if response['send_sd_blob']:
-            await sd_blob.sendfile(self)
-            received = await self.response_queue.get()
+            try:
+                sent = await sd_blob.sendfile(self)
+                if sent == -1:
+                    log.warning("failed to send sd blob")
+                    raise asyncio.CancelledError()
+                received = await asyncio.wait_for(self.response_queue.get(), 30)
+            except asyncio.CancelledError as err:
+                if self.transport:
+                    self.transport.close()
+                raise err
             if received.get('received_sd_blob'):
                 sent_sd = True
                 if not needed:
@@ -110,8 +126,16 @@ class StreamReflectorClient(asyncio.Protocol):
         if 'send_blob' not in response:
             raise ValueError("I don't know whether to send the blob or not!")
         if response['send_blob']:
-            await blob.sendfile(self)
-            received = await self.response_queue.get()
+            try:
+                sent = await blob.sendfile(self)
+                if sent == -1:
+                    log.warning("failed to send blob")
+                    raise asyncio.CancelledError()
+                received = await asyncio.wait_for(self.response_queue.get(), 30)
+            except asyncio.CancelledError as err:
+                if self.transport:
+                    self.transport.close()
+                raise err
             if received.get('received_blob'):
                 self.reflected_blobs.append(blob.blob_hash)
                 log.info("Sent reflector blob %s", blob.blob_hash[:8])

lbry/stream/reflector/server.py

@@ -15,11 +15,13 @@ log = logging.getLogger(__name__)
 
 
 class ReflectorServerProtocol(asyncio.Protocol):
-    def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000):
+    def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000,
+                 stop_event: asyncio.Event = None, incoming_event: asyncio.Event = None,
+                 not_incoming_event: asyncio.Event = None, partial_event: asyncio.Event = None):
         self.loop = asyncio.get_event_loop()
         self.blob_manager = blob_manager
         self.server_task: asyncio.Task = None
-        self.started_listening = asyncio.Event(loop=self.loop)
+        self.started_listening = asyncio.Event()
         self.buf = b''
         self.transport: asyncio.StreamWriter = None
         self.writer: typing.Optional['HashBlobWriter'] = None
@@ -27,11 +29,26 @@ class ReflectorServerProtocol(asyncio.Protocol):
         self.descriptor: typing.Optional['StreamDescriptor'] = None
         self.sd_blob: typing.Optional['BlobFile'] = None
         self.received = []
-        self.incoming = asyncio.Event(loop=self.loop)
+        self.incoming = incoming_event or asyncio.Event()
+        self.not_incoming = not_incoming_event or asyncio.Event()
+        self.stop_event = stop_event or asyncio.Event()
         self.chunk_size = response_chunk_size
+        self.wait_for_stop_task: typing.Optional[asyncio.Task] = None
+        self.partial_event = partial_event
+
+    async def wait_for_stop(self):
+        await self.stop_event.wait()
+        if self.transport:
+            self.transport.close()
 
     def connection_made(self, transport):
         self.transport = transport
+        self.wait_for_stop_task = self.loop.create_task(self.wait_for_stop())
+
+    def connection_lost(self, exc):
+        if self.wait_for_stop_task:
+            self.wait_for_stop_task.cancel()
+            self.wait_for_stop_task = None
 
     def data_received(self, data: bytes):
         if self.incoming.is_set():
@@ -73,10 +90,11 @@ class ReflectorServerProtocol(asyncio.Protocol):
             self.sd_blob = self.blob_manager.get_blob(request['sd_blob_hash'], request['sd_blob_size'])
             if not self.sd_blob.get_is_verified():
                 self.writer = self.sd_blob.get_blob_writer(self.transport.get_extra_info('peername'))
+                self.not_incoming.clear()
                 self.incoming.set()
                 self.send_response({"send_sd_blob": True})
                 try:
-                    await asyncio.wait_for(self.sd_blob.verified.wait(), 30, loop=self.loop)
+                    await asyncio.wait_for(self.sd_blob.verified.wait(), 30)
                     self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
                         self.loop, self.blob_manager.blob_dir, self.sd_blob
                     )
@@ -86,6 +104,7 @@ class ReflectorServerProtocol(asyncio.Protocol):
                     self.transport.close()
                 finally:
                     self.incoming.clear()
+                    self.not_incoming.set()
                     self.writer.close_handle()
                     self.writer = None
             else:
@@ -93,13 +112,18 @@ class ReflectorServerProtocol(asyncio.Protocol):
                     self.loop, self.blob_manager.blob_dir, self.sd_blob
                 )
                 self.incoming.clear()
+                self.not_incoming.set()
                 if self.writer:
                     self.writer.close_handle()
                     self.writer = None
-                self.send_response({"send_sd_blob": False, 'needed': [
-                    blob.blob_hash for blob in self.descriptor.blobs[:-1]
-                    if not self.blob_manager.get_blob(blob.blob_hash).get_is_verified()
-                ]})
+
+                needs = [blob.blob_hash
+                         for blob in self.descriptor.blobs[:-1]
+                         if not self.blob_manager.get_blob(blob.blob_hash).get_is_verified()]
+                if needs and not self.partial_event.is_set():
+                    needs = needs[:3]
+                    self.partial_event.set()
+                self.send_response({"send_sd_blob": False, 'needed_blobs': needs})
                 return
             return
         elif self.descriptor:
@@ -112,14 +136,16 @@ class ReflectorServerProtocol(asyncio.Protocol):
             blob = self.blob_manager.get_blob(request['blob_hash'], request['blob_size'])
             if not blob.get_is_verified():
                 self.writer = blob.get_blob_writer(self.transport.get_extra_info('peername'))
+                self.not_incoming.clear()
                 self.incoming.set()
                 self.send_response({"send_blob": True})
                 try:
-                    await asyncio.wait_for(blob.verified.wait(), 30, loop=self.loop)
+                    await asyncio.wait_for(blob.verified.wait(), 30)
                     self.send_response({"received_blob": True})
                 except asyncio.TimeoutError:
                     self.send_response({"received_blob": False})
                 self.incoming.clear()
+                self.not_incoming.set()
                 self.writer.close_handle()
                 self.writer = None
             else:
@@ -130,26 +156,39 @@ class ReflectorServerProtocol(asyncio.Protocol):
 
 
 class ReflectorServer:
-    def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000):
+    def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000,
+                 stop_event: asyncio.Event = None, incoming_event: asyncio.Event = None,
+                 not_incoming_event: asyncio.Event = None, partial_needs=False):
         self.loop = asyncio.get_event_loop()
         self.blob_manager = blob_manager
         self.server_task: typing.Optional[asyncio.Task] = None
-        self.started_listening = asyncio.Event(loop=self.loop)
+        self.started_listening = asyncio.Event()
+        self.stopped_listening = asyncio.Event()
+        self.incoming_event = incoming_event or asyncio.Event()
+        self.not_incoming_event = not_incoming_event or asyncio.Event()
         self.response_chunk_size = response_chunk_size
+        self.stop_event = stop_event
+        self.partial_needs = partial_needs  # for testing cases where it doesn't know what it wants
 
     def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
         if self.server_task is not None:
             raise Exception("already running")
 
         async def _start_server():
-            server = await self.loop.create_server(
-                lambda: ReflectorServerProtocol(self.blob_manager, self.response_chunk_size),
-                interface, port
-            )
+            partial_event = asyncio.Event()
+            if not self.partial_needs:
+                partial_event.set()
+            server = await self.loop.create_server(lambda: ReflectorServerProtocol(
+                self.blob_manager, self.response_chunk_size, self.stop_event, self.incoming_event,
+                self.not_incoming_event, partial_event), interface, port)
             self.started_listening.set()
+            self.stopped_listening.clear()
             log.info("Reflector server listening on TCP %s:%i", interface, port)
-            async with server:
-                await server.serve_forever()
+            try:
+                async with server:
+                    await server.serve_forever()
+            finally:
+                self.stopped_listening.set()
 
         self.server_task = self.loop.create_task(_start_server())
 

lbry/stream/stream_manager.py

@@ -6,86 +6,71 @@ import random
 import typing
 from typing import Optional
 from aiohttp.web import Request
-from lbry.error import ResolveError, InvalidStreamDescriptorError, DownloadSDTimeoutError, InsufficientFundsError
-from lbry.error import ResolveTimeoutError, DownloadDataTimeoutError, KeyFeeAboveMaxAllowedError
-from lbry.utils import cache_concurrent
+from lbry.error import InvalidStreamDescriptorError
+from lbry.file.source_manager import SourceManager
 from lbry.stream.descriptor import StreamDescriptor
 from lbry.stream.managed_stream import ManagedStream
-from lbry.schema.claim import Claim
-from lbry.schema.url import URL
-from lbry.wallet.dewies import dewies_to_lbc
-from lbry.wallet import Output
-
+from lbry.file.source import ManagedDownloadSource
 if typing.TYPE_CHECKING:
     from lbry.conf import Config
     from lbry.blob.blob_manager import BlobManager
     from lbry.dht.node import Node
+    from lbry.wallet.wallet import WalletManager
+    from lbry.wallet.transaction import Transaction
     from lbry.extras.daemon.analytics import AnalyticsManager
     from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim
-    from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
 
 log = logging.getLogger(__name__)
 
-FILTER_FIELDS = [
-    'rowid',
-    'status',
-    'file_name',
-    'added_on',
-    'sd_hash',
-    'stream_hash',
-    'claim_name',
-    'claim_height',
-    'claim_id',
-    'outpoint',
-    'txid',
-    'nout',
-    'channel_claim_id',
-    'channel_name',
-    'full_status',  # TODO: remove
-    'blobs_remaining',
-    'blobs_in_stream'
-]
 
-COMPARISON_OPERATORS = {
-    'eq': lambda a, b: a == b,
-    'ne': lambda a, b: a != b,
-    'g': lambda a, b: a > b,
-    'l': lambda a, b: a < b,
-    'ge': lambda a, b: a >= b,
-    'le': lambda a, b: a <= b,
-}
-
-
-def path_or_none(path) -> Optional[str]:
-    if not path:
+def path_or_none(encoded_path) -> Optional[str]:
+    if not encoded_path:
         return
-    return binascii.unhexlify(path).decode()
+    return binascii.unhexlify(encoded_path).decode()
 
 
-class StreamManager:
+class StreamManager(SourceManager):
+    _sources: typing.Dict[str, ManagedStream]
+
+    filter_fields = SourceManager.filter_fields
+    filter_fields.update({
+        'sd_hash',
+        'stream_hash',
+        'full_status',  # TODO: remove
+        'blobs_remaining',
+        'blobs_in_stream',
+        'uploading_to_reflector',
+        'is_fully_reflected'
+    })
+
     def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
                  wallet_manager: 'WalletManager', storage: 'SQLiteStorage', node: Optional['Node'],
                  analytics_manager: Optional['AnalyticsManager'] = None):
-        self.loop = loop
-        self.config = config
+        super().__init__(loop, config, storage, analytics_manager)
         self.blob_manager = blob_manager
         self.wallet_manager = wallet_manager
-        self.storage = storage
         self.node = node
-        self.analytics_manager = analytics_manager
-        self.streams: typing.Dict[str, ManagedStream] = {}
         self.resume_saving_task: Optional[asyncio.Task] = None
         self.re_reflect_task: Optional[asyncio.Task] = None
         self.update_stream_finished_futs: typing.List[asyncio.Future] = []
         self.running_reflector_uploads: typing.Dict[str, asyncio.Task] = {}
-        self.started = asyncio.Event(loop=self.loop)
+        self.started = asyncio.Event()
+
+    @property
+    def streams(self):
+        return self._sources
+
+    def add(self, source: ManagedStream):
+        super().add(source)
+        self.storage.content_claim_callbacks[source.stream_hash] = lambda: self._update_content_claim(source)
 
     async def _update_content_claim(self, stream: ManagedStream):
         claim_info = await self.storage.get_content_claim(stream.stream_hash)
-        self.streams.setdefault(stream.sd_hash, stream).set_claim(claim_info, claim_info['value'])
+        self._sources.setdefault(stream.sd_hash, stream).set_claim(claim_info, claim_info['value'])
 
     async def recover_streams(self, file_infos: typing.List[typing.Dict]):
         to_restore = []
+        to_check = []
 
         async def recover_stream(sd_hash: str, stream_hash: str, stream_name: str,
                                  suggested_file_name: str, key: str,
@@ -98,6 +83,7 @@ class StreamManager:
             if not descriptor:
                 return
             to_restore.append((descriptor, sd_blob, content_fee))
+            to_check.extend([sd_blob.blob_hash] + [blob.blob_hash for blob in descriptor.blobs[:-1]])
 
         await asyncio.gather(*[
             recover_stream(
@@ -109,14 +95,16 @@ class StreamManager:
 
         if to_restore:
             await self.storage.recover_streams(to_restore, self.config.download_dir)
+        if to_check:
+            await self.blob_manager.ensure_completed_blobs_status(to_check)
 
         # if self.blob_manager._save_blobs:
         #     log.info("Recovered %i/%i attempted streams", len(to_restore), len(file_infos))
 
-    async def add_stream(self, rowid: int, sd_hash: str, file_name: Optional[str],
-                         download_directory: Optional[str], status: str,
-                         claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
-                         added_on: Optional[int]):
+    async def _load_stream(self, rowid: int, sd_hash: str, file_name: Optional[str],
+                           download_directory: Optional[str], status: str,
+                           claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
+                           added_on: Optional[int], fully_reflected: Optional[bool]):
         try:
             descriptor = await self.blob_manager.get_stream_descriptor(sd_hash)
         except InvalidStreamDescriptorError as err:
@@ -127,10 +115,11 @@ class StreamManager:
             claim, content_fee=content_fee, rowid=rowid, descriptor=descriptor,
             analytics_manager=self.analytics_manager, added_on=added_on
         )
-        self.streams[sd_hash] = stream
-        self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
+        if fully_reflected:
+            stream.fully_reflected.set()
+        self.add(stream)
 
-    async def load_and_resume_streams_from_database(self):
+    async def initialize_from_database(self):
         to_recover = []
         to_start = []
 
@@ -144,7 +133,6 @@ class StreamManager:
                 to_recover.append(file_info)
             to_start.append(file_info)
         if to_recover:
-            log.info("Recover %i files", len(to_recover))
             await self.recover_streams(to_recover)
 
         log.info("Initializing %i files", len(to_start))
@@ -155,33 +143,37 @@ class StreamManager:
             download_directory = path_or_none(file_info['download_directory'])
             if file_name and download_directory and not file_info['saved_file'] and file_info['status'] == 'running':
                 to_resume_saving.append((file_name, download_directory, file_info['sd_hash']))
-            add_stream_tasks.append(self.loop.create_task(self.add_stream(
+            add_stream_tasks.append(self.loop.create_task(self._load_stream(
                 file_info['rowid'], file_info['sd_hash'], file_name,
                 download_directory, file_info['status'],
                 file_info['claim'], file_info['content_fee'],
-                file_info['added_on']
+                file_info['added_on'], file_info['fully_reflected']
             )))
         if add_stream_tasks:
-            await asyncio.gather(*add_stream_tasks, loop=self.loop)
-        log.info("Started stream manager with %i files", len(self.streams))
+            await asyncio.gather(*add_stream_tasks)
+        log.info("Started stream manager with %i files", len(self._sources))
         if not self.node:
             log.info("no DHT node given, resuming downloads trusting that we can contact reflector")
         if to_resume_saving:
-            self.resume_saving_task = self.loop.create_task(self.resume(to_resume_saving))
-
-    async def resume(self, to_resume_saving):
-        log.info("Resuming saving %i files", len(to_resume_saving))
-        await asyncio.gather(
-            *(self.streams[sd_hash].save_file(file_name, download_directory, node=self.node)
-              for (file_name, download_directory, sd_hash) in to_resume_saving),
-            loop=self.loop
-        )
+            log.info("Resuming saving %i files", len(to_resume_saving))
+            self.resume_saving_task = asyncio.ensure_future(asyncio.gather(
+                *(self._sources[sd_hash].save_file(file_name, download_directory)
+                  for (file_name, download_directory, sd_hash) in to_resume_saving),
+            ))
 
     async def reflect_streams(self):
+        try:
+            return await self._reflect_streams()
+        except Exception:
+            log.exception("reflector task encountered an unexpected error!")
+
+    async def _reflect_streams(self):
+        # todo: those debug statements are temporary for #2987 - remove them if its closed
         while True:
             if self.config.reflect_streams and self.config.reflector_servers:
+                log.debug("collecting streams to reflect")
                 sd_hashes = await self.storage.get_streams_to_re_reflect()
-                sd_hashes = [sd for sd in sd_hashes if sd in self.streams]
+                sd_hashes = [sd for sd in sd_hashes if sd in self._sources]
                 batch = []
                 while sd_hashes:
                     stream = self.streams[sd_hashes.pop()]
@@ -190,25 +182,26 @@ class StreamManager:
                             stream.fully_reflected.is_set():
                         batch.append(self.reflect_stream(stream))
                     if len(batch) >= self.config.concurrent_reflector_uploads:
-                        await asyncio.gather(*batch, loop=self.loop)
+                        log.debug("waiting for batch of %s reflecting streams", len(batch))
+                        await asyncio.gather(*batch)
+                        log.debug("done processing %s streams", len(batch))
                         batch = []
                 if batch:
-                    await asyncio.gather(*batch, loop=self.loop)
-            await asyncio.sleep(300, loop=self.loop)
+                    log.debug("waiting for batch of %s reflecting streams", len(batch))
+                    await asyncio.gather(*batch)
+                    log.debug("done processing %s streams", len(batch))
+            await asyncio.sleep(300)
 
     async def start(self):
-        await self.load_and_resume_streams_from_database()
+        await super().start()
         self.re_reflect_task = self.loop.create_task(self.reflect_streams())
-        self.started.set()
 
-    def stop(self):
+    async def stop(self):
+        await super().stop()
         if self.resume_saving_task and not self.resume_saving_task.done():
             self.resume_saving_task.cancel()
         if self.re_reflect_task and not self.re_reflect_task.done():
             self.re_reflect_task.cancel()
-        while self.streams:
-            _, stream = self.streams.popitem()
-            stream.stop_tasks()
         while self.update_stream_finished_futs:
             self.update_stream_finished_futs.pop().cancel()
         while self.running_reflector_uploads:
@@ -223,7 +216,7 @@ class StreamManager:
             server, port = random.choice(self.config.reflector_servers)
         if stream.sd_hash in self.running_reflector_uploads:
             return self.running_reflector_uploads[stream.sd_hash]
-        task = self.loop.create_task(stream.upload_to_reflector(server, port))
+        task = self.loop.create_task(self._retriable_reflect_stream(stream, server, port))
         self.running_reflector_uploads[stream.sd_hash] = task
         task.add_done_callback(
             lambda _: None if stream.sd_hash not in self.running_reflector_uploads else
@@ -231,261 +224,53 @@ class StreamManager:
         )
         return task
 
-    async def create_stream(self, file_path: str, key: Optional[bytes] = None,
-                            iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
-        stream = await ManagedStream.create(self.loop, self.config, self.blob_manager, file_path, key, iv_generator)
+    @staticmethod
+    async def _retriable_reflect_stream(stream, host, port):
+        sent = await stream.upload_to_reflector(host, port)
+        while not stream.is_fully_reflected and stream.reflector_progress > 0 and len(sent) > 0:
+            stream.reflector_progress = 0
+            sent = await stream.upload_to_reflector(host, port)
+        return sent
+
+    async def create(self, file_path: str, key: Optional[bytes] = None,
+                     iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
+        descriptor = await StreamDescriptor.create_stream(
+            self.loop, self.blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
+            blob_completed_callback=self.blob_manager.blob_completed
+        )
+        await self.storage.store_stream(
+            self.blob_manager.get_blob(descriptor.sd_hash, is_mine=True), descriptor
+        )
+        row_id = await self.storage.save_published_file(
+            descriptor.stream_hash, os.path.basename(file_path), os.path.dirname(file_path), 0
+        )
+        stream = ManagedStream(
+            self.loop, self.config, self.blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
+            os.path.basename(file_path), status=ManagedDownloadSource.STATUS_FINISHED,
+            rowid=row_id, descriptor=descriptor
+        )
         self.streams[stream.sd_hash] = stream
         self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
         if self.config.reflect_streams and self.config.reflector_servers:
             self.reflect_stream(stream)
         return stream
 
-    async def delete_stream(self, stream: ManagedStream, delete_file: Optional[bool] = False):
-        if stream.sd_hash in self.running_reflector_uploads:
-            self.running_reflector_uploads[stream.sd_hash].cancel()
-        stream.stop_tasks()
-        if stream.sd_hash in self.streams:
-            del self.streams[stream.sd_hash]
-        blob_hashes = [stream.sd_hash] + [b.blob_hash for b in stream.descriptor.blobs[:-1]]
+    async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
+        if not isinstance(source, ManagedStream):
+            return
+        if source.identifier in self.running_reflector_uploads:
+            self.running_reflector_uploads[source.identifier].cancel()
+        await source.stop_tasks()
+        if source.identifier in self.streams:
+            del self.streams[source.identifier]
+        blob_hashes = [source.identifier] + [b.blob_hash for b in source.descriptor.blobs[:-1]]
         await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False)
-        await self.storage.delete_stream(stream.descriptor)
-        if delete_file and stream.output_file_exists:
-            os.remove(stream.full_path)
-
-    def get_stream_by_stream_hash(self, stream_hash: str) -> Optional[ManagedStream]:
-        streams = tuple(filter(lambda stream: stream.stream_hash == stream_hash, self.streams.values()))
-        if streams:
-            return streams[0]
-
-    def get_filtered_streams(self, sort_by: Optional[str] = None, reverse: Optional[bool] = False,
-                             comparison: Optional[str] = None,
-                             **search_by) -> typing.List[ManagedStream]:
-        """
-        Get a list of filtered and sorted ManagedStream objects
-
-        :param sort_by: field to sort by
-        :param reverse: reverse sorting
-        :param comparison: comparison operator used for filtering
-        :param search_by: fields and values to filter by
-        """
-        if sort_by and sort_by not in FILTER_FIELDS:
-            raise ValueError(f"'{sort_by}' is not a valid field to sort by")
-        if comparison and comparison not in COMPARISON_OPERATORS:
-            raise ValueError(f"'{comparison}' is not a valid comparison")
-        if 'full_status' in search_by:
-            del search_by['full_status']
-        for search in search_by:
-            if search not in FILTER_FIELDS:
-                raise ValueError(f"'{search}' is not a valid search operation")
-        if search_by:
-            comparison = comparison or 'eq'
-            streams = []
-            for stream in self.streams.values():
-                for search, val in search_by.items():
-                    if COMPARISON_OPERATORS[comparison](getattr(stream, search), val):
-                        streams.append(stream)
-                        break
-        else:
-            streams = list(self.streams.values())
-        if sort_by:
-            streams.sort(key=lambda s: getattr(s, sort_by))
-            if reverse:
-                streams.reverse()
-        return streams
-
-    async def _check_update_or_replace(self, outpoint: str, claim_id: str, claim: Claim
-                                       ) -> typing.Tuple[Optional[ManagedStream], Optional[ManagedStream]]:
-        existing = self.get_filtered_streams(outpoint=outpoint)
-        if existing:
-            return existing[0], None
-        existing = self.get_filtered_streams(sd_hash=claim.stream.source.sd_hash)
-        if existing and existing[0].claim_id != claim_id:
-            raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {claim_id}")
-        if existing:
-            log.info("claim contains a metadata only update to a stream we have")
-            await self.storage.save_content_claim(
-                existing[0].stream_hash, outpoint
-            )
-            await self._update_content_claim(existing[0])
-            return existing[0], None
-        else:
-            existing_for_claim_id = self.get_filtered_streams(claim_id=claim_id)
-            if existing_for_claim_id:
-                log.info("claim contains an update to a stream we have, downloading it")
-                return None, existing_for_claim_id[0]
-        return None, None
-
-    @staticmethod
-    def _convert_to_old_resolve_output(wallet_manager, resolves):
-        result = {}
-        for url, txo in resolves.items():
-            if isinstance(txo, Output):
-                tx_height = txo.tx_ref.height
-                best_height = wallet_manager.ledger.headers.height
-                result[url] = {
-                    'name': txo.claim_name,
-                    'value': txo.claim,
-                    'protobuf': binascii.hexlify(txo.claim.to_bytes()),
-                    'claim_id': txo.claim_id,
-                    'txid': txo.tx_ref.id,
-                    'nout': txo.position,
-                    'amount': dewies_to_lbc(txo.amount),
-                    'effective_amount': txo.meta.get('effective_amount', 0),
-                    'height': tx_height,
-                    'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
-                    'claim_sequence': -1,
-                    'address': txo.get_address(wallet_manager.ledger),
-                    'valid_at_height': txo.meta.get('activation_height', None),
-                    'timestamp': wallet_manager.ledger.headers[tx_height]['timestamp'],
-                    'supports': []
-                }
-            else:
-                result[url] = txo
-        return result
-
-    @cache_concurrent
-    async def download_stream_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
-                                       timeout: Optional[float] = None,
-                                       file_name: Optional[str] = None,
-                                       download_directory: Optional[str] = None,
-                                       save_file: Optional[bool] = None,
-                                       resolve_timeout: float = 3.0,
-                                       wallet: Optional['Wallet'] = None) -> ManagedStream:
-        manager = self.wallet_manager
-        wallet = wallet or manager.default_wallet
-        timeout = timeout or self.config.download_timeout
-        start_time = self.loop.time()
-        resolved_time = None
-        stream = None
-        txo: Optional[Output] = None
-        error = None
-        outpoint = None
-        if save_file is None:
-            save_file = self.config.save_files
-        if file_name and not save_file:
-            save_file = True
-        if save_file:
-            download_directory = download_directory or self.config.download_dir
-        else:
-            download_directory = None
-
-        payment = None
-        try:
-            # resolve the claim
-            if not URL.parse(uri).has_stream:
-                raise ResolveError("cannot download a channel claim, specify a /path")
-            try:
-                response = await asyncio.wait_for(
-                    manager.ledger.resolve(wallet.accounts, [uri]),
-                    resolve_timeout
-                )
-                resolved_result = self._convert_to_old_resolve_output(manager, response)
-            except asyncio.TimeoutError:
-                raise ResolveTimeoutError(uri)
-            except Exception as err:
-                if isinstance(err, asyncio.CancelledError):  # TODO: remove when updated to 3.8
-                    raise
-                log.exception("Unexpected error resolving stream:")
-                raise ResolveError(f"Unexpected error resolving stream: {str(err)}")
-            await self.storage.save_claims_for_resolve([
-                value for value in resolved_result.values() if 'error' not in value
-            ])
-            resolved = resolved_result.get(uri, {})
-            resolved = resolved if 'value' in resolved else resolved.get('claim')
-            if not resolved:
-                raise ResolveError(f"Failed to resolve stream at '{uri}'")
-            if 'error' in resolved:
-                raise ResolveError(f"error resolving stream: {resolved['error']}")
-            txo = response[uri]
-
-            claim = Claim.from_bytes(binascii.unhexlify(resolved['protobuf']))
-            outpoint = f"{resolved['txid']}:{resolved['nout']}"
-            resolved_time = self.loop.time() - start_time
-
-            # resume or update an existing stream, if the stream changed: download it and delete the old one after
-            updated_stream, to_replace = await self._check_update_or_replace(outpoint, resolved['claim_id'], claim)
-            if updated_stream:
-                log.info("already have stream for %s", uri)
-                if save_file and updated_stream.output_file_exists:
-                    save_file = False
-                await updated_stream.start(node=self.node, timeout=timeout, save_now=save_file)
-                if not updated_stream.output_file_exists and (save_file or file_name or download_directory):
-                    await updated_stream.save_file(
-                        file_name=file_name, download_directory=download_directory, node=self.node
-                    )
-                return updated_stream
-
-            if not to_replace and txo.has_price and not txo.purchase_receipt:
-                payment = await manager.create_purchase_transaction(
-                    wallet.accounts, txo, exchange_rate_manager
-                )
-
-            stream = ManagedStream(
-                self.loop, self.config, self.blob_manager, claim.stream.source.sd_hash, download_directory,
-                file_name, ManagedStream.STATUS_RUNNING, content_fee=payment,
-                analytics_manager=self.analytics_manager
-            )
-            log.info("starting download for %s", uri)
-
-            before_download = self.loop.time()
-            await stream.start(self.node, timeout)
-            stream.set_claim(resolved, claim)
-            if to_replace:  # delete old stream now that the replacement has started downloading
-                await self.delete_stream(to_replace)
-
-            if payment is not None:
-                await manager.broadcast_or_release(payment)
-                payment = None  # to avoid releasing in `finally` later
-                log.info("paid fee of %s for %s", dewies_to_lbc(stream.content_fee.outputs[0].amount), uri)
-                await self.storage.save_content_fee(stream.stream_hash, stream.content_fee)
-
-            self.streams[stream.sd_hash] = stream
-            self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
-            await self.storage.save_content_claim(stream.stream_hash, outpoint)
-            if save_file:
-                await asyncio.wait_for(stream.save_file(node=self.node), timeout - (self.loop.time() - before_download),
-                                       loop=self.loop)
-            return stream
-        except asyncio.TimeoutError:
-            error = DownloadDataTimeoutError(stream.sd_hash)
-            raise error
-        except Exception as err:  # forgive data timeout, don't delete stream
-            expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError,
-                        KeyFeeAboveMaxAllowedError)
-            if isinstance(err, expected):
-                log.warning("Failed to download %s: %s", uri, str(err))
-            elif isinstance(err, asyncio.CancelledError):
-                pass
-            else:
-                log.exception("Unexpected error downloading stream:")
-            error = err
-            raise
-        finally:
-            if payment is not None:
-                # payment is set to None after broadcasting, if we're here an exception probably happened
-                await manager.ledger.release_tx(payment)
-            if self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or
-                                                                 stream.downloader.time_to_first_bytes))):
-                server = self.wallet_manager.ledger.network.client.server
-                self.loop.create_task(
-                    self.analytics_manager.send_time_to_first_bytes(
-                        resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id,
-                        uri, outpoint,
-                        None if not stream else len(stream.downloader.blob_downloader.active_connections),
-                        None if not stream else len(stream.downloader.blob_downloader.scores),
-                        None if not stream else len(stream.downloader.blob_downloader.connection_failures),
-                        False if not stream else stream.downloader.added_fixed_peers,
-                        self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay,
-                        None if not stream else stream.sd_hash,
-                        None if not stream else stream.downloader.time_to_descriptor,
-                        None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
-                        None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
-                        None if not stream else stream.downloader.time_to_first_bytes,
-                        None if not error else error.__class__.__name__,
-                        None if not error else str(error),
-                        None if not server else f"{server[0]}:{server[1]}"
-                    )
-                )
+        await self.storage.delete_stream(source.descriptor)
+        if delete_file and source.output_file_exists:
+            os.remove(source.full_path)
 
     async def stream_partial_content(self, request: Request, sd_hash: str):
-        return await self.streams[sd_hash].stream_file(request, self.node)
+        stream = self._sources[sd_hash]
+        if not stream.downloader.node:
+            stream.downloader.node = self.node
+        return await stream.stream_file(request)

lbry/testcase.py

@@ -17,18 +17,21 @@ from functools import partial
 from lbry.wallet import WalletManager, Wallet, Ledger, Account, Transaction
 from lbry.conf import Config
 from lbry.wallet.util import satoshis_to_coins
+from lbry.wallet.dewies import lbc_to_dewies
 from lbry.wallet.orchstr8 import Conductor
-from lbry.wallet.orchstr8.node import BlockchainNode, WalletNode
+from lbry.wallet.orchstr8.node import LBCWalletNode, WalletNode
+from lbry.schema.claim import Claim
 
 from lbry.extras.daemon.daemon import Daemon, jsonrpc_dumps_pretty
 from lbry.extras.daemon.components import Component, WalletComponent
 from lbry.extras.daemon.components import (
-    DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
-    UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
+    DHT_COMPONENT,
+    HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
+    UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, LIBTORRENT_COMPONENT
 )
 from lbry.extras.daemon.componentmanager import ComponentManager
 from lbry.extras.daemon.exchange_rate_manager import (
-    ExchangeRateManager, ExchangeRate, LBRYFeed, LBRYBTCFeed
+    ExchangeRateManager, ExchangeRate, BittrexBTCFeed, BittrexUSDFeed
 )
 from lbry.extras.daemon.storage import SQLiteStorage
 from lbry.blob.blob_manager import BlobManager
@@ -83,6 +86,7 @@ class AsyncioTestCase(unittest.TestCase):
     #  https://bugs.python.org/issue32972
 
     LOOP_SLOW_CALLBACK_DURATION = 0.2
+    TIMEOUT = 120.0
 
     maxDiff = None
 
@@ -130,15 +134,18 @@ class AsyncioTestCase(unittest.TestCase):
 
             with outcome.testPartExecutor(self):
                 self.setUp()
+                self.add_timeout()
                 self.loop.run_until_complete(self.asyncSetUp())
             if outcome.success:
                 outcome.expecting_failure = expecting_failure
                 with outcome.testPartExecutor(self, isTest=True):
                     maybe_coroutine = testMethod()
                     if asyncio.iscoroutine(maybe_coroutine):
+                        self.add_timeout()
                         self.loop.run_until_complete(maybe_coroutine)
                 outcome.expecting_failure = False
                 with outcome.testPartExecutor(self):
+                    self.add_timeout()
                     self.loop.run_until_complete(self.asyncTearDown())
                     self.tearDown()
 
@@ -186,8 +193,25 @@ class AsyncioTestCase(unittest.TestCase):
             with outcome.testPartExecutor(self):
                 maybe_coroutine = function(*args, **kwargs)
                 if asyncio.iscoroutine(maybe_coroutine):
+                    self.add_timeout()
                     self.loop.run_until_complete(maybe_coroutine)
 
+    def cancel(self):
+        for task in asyncio.all_tasks(self.loop):
+            if not task.done():
+                task.print_stack()
+                task.cancel()
+
+    def add_timeout(self):
+        if self.TIMEOUT:
+            self.loop.call_later(self.TIMEOUT, self.check_timeout, time())
+
+    def check_timeout(self, started):
+        if time() - started >= self.TIMEOUT:
+            self.cancel()
+        else:
+            self.loop.call_later(self.TIMEOUT, self.check_timeout, started)
+
 
 class AdvanceTimeTestCase(AsyncioTestCase):
 
@@ -212,7 +236,7 @@ class IntegrationTestCase(AsyncioTestCase):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.conductor: Optional[Conductor] = None
-        self.blockchain: Optional[BlockchainNode] = None
+        self.blockchain: Optional[LBCWalletNode] = None
         self.wallet_node: Optional[WalletNode] = None
         self.manager: Optional[WalletManager] = None
         self.ledger: Optional[Ledger] = None
@@ -221,13 +245,15 @@ class IntegrationTestCase(AsyncioTestCase):
 
     async def asyncSetUp(self):
         self.conductor = Conductor(seed=self.SEED)
-        await self.conductor.start_blockchain()
-        self.addCleanup(self.conductor.stop_blockchain)
+        await self.conductor.start_lbcd()
+        self.addCleanup(self.conductor.stop_lbcd)
+        await self.conductor.start_lbcwallet()
+        self.addCleanup(self.conductor.stop_lbcwallet)
         await self.conductor.start_spv()
         self.addCleanup(self.conductor.stop_spv)
         await self.conductor.start_wallet()
         self.addCleanup(self.conductor.stop_wallet)
-        self.blockchain = self.conductor.blockchain_node
+        self.blockchain = self.conductor.lbcwallet_node
         self.wallet_node = self.conductor.wallet_node
         self.manager = self.wallet_node.manager
         self.ledger = self.wallet_node.ledger
@@ -241,6 +267,13 @@ class IntegrationTestCase(AsyncioTestCase):
     def broadcast(self, tx):
         return self.ledger.broadcast(tx)
 
+    async def broadcast_and_confirm(self, tx, ledger=None):
+        ledger = ledger or self.ledger
+        notifications = asyncio.create_task(ledger.wait(tx))
+        await ledger.broadcast(tx)
+        await notifications
+        await self.generate_and_wait(1, [tx.id], ledger)
+
     async def on_header(self, height):
         if self.ledger.headers.height < height:
             await self.ledger.on_header.where(
@@ -248,9 +281,32 @@ class IntegrationTestCase(AsyncioTestCase):
             )
         return True
 
-    def on_transaction_id(self, txid, ledger=None):
-        return (ledger or self.ledger).on_transaction.where(
-            lambda e: e.tx.id == txid
+    async def send_to_address_and_wait(self, address, amount, blocks_to_generate=0, ledger=None):
+        tx_watch = []
+        txid = None
+        done = False
+        watcher = (ledger or self.ledger).on_transaction.where(
+            lambda e: e.tx.id == txid or done or tx_watch.append(e.tx.id)
+        )
+
+        txid = await self.blockchain.send_to_address(address, amount)
+        done = txid in tx_watch
+        await watcher
+
+        await self.generate_and_wait(blocks_to_generate, [txid], ledger)
+        return txid
+
+    async def generate_and_wait(self, blocks_to_generate, txids, ledger=None):
+        if blocks_to_generate > 0:
+            watcher = (ledger or self.ledger).on_transaction.where(
+                lambda e: ((e.tx.id in txids and txids.remove(e.tx.id)), len(txids) <= 0)[-1]  # multi-statement lambda
+            )
+            await self.generate(blocks_to_generate)
+            await watcher
+
+    def on_address_update(self, address):
+        return self.ledger.on_transaction.where(
+            lambda e: e.address == address
         )
 
     def on_transaction_address(self, tx, address):
@@ -258,6 +314,22 @@ class IntegrationTestCase(AsyncioTestCase):
             lambda e: e.tx.id == tx.id and e.address == address
         )
 
+    async def generate(self, blocks):
+        """ Ask lbrycrd to generate some blocks and wait until ledger has them. """
+        prepare = self.ledger.on_header.where(self.blockchain.is_expected_block)
+        self.conductor.spv_node.server.synchronized.clear()
+        await self.blockchain.generate(blocks)
+        height = self.blockchain.block_expected
+        await prepare  # no guarantee that it didn't happen already, so start waiting from before calling generate
+        while True:
+            await self.conductor.spv_node.server.synchronized.wait()
+            self.conductor.spv_node.server.synchronized.clear()
+            if self.conductor.spv_node.server.db.db_height < height:
+                continue
+            if self.conductor.spv_node.server._es_height < height:
+                continue
+            break
+
 
 class FakeExchangeRateManager(ExchangeRateManager):
 
@@ -276,8 +348,8 @@ class FakeExchangeRateManager(ExchangeRateManager):
 
 def get_fake_exchange_rate_manager(rates=None):
     return FakeExchangeRateManager(
-        [LBRYFeed(), LBRYBTCFeed()],
-        rates or {'BTCLBC': 3.0, 'USDBTC': 2.0}
+        [BittrexBTCFeed(), BittrexUSDFeed()],
+        rates or {'BTCLBC': 3.0, 'USDLBC': 2.0}
     )
 
 
@@ -315,26 +387,32 @@ class CommandTestCase(IntegrationTestCase):
         self.server_blob_manager = None
         self.server = None
         self.reflector = None
+        self.skip_libtorrent = True
 
     async def asyncSetUp(self):
-        await super().asyncSetUp()
 
         logging.getLogger('lbry.blob_exchange').setLevel(self.VERBOSITY)
         logging.getLogger('lbry.daemon').setLevel(self.VERBOSITY)
         logging.getLogger('lbry.stream').setLevel(self.VERBOSITY)
         logging.getLogger('lbry.wallet').setLevel(self.VERBOSITY)
 
+        await super().asyncSetUp()
+
         self.daemon = await self.add_daemon(self.wallet_node)
 
         await self.account.ensure_address_gap()
         address = (await self.account.receiving.get_addresses(limit=1, only_usable=True))[0]
-        sendtxid = await self.blockchain.send_to_address(address, 10)
-        await self.confirm_tx(sendtxid)
-        await self.generate(5)
+        await self.send_to_address_and_wait(address, 10, 6)
 
         server_tmp_dir = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, server_tmp_dir)
-        self.server_config = Config()
+        self.server_config = Config(
+            data_dir=server_tmp_dir,
+            wallet_dir=server_tmp_dir,
+            save_files=True,
+            download_dir=server_tmp_dir
+        )
+        self.server_config.transaction_cache_size = 10000
         self.server_storage = SQLiteStorage(self.server_config, ':memory:')
         await self.server_storage.open()
 
@@ -357,6 +435,7 @@ class CommandTestCase(IntegrationTestCase):
             await daemon.stop()
 
     async def add_daemon(self, wallet_node=None, seed=None):
+        start_wallet_node = False
         if wallet_node is None:
             wallet_node = WalletNode(
                 self.wallet_node.manager_class,
@@ -364,30 +443,42 @@ class CommandTestCase(IntegrationTestCase):
                 port=self.extra_wallet_node_port
             )
             self.extra_wallet_node_port += 1
-            await wallet_node.start(self.conductor.spv_node, seed=seed)
-            self.extra_wallet_nodes.append(wallet_node)
+            start_wallet_node = True
 
         upload_dir = os.path.join(wallet_node.data_path, 'uploads')
         os.mkdir(upload_dir)
 
-        conf = Config()
-        conf.data_dir = wallet_node.data_path
-        conf.wallet_dir = wallet_node.data_path
-        conf.download_dir = wallet_node.data_path
+        conf = Config(
+            # needed during instantiation to access known_hubs path
+            data_dir=wallet_node.data_path,
+            wallet_dir=wallet_node.data_path,
+            save_files=True,
+            download_dir=wallet_node.data_path
+        )
         conf.upload_dir = upload_dir  # not a real conf setting
         conf.share_usage_data = False
         conf.use_upnp = False
         conf.reflect_streams = True
         conf.blockchain_name = 'lbrycrd_regtest'
-        conf.lbryum_servers = [('127.0.0.1', 50001)]
+        conf.lbryum_servers = [(self.conductor.spv_node.hostname, self.conductor.spv_node.port)]
         conf.reflector_servers = [('127.0.0.1', 5566)]
+        conf.fixed_peers = [('127.0.0.1', 5567)]
         conf.known_dht_nodes = []
         conf.blob_lru_cache_size = self.blob_lru_cache_size
+        conf.transaction_cache_size = 10000
         conf.components_to_skip = [
             DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
             PEER_PROTOCOL_SERVER_COMPONENT
         ]
-        wallet_node.manager.config = conf
+        if self.skip_libtorrent:
+            conf.components_to_skip.append(LIBTORRENT_COMPONENT)
+
+        if start_wallet_node:
+            await wallet_node.start(self.conductor.spv_node, seed=seed, config=conf)
+            self.extra_wallet_nodes.append(wallet_node)
+        else:
+            wallet_node.manager.config = conf
+            wallet_node.manager.ledger.config['known_hubs'] = conf.known_hubs
 
         def wallet_maker(component_manager):
             wallet_component = WalletComponent(component_manager)
@@ -398,7 +489,7 @@ class CommandTestCase(IntegrationTestCase):
         daemon = Daemon(conf, ComponentManager(
             conf, skip_components=conf.components_to_skip, wallet=wallet_maker,
             exchange_rate_manager=partial(ExchangeRateManagerComponent, rates={
-                'BTCLBC': 1.0, 'USDBTC': 2.0
+                'BTCLBC': 1.0, 'USDLBC': 2.0
             })
         ))
         await daemon.initialize()
@@ -408,9 +499,14 @@ class CommandTestCase(IntegrationTestCase):
 
     async def confirm_tx(self, txid, ledger=None):
         """ Wait for tx to be in mempool, then generate a block, wait for tx to be in a block. """
-        await self.on_transaction_id(txid, ledger)
-        await self.generate(1)
-        await self.on_transaction_id(txid, ledger)
+        # await (ledger or self.ledger).on_transaction.where(lambda e: e.tx.id == txid)
+        on_tx = (ledger or self.ledger).on_transaction.where(lambda e: e.tx.id == txid)
+        await asyncio.wait([self.generate(1), on_tx], timeout=5)
+
+        # # actually, if it's in the mempool or in the block we're fine
+        # await self.generate_and_wait(1, [txid], ledger=ledger)
+        # return txid
+
         return txid
 
     async def on_transaction_dict(self, tx):
@@ -425,11 +521,6 @@ class CommandTestCase(IntegrationTestCase):
             addresses.add(txo['address'])
         return list(addresses)
 
-    async def generate(self, blocks):
-        """ Ask lbrycrd to generate some blocks and wait until ledger has them. """
-        await self.blockchain.generate(blocks)
-        await self.ledger.on_header.where(self.blockchain.is_expected_block)
-
     async def blockchain_claim_name(self, name: str, value: str, amount: str, confirm=True):
         txid = await self.blockchain._cli_cmnd('claimname', name, value, amount)
         if confirm:
@@ -450,12 +541,27 @@ class CommandTestCase(IntegrationTestCase):
         """ Synchronous version of `out` method. """
         return json.loads(jsonrpc_dumps_pretty(value, ledger=self.ledger))['result']
 
-    async def confirm_and_render(self, awaitable, confirm) -> Transaction:
+    async def confirm_and_render(self, awaitable, confirm, return_tx=False) -> Transaction:
         tx = await awaitable
         if confirm:
             await self.ledger.wait(tx)
             await self.generate(1)
             await self.ledger.wait(tx, self.blockchain.block_expected)
+        if not return_tx:
+            return self.sout(tx)
+        return tx
+
+    async def create_nondeterministic_channel(self, name, price, pubkey_bytes, daemon=None, blocking=False):
+        account = (daemon or self.daemon).wallet_manager.default_account
+        claim_address = await account.receiving.get_or_create_usable_address()
+        claim = Claim()
+        claim.channel.public_key_bytes = pubkey_bytes
+        tx = await Transaction.claim_create(
+            name, claim, lbc_to_dewies(price),
+            claim_address, [self.account], self.account
+        )
+        await tx.sign([self.account])
+        await (daemon or self.daemon).broadcast_or_release(tx, blocking)
         return self.sout(tx)
 
     def create_upload_file(self, data, prefix=None, suffix=None):
@@ -467,26 +573,26 @@ class CommandTestCase(IntegrationTestCase):
 
     async def stream_create(
             self, name='hovercraft', bid='1.0', file_path=None,
-            data=b'hi!', confirm=True, prefix=None, suffix=None, **kwargs):
-        if file_path is None:
+            data=b'hi!', confirm=True, prefix=None, suffix=None, return_tx=False, **kwargs):
+        if file_path is None and data is not None:
             file_path = self.create_upload_file(data=data, prefix=prefix, suffix=suffix)
         return await self.confirm_and_render(
-            self.daemon.jsonrpc_stream_create(name, bid, file_path=file_path, **kwargs), confirm
+            self.daemon.jsonrpc_stream_create(name, bid, file_path=file_path, **kwargs), confirm, return_tx
         )
 
     async def stream_update(
-            self, claim_id, data=None, prefix=None, suffix=None, confirm=True, **kwargs):
+            self, claim_id, data=None, prefix=None, suffix=None, confirm=True, return_tx=False, **kwargs):
         if data is not None:
             file_path = self.create_upload_file(data=data, prefix=prefix, suffix=suffix)
             return await self.confirm_and_render(
-                self.daemon.jsonrpc_stream_update(claim_id, file_path=file_path, **kwargs), confirm
+                self.daemon.jsonrpc_stream_update(claim_id, file_path=file_path, **kwargs), confirm, return_tx
             )
         return await self.confirm_and_render(
             self.daemon.jsonrpc_stream_update(claim_id, **kwargs), confirm
         )
 
-    def stream_repost(self, claim_id, name='repost', bid='1.0', confirm=True, **kwargs):
-        return self.confirm_and_render(
+    async def stream_repost(self, claim_id, name='repost', bid='1.0', confirm=True, **kwargs):
+        return await self.confirm_and_render(
             self.daemon.jsonrpc_stream_repost(claim_id=claim_id, name=name, bid=bid, **kwargs), confirm
         )
 
@@ -497,6 +603,11 @@ class CommandTestCase(IntegrationTestCase):
             self.daemon.jsonrpc_stream_abandon(*args, **kwargs), confirm
         )
 
+    async def purchase_create(self, *args, confirm=True, **kwargs):
+        return await self.confirm_and_render(
+            self.daemon.jsonrpc_purchase_create(*args, **kwargs), confirm
+        )
+
     async def publish(self, name, *args, confirm=True, **kwargs):
         return await self.confirm_and_render(
             self.daemon.jsonrpc_publish(name, *args, **kwargs), confirm
@@ -543,15 +654,58 @@ class CommandTestCase(IntegrationTestCase):
             self.daemon.jsonrpc_support_create(claim_id, bid, **kwargs), confirm
         )
 
-    async def resolve(self, uri):
-        return await self.out(self.daemon.jsonrpc_resolve(uri))
+    async def support_abandon(self, *args, confirm=True, **kwargs):
+        if 'blocking' not in kwargs:
+            kwargs['blocking'] = False
+        return await self.confirm_and_render(
+            self.daemon.jsonrpc_support_abandon(*args, **kwargs), confirm
+        )
+
+    async def account_send(self, *args, confirm=True, **kwargs):
+        return await self.confirm_and_render(
+            self.daemon.jsonrpc_account_send(*args, **kwargs), confirm
+        )
+
+    async def wallet_send(self, *args, confirm=True, **kwargs):
+        return await self.confirm_and_render(
+            self.daemon.jsonrpc_wallet_send(*args, **kwargs), confirm
+        )
+
+    async def txo_spend(self, *args, confirm=True, **kwargs):
+        txs = await self.daemon.jsonrpc_txo_spend(*args, **kwargs)
+        if confirm:
+            await asyncio.wait([self.ledger.wait(tx) for tx in txs])
+            await self.generate(1)
+            await asyncio.wait([self.ledger.wait(tx, self.blockchain.block_expected) for tx in txs])
+        return self.sout(txs)
+
+    async def blob_clean(self):
+        return await self.out(self.daemon.jsonrpc_blob_clean())
+
+    async def status(self):
+        return await self.out(self.daemon.jsonrpc_status())
+
+    async def resolve(self, uri, **kwargs):
+        return (await self.out(self.daemon.jsonrpc_resolve(uri, **kwargs)))[uri]
 
     async def claim_search(self, **kwargs):
         return (await self.out(self.daemon.jsonrpc_claim_search(**kwargs)))['items']
 
+    async def get_claim_by_claim_id(self, claim_id):
+        return await self.out(self.ledger.get_claim_by_claim_id(claim_id))
+
     async def file_list(self, *args, **kwargs):
         return (await self.out(self.daemon.jsonrpc_file_list(*args, **kwargs)))['items']
 
+    async def txo_list(self, *args, **kwargs):
+        return (await self.out(self.daemon.jsonrpc_txo_list(*args, **kwargs)))['items']
+
+    async def txo_sum(self, *args, **kwargs):
+        return await self.out(self.daemon.jsonrpc_txo_sum(*args, **kwargs))
+
+    async def txo_plot(self, *args, **kwargs):
+        return await self.out(self.daemon.jsonrpc_txo_plot(*args, **kwargs))
+
     async def claim_list(self, *args, **kwargs):
         return (await self.out(self.daemon.jsonrpc_claim_list(*args, **kwargs)))['items']
 
@@ -561,9 +715,15 @@ class CommandTestCase(IntegrationTestCase):
     async def channel_list(self, *args, **kwargs):
         return (await self.out(self.daemon.jsonrpc_channel_list(*args, **kwargs)))['items']
 
+    async def transaction_list(self, *args, **kwargs):
+        return (await self.out(self.daemon.jsonrpc_transaction_list(*args, **kwargs)))['items']
+
+    async def blob_list(self, *args, **kwargs):
+        return (await self.out(self.daemon.jsonrpc_blob_list(*args, **kwargs)))['items']
+
     @staticmethod
     def get_claim_id(tx):
         return tx['outputs'][0]['claim_id']
 
     def assertItemCount(self, result, count):  # pylint: disable=invalid-name
-        self.assertEqual(result['total_items'], count)
+        self.assertEqual(count, result['total_items'])

lbry/torrent/session.py

@@ -0,0 +1,254 @@
+import asyncio
+import binascii
+import os
+import logging
+import random
+from hashlib import sha1
+from tempfile import mkdtemp
+from typing import Optional
+
+import libtorrent
+
+
+log = logging.getLogger(__name__)
+DEFAULT_FLAGS = (  # fixme: somehow the logic here is inverted?
+        libtorrent.add_torrent_params_flags_t.flag_auto_managed
+        | libtorrent.add_torrent_params_flags_t.flag_update_subscribe
+)
+
+
+class TorrentHandle:
+    def __init__(self, loop, executor, handle):
+        self._loop = loop
+        self._executor = executor
+        self._handle: libtorrent.torrent_handle = handle
+        self.started = asyncio.Event(loop=loop)
+        self.finished = asyncio.Event(loop=loop)
+        self.metadata_completed = asyncio.Event(loop=loop)
+        self.size = 0
+        self.total_wanted_done = 0
+        self.name = ''
+        self.tasks = []
+        self.torrent_file: Optional[libtorrent.file_storage] = None
+        self._base_path = None
+        self._handle.set_sequential_download(1)
+
+    @property
+    def largest_file(self) -> Optional[str]:
+        if not self.torrent_file:
+            return None
+        index = self.largest_file_index
+        return os.path.join(self._base_path, self.torrent_file.at(index).path)
+
+    @property
+    def largest_file_index(self):
+        largest_size, index = 0, 0
+        for file_num in range(self.torrent_file.num_files()):
+            if self.torrent_file.file_size(file_num) > largest_size:
+                largest_size = self.torrent_file.file_size(file_num)
+                index = file_num
+        return index
+
+    def stop_tasks(self):
+        while self.tasks:
+            self.tasks.pop().cancel()
+
+    def _show_status(self):
+        # fixme: cleanup
+        if not self._handle.is_valid():
+            return
+        status = self._handle.status()
+        if status.has_metadata:
+            self.size = status.total_wanted
+            self.total_wanted_done = status.total_wanted_done
+            self.name = status.name
+            if not self.metadata_completed.is_set():
+                self.metadata_completed.set()
+                log.info("Metadata completed for btih:%s - %s", status.info_hash, self.name)
+                self.torrent_file = self._handle.get_torrent_info().files()
+                self._base_path = status.save_path
+            first_piece = self.torrent_file.at(self.largest_file_index).offset
+            if not self.started.is_set():
+                if self._handle.have_piece(first_piece):
+                    self.started.set()
+                else:
+                    # prioritize it
+                    self._handle.set_piece_deadline(first_piece, 100)
+        if not status.is_seeding:
+            log.debug('%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d seeds: %d) %s - %s',
+                      status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000,
+                      status.num_peers, status.num_seeds, status.state, status.save_path)
+        elif not self.finished.is_set():
+            self.finished.set()
+            log.info("Torrent finished: %s", self.name)
+
+    async def status_loop(self):
+        while True:
+            self._show_status()
+            if self.finished.is_set():
+                break
+            await asyncio.sleep(0.1)
+
+    async def pause(self):
+        await self._loop.run_in_executor(
+            self._executor, self._handle.pause
+        )
+
+    async def resume(self):
+        await self._loop.run_in_executor(
+            self._executor, lambda: self._handle.resume()  # pylint: disable=unnecessary-lambda
+        )
+
+
+class TorrentSession:
+    def __init__(self, loop, executor):
+        self._loop = loop
+        self._executor = executor
+        self._session: Optional[libtorrent.session] = None
+        self._handles = {}
+        self.tasks = []
+        self.wait_start = True
+
+    async def add_fake_torrent(self):
+        tmpdir = mkdtemp()
+        info, btih = _create_fake_torrent(tmpdir)
+        flags = libtorrent.add_torrent_params_flags_t.flag_seed_mode
+        handle = self._session.add_torrent({
+            'ti': info, 'save_path': tmpdir, 'flags': flags
+        })
+        self._handles[btih] = TorrentHandle(self._loop, self._executor, handle)
+        return btih
+
+    async def bind(self, interface: str = '0.0.0.0', port: int = 10889):
+        settings = {
+            'listen_interfaces': f"{interface}:{port}",
+            'enable_natpmp': False,
+            'enable_upnp': False
+        }
+        self._session = await self._loop.run_in_executor(
+            self._executor, libtorrent.session, settings  # pylint: disable=c-extension-no-member
+        )
+        self.tasks.append(self._loop.create_task(self.process_alerts()))
+
+    def stop(self):
+        while self.tasks:
+            self.tasks.pop().cancel()
+        self._session.save_state()
+        self._session.pause()
+        self._session.stop_dht()
+        self._session.stop_lsd()
+        self._session.stop_natpmp()
+        self._session.stop_upnp()
+        self._session = None
+
+    def _pop_alerts(self):
+        for alert in self._session.pop_alerts():
+            log.info("torrent alert: %s", alert)
+
+    async def process_alerts(self):
+        while True:
+            await self._loop.run_in_executor(
+                self._executor, self._pop_alerts
+            )
+            await asyncio.sleep(1)
+
+    async def pause(self):
+        await self._loop.run_in_executor(
+            self._executor, lambda: self._session.save_state()  # pylint: disable=unnecessary-lambda
+        )
+        await self._loop.run_in_executor(
+            self._executor, lambda: self._session.pause()  # pylint: disable=unnecessary-lambda
+        )
+
+    async def resume(self):
+        await self._loop.run_in_executor(
+            self._executor, self._session.resume
+        )
+
+    def _add_torrent(self, btih: str, download_directory: Optional[str]):
+        params = {'info_hash': binascii.unhexlify(btih.encode()), 'flags': DEFAULT_FLAGS}
+        if download_directory:
+            params['save_path'] = download_directory
+        handle = self._session.add_torrent(params)
+        handle.force_dht_announce()
+        self._handles[btih] = TorrentHandle(self._loop, self._executor, handle)
+
+    def full_path(self, btih):
+        return self._handles[btih].largest_file
+
+    async def add_torrent(self, btih, download_path):
+        await self._loop.run_in_executor(
+            self._executor, self._add_torrent, btih, download_path
+        )
+        self._handles[btih].tasks.append(self._loop.create_task(self._handles[btih].status_loop()))
+        await self._handles[btih].metadata_completed.wait()
+        if self.wait_start:
+            # fixme: temporary until we add streaming support, otherwise playback fails!
+            await self._handles[btih].started.wait()
+
+    def remove_torrent(self, btih, remove_files=False):
+        if btih in self._handles:
+            handle = self._handles[btih]
+            handle.stop_tasks()
+            self._session.remove_torrent(handle._handle, 1 if remove_files else 0)
+            self._handles.pop(btih)
+
+    async def save_file(self, btih, download_directory):
+        handle = self._handles[btih]
+        await handle.resume()
+
+    def get_size(self, btih):
+        return self._handles[btih].size
+
+    def get_name(self, btih):
+        return self._handles[btih].name
+
+    def get_downloaded(self, btih):
+        return self._handles[btih].total_wanted_done
+
+    def is_completed(self, btih):
+        return self._handles[btih].finished.is_set()
+
+
+def get_magnet_uri(btih):
+    return f"magnet:?xt=urn:btih:{btih}"
+
+
+def _create_fake_torrent(tmpdir):
+    # beware, that's just for testing
+    path = os.path.join(tmpdir, 'tmp')
+    with open(path, 'wb') as myfile:
+        size = myfile.write(bytes([random.randint(0, 255) for _ in range(40)]) * 1024)
+    file_storage = libtorrent.file_storage()
+    file_storage.add_file('tmp', size)
+    t = libtorrent.create_torrent(file_storage, 0, 4 * 1024 * 1024)
+    libtorrent.set_piece_hashes(t, tmpdir)
+    info = libtorrent.torrent_info(t.generate())
+    btih = sha1(info.metadata()).hexdigest()
+    return info, btih
+
+
+async def main():
+    if os.path.exists("~/Downloads/ubuntu-18.04.3-live-server-amd64.torrent"):
+        os.remove("~/Downloads/ubuntu-18.04.3-live-server-amd64.torrent")
+    if os.path.exists("~/Downloads/ubuntu-18.04.3-live-server-amd64.iso"):
+        os.remove("~/Downloads/ubuntu-18.04.3-live-server-amd64.iso")
+
+    btih = "dd8255ecdc7ca55fb0bbf81323d87062db1f6d1c"
+
+    executor = None
+    session = TorrentSession(asyncio.get_event_loop(), executor)
+    session2 = TorrentSession(asyncio.get_event_loop(), executor)
+    await session.bind('localhost', port=4040)
+    await session2.bind('localhost', port=4041)
+    btih = await session.add_fake_torrent()
+    session2._session.add_dht_node(('localhost', 4040))
+    await session2.add_torrent(btih, "/tmp/down")
+    while True:
+        await asyncio.sleep(100)
+    await session.pause()
+    executor.shutdown()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

lbry/torrent/torrent.py

@@ -0,0 +1,72 @@
+import asyncio
+import logging
+import typing
+
+
+log = logging.getLogger(__name__)
+
+
+class TorrentInfo:
+    __slots__ = ('dht_seeds', 'http_seeds', 'trackers', 'total_size')
+
+    def __init__(self, dht_seeds: typing.Tuple[typing.Tuple[str, int]],
+                 http_seeds: typing.Tuple[typing.Dict[str, typing.Any]],
+                 trackers: typing.Tuple[typing.Tuple[str, int]], total_size: int):
+        self.dht_seeds = dht_seeds
+        self.http_seeds = http_seeds
+        self.trackers = trackers
+        self.total_size = total_size
+
+    @classmethod
+    def from_libtorrent_info(cls, torrent_info):
+        return cls(
+            torrent_info.nodes(), tuple(
+                {
+                    'url': web_seed['url'],
+                    'type': web_seed['type'],
+                    'auth': web_seed['auth']
+                } for web_seed in torrent_info.web_seeds()
+            ), tuple(
+                (tracker.url, tracker.tier) for tracker in torrent_info.trackers()
+            ), torrent_info.total_size()
+        )
+
+
+class Torrent:
+    def __init__(self, loop, handle):
+        self._loop = loop
+        self._handle = handle
+        self.finished = asyncio.Event()
+
+    def _threaded_update_status(self):
+        status = self._handle.status()
+        if not status.is_seeding:
+            log.info(
+                '%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d) %s',
+                status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000,
+                status.num_peers, status.state
+            )
+        elif not self.finished.is_set():
+            self.finished.set()
+
+    async def wait_for_finished(self):
+        while True:
+            await self._loop.run_in_executor(
+                None, self._threaded_update_status
+            )
+            if self.finished.is_set():
+                log.info("finished downloading torrent!")
+                await self.pause()
+                break
+            await asyncio.sleep(1)
+
+    async def pause(self):
+        log.info("pause torrent")
+        await self._loop.run_in_executor(
+            None, self._handle.pause
+        )
+
+    async def resume(self):
+        await self._loop.run_in_executor(
+            None, self._handle.resume
+        )

lbry/torrent/torrent_manager.py

@@ -0,0 +1,140 @@
+import asyncio
+import binascii
+import logging
+import os
+import typing
+from typing import Optional
+from aiohttp.web import Request
+from lbry.file.source_manager import SourceManager
+from lbry.file.source import ManagedDownloadSource
+
+if typing.TYPE_CHECKING:
+    from lbry.torrent.session import TorrentSession
+    from lbry.conf import Config
+    from lbry.wallet.transaction import Transaction
+    from lbry.extras.daemon.analytics import AnalyticsManager
+    from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim
+    from lbry.extras.daemon.storage import StoredContentClaim
+
+log = logging.getLogger(__name__)
+
+
+def path_or_none(encoded_path) -> Optional[str]:
+    if not encoded_path:
+        return
+    return binascii.unhexlify(encoded_path).decode()
+
+
+class TorrentSource(ManagedDownloadSource):
+    STATUS_STOPPED = "stopped"
+    filter_fields = SourceManager.filter_fields
+    filter_fields.update({
+        'bt_infohash'
+    })
+
+    def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage', identifier: str,
+                 file_name: Optional[str] = None, download_directory: Optional[str] = None,
+                 status: Optional[str] = STATUS_STOPPED, claim: Optional['StoredContentClaim'] = None,
+                 download_id: Optional[str] = None, rowid: Optional[int] = None,
+                 content_fee: Optional['Transaction'] = None,
+                 analytics_manager: Optional['AnalyticsManager'] = None,
+                 added_on: Optional[int] = None, torrent_session: Optional['TorrentSession'] = None):
+        super().__init__(loop, config, storage, identifier, file_name, download_directory, status, claim, download_id,
+                         rowid, content_fee, analytics_manager, added_on)
+        self.torrent_session = torrent_session
+
+    @property
+    def full_path(self) -> Optional[str]:
+        full_path = self.torrent_session.full_path(self.identifier)
+        self.download_directory = os.path.dirname(full_path)
+        return full_path
+
+    async def start(self, timeout: Optional[float] = None, save_now: Optional[bool] = False):
+        await self.torrent_session.add_torrent(self.identifier, self.download_directory)
+
+    async def stop(self, finished: bool = False):
+        await self.torrent_session.remove_torrent(self.identifier)
+
+    async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
+        await self.torrent_session.save_file(self.identifier, download_directory)
+
+    @property
+    def torrent_length(self):
+        return self.torrent_session.get_size(self.identifier)
+
+    @property
+    def written_bytes(self):
+        return self.torrent_session.get_downloaded(self.identifier)
+
+    @property
+    def torrent_name(self):
+        return self.torrent_session.get_name(self.identifier)
+
+    @property
+    def bt_infohash(self):
+        return self.identifier
+
+    async def stop_tasks(self):
+        pass
+
+    @property
+    def completed(self):
+        return self.torrent_session.is_completed(self.identifier)
+
+
+class TorrentManager(SourceManager):
+    _sources: typing.Dict[str, ManagedDownloadSource]
+
+    filter_fields = set(SourceManager.filter_fields)
+    filter_fields.update({
+        'bt_infohash',
+        'blobs_remaining',  # TODO: here they call them "parts", but its pretty much the same concept
+        'blobs_in_stream'
+    })
+
+    def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', torrent_session: 'TorrentSession',
+                 storage: 'SQLiteStorage', analytics_manager: Optional['AnalyticsManager'] = None):
+        super().__init__(loop, config, storage, analytics_manager)
+        self.torrent_session: 'TorrentSession' = torrent_session
+
+    async def recover_streams(self, file_infos: typing.List[typing.Dict]):
+        raise NotImplementedError
+
+    async def _load_stream(self, rowid: int, bt_infohash: str, file_name: Optional[str],
+                           download_directory: Optional[str], status: str,
+                           claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
+                           added_on: Optional[int]):
+        stream = TorrentSource(
+            self.loop, self.config, self.storage, identifier=bt_infohash, file_name=file_name,
+            download_directory=download_directory, status=status, claim=claim, rowid=rowid,
+            content_fee=content_fee, analytics_manager=self.analytics_manager, added_on=added_on,
+            torrent_session=self.torrent_session
+        )
+        self.add(stream)
+
+    async def initialize_from_database(self):
+        pass
+
+    async def start(self):
+        await super().start()
+
+    async def stop(self):
+        await super().stop()
+        log.info("finished stopping the torrent manager")
+
+    async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
+        await super().delete(source, delete_file)
+        self.torrent_session.remove_torrent(source.identifier, delete_file)
+
+    async def create(self, file_path: str, key: Optional[bytes] = None,
+                     iv_generator: Optional[typing.Generator[bytes, None, None]] = None):
+        raise NotImplementedError
+
+    async def _delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
+        raise NotImplementedError
+        # blob_hashes = [source.sd_hash] + [b.blob_hash for b in source.descriptor.blobs[:-1]]
+        # await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False)
+        # await self.storage.delete_stream(source.descriptor)
+
+    async def stream_partial_content(self, request: Request, sd_hash: str):
+        raise NotImplementedError

lbry/torrent/tracker.py

@@ -0,0 +1,285 @@
+import random
+import socket
+import string
+import struct
+import asyncio
+import logging
+import time
+import ipaddress
+from collections import namedtuple
+from functools import reduce
+from typing import Optional
+
+from lbry.dht.node import get_kademlia_peers_from_hosts
+from lbry.utils import resolve_host, async_timed_cache, cache_concurrent
+from lbry.wallet.stream import StreamController
+from lbry import version
+
+log = logging.getLogger(__name__)
+CONNECTION_EXPIRES_AFTER_SECONDS = 50
+PREFIX = 'LB'  # todo: PR BEP20 to add ourselves
+DEFAULT_TIMEOUT_SECONDS = 10.0
+DEFAULT_CONCURRENCY_LIMIT = 100
+# see: http://bittorrent.org/beps/bep_0015.html and http://xbtt.sourceforge.net/udp_tracker_protocol.html
+ConnectRequest = namedtuple("ConnectRequest", ["connection_id", "action", "transaction_id"])
+ConnectResponse = namedtuple("ConnectResponse", ["action", "transaction_id", "connection_id"])
+AnnounceRequest = namedtuple("AnnounceRequest",
+                             ["connection_id", "action", "transaction_id", "info_hash", "peer_id", "downloaded", "left",
+                              "uploaded", "event", "ip_addr", "key", "num_want", "port"])
+AnnounceResponse = namedtuple("AnnounceResponse",
+                              ["action", "transaction_id", "interval", "leechers", "seeders", "peers"])
+CompactIPv4Peer = namedtuple("CompactPeer", ["address", "port"])
+ScrapeRequest = namedtuple("ScrapeRequest", ["connection_id", "action", "transaction_id", "infohashes"])
+ScrapeResponse = namedtuple("ScrapeResponse", ["action", "transaction_id", "items"])
+ScrapeResponseItem = namedtuple("ScrapeResponseItem", ["seeders", "completed", "leechers"])
+ErrorResponse = namedtuple("ErrorResponse", ["action", "transaction_id", "message"])
+structs = {
+    ConnectRequest: struct.Struct(">QII"),
+    ConnectResponse: struct.Struct(">IIQ"),
+    AnnounceRequest: struct.Struct(">QII20s20sQQQIIIiH"),
+    AnnounceResponse: struct.Struct(">IIIII"),
+    CompactIPv4Peer: struct.Struct(">IH"),
+    ScrapeRequest: struct.Struct(">QII"),
+    ScrapeResponse: struct.Struct(">II"),
+    ScrapeResponseItem: struct.Struct(">III"),
+    ErrorResponse: struct.Struct(">II")
+}
+
+
+def decode(cls, data, offset=0):
+    decoder = structs[cls]
+    if cls is AnnounceResponse:
+        return AnnounceResponse(*decoder.unpack_from(data, offset),
+                                peers=[decode(CompactIPv4Peer, data, index) for index in range(20, len(data), 6)])
+    elif cls is ScrapeResponse:
+        return ScrapeResponse(*decoder.unpack_from(data, offset),
+                              items=[decode(ScrapeResponseItem, data, index) for index in range(8, len(data), 12)])
+    elif cls is ErrorResponse:
+        return ErrorResponse(*decoder.unpack_from(data, offset), data[decoder.size:])
+    return cls(*decoder.unpack_from(data, offset))
+
+
+def encode(obj):
+    if isinstance(obj, ScrapeRequest):
+        return structs[ScrapeRequest].pack(*obj[:-1]) + b''.join(obj.infohashes)
+    elif isinstance(obj, ErrorResponse):
+        return structs[ErrorResponse].pack(*obj[:-1]) + obj.message
+    elif isinstance(obj, AnnounceResponse):
+        return structs[AnnounceResponse].pack(*obj[:-1]) + b''.join([encode(peer) for peer in obj.peers])
+    return structs[type(obj)].pack(*obj)
+
+
+def make_peer_id(random_part: Optional[str] = None) -> bytes:
+    # see https://wiki.theory.org/BitTorrentSpecification#peer_id and https://www.bittorrent.org/beps/bep_0020.html
+    # not to confuse with node id; peer id identifies uniquely the software, version and instance
+    random_part = random_part or ''.join(random.choice(string.ascii_letters) for _ in range(20))
+    return f"{PREFIX}-{'-'.join(map(str, version))}-{random_part}"[:20].encode()
+
+
+class UDPTrackerClientProtocol(asyncio.DatagramProtocol):
+    def __init__(self, timeout: float = DEFAULT_TIMEOUT_SECONDS):
+        self.transport = None
+        self.data_queue = {}
+        self.timeout = timeout
+        self.semaphore = asyncio.Semaphore(DEFAULT_CONCURRENCY_LIMIT)
+
+    def connection_made(self, transport: asyncio.DatagramTransport) -> None:
+        self.transport = transport
+
+    async def request(self, obj, tracker_ip, tracker_port):
+        self.data_queue[obj.transaction_id] = asyncio.get_running_loop().create_future()
+        try:
+            async with self.semaphore:
+                self.transport.sendto(encode(obj), (tracker_ip, tracker_port))
+                return await asyncio.wait_for(self.data_queue[obj.transaction_id], self.timeout)
+        finally:
+            self.data_queue.pop(obj.transaction_id, None)
+
+    async def connect(self, tracker_ip, tracker_port):
+        transaction_id = random.getrandbits(32)
+        return decode(ConnectResponse,
+                      await self.request(ConnectRequest(0x41727101980, 0, transaction_id), tracker_ip, tracker_port))
+
+    @cache_concurrent
+    @async_timed_cache(CONNECTION_EXPIRES_AFTER_SECONDS)
+    async def ensure_connection_id(self, peer_id, tracker_ip, tracker_port):
+        # peer_id is just to ensure cache coherency
+        return (await self.connect(tracker_ip, tracker_port)).connection_id
+
+    async def announce(self, info_hash, peer_id, port, tracker_ip, tracker_port, stopped=False):
+        connection_id = await self.ensure_connection_id(peer_id, tracker_ip, tracker_port)
+        # this should make the key deterministic but unique per info hash + peer id
+        key = int.from_bytes(info_hash[:4], "big") ^ int.from_bytes(peer_id[:4], "big") ^ port
+        transaction_id = random.getrandbits(32)
+        req = AnnounceRequest(
+            connection_id, 1, transaction_id, info_hash, peer_id, 0, 0, 0, 3 if stopped else 1, 0, key, -1, port)
+        return decode(AnnounceResponse, await self.request(req, tracker_ip, tracker_port))
+
+    async def scrape(self, infohashes, tracker_ip, tracker_port, connection_id=None):
+        connection_id = await self.ensure_connection_id(None, tracker_ip, tracker_port)
+        transaction_id = random.getrandbits(32)
+        reply = await self.request(
+            ScrapeRequest(connection_id, 2, transaction_id, infohashes), tracker_ip, tracker_port)
+        return decode(ScrapeResponse, reply), connection_id
+
+    def datagram_received(self, data: bytes, addr: (str, int)) -> None:
+        if len(data) < 8:
+            return
+        transaction_id = int.from_bytes(data[4:8], byteorder="big", signed=False)
+        if transaction_id in self.data_queue:
+            if not self.data_queue[transaction_id].done():
+                if data[3] == 3:
+                    return self.data_queue[transaction_id].set_exception(Exception(decode(ErrorResponse, data).message))
+                return self.data_queue[transaction_id].set_result(data)
+        log.debug("unexpected packet (can be a response for a previously timed out request): %s", data.hex())
+
+    def connection_lost(self, exc: Exception = None) -> None:
+        self.transport = None
+
+
+class TrackerClient:
+    event_controller = StreamController()
+
+    def __init__(self, node_id, announce_port, get_servers, timeout=10.0):
+        self.client = UDPTrackerClientProtocol(timeout=timeout)
+        self.transport = None
+        self.peer_id = make_peer_id(node_id.hex() if node_id else None)
+        self.announce_port = announce_port
+        self._get_servers = get_servers
+        self.results = {}  # we can't probe the server before the interval, so we keep the result here until it expires
+        self.tasks = {}
+
+    async def start(self):
+        self.transport, _ = await asyncio.get_running_loop().create_datagram_endpoint(
+            lambda: self.client, local_addr=("0.0.0.0", 0))
+        self.event_controller.stream.listen(
+            lambda request: self.on_hash(request[1], request[2]) if request[0] == 'search' else None)
+
+    def stop(self):
+        while self.tasks:
+            self.tasks.popitem()[1].cancel()
+        if self.transport is not None:
+            self.transport.close()
+        self.client = None
+        self.transport = None
+        self.event_controller.close()
+
+    def on_hash(self, info_hash, on_announcement=None):
+        if info_hash not in self.tasks:
+            task = asyncio.create_task(self.get_peer_list(info_hash, on_announcement=on_announcement))
+            task.add_done_callback(lambda *_: self.tasks.pop(info_hash, None))
+            self.tasks[info_hash] = task
+
+    async def announce_many(self, *info_hashes, stopped=False):
+        await asyncio.gather(
+            *[self._announce_many(server, info_hashes, stopped=stopped) for server in self._get_servers()],
+            return_exceptions=True)
+
+    async def _announce_many(self, server, info_hashes, stopped=False):
+        tracker_ip = await resolve_host(*server, 'udp')
+        still_good_info_hashes = {
+            info_hash for (info_hash, (next_announcement, _)) in self.results.get(tracker_ip, {}).items()
+            if time.time() < next_announcement
+        }
+        results = await asyncio.gather(
+            *[self._probe_server(info_hash, tracker_ip, server[1], stopped=stopped)
+              for info_hash in info_hashes if info_hash not in still_good_info_hashes],
+            return_exceptions=True)
+        if results:
+            errors = sum([1 for result in results if result is None or isinstance(result, Exception)])
+            log.info("Tracker: finished announcing %d files to %s:%d, %d errors", len(results), *server, errors)
+
+    async def get_peer_list(self, info_hash, stopped=False, on_announcement=None, no_port=False):
+        found = []
+        probes = [self._probe_server(info_hash, *server, stopped, no_port) for server in self._get_servers()]
+        for done in asyncio.as_completed(probes):
+            result = await done
+            if result is not None:
+                await asyncio.gather(*filter(asyncio.iscoroutine, [on_announcement(result)] if on_announcement else []))
+                found.append(result)
+        return found
+
+    async def get_kademlia_peer_list(self, info_hash):
+        responses = await self.get_peer_list(info_hash, no_port=True)
+        return await announcement_to_kademlia_peers(*responses)
+
+    async def _probe_server(self, info_hash, tracker_host, tracker_port, stopped=False, no_port=False):
+        result = None
+        try:
+            tracker_host = await resolve_host(tracker_host, tracker_port, 'udp')
+        except socket.error:
+            log.warning("DNS failure while resolving tracker host: %s, skipping.", tracker_host)
+            return
+        self.results.setdefault(tracker_host, {})
+        if info_hash in self.results[tracker_host]:
+            next_announcement, result = self.results[tracker_host][info_hash]
+            if time.time() < next_announcement:
+                return result
+        try:
+            result = await self.client.announce(
+                info_hash, self.peer_id, 0 if no_port else self.announce_port, tracker_host, tracker_port, stopped)
+            self.results[tracker_host][info_hash] = (time.time() + result.interval, result)
+        except asyncio.TimeoutError:  # todo: this is UDP, timeout is common, we need a better metric for failures
+            self.results[tracker_host][info_hash] = (time.time() + 60.0, result)
+            log.debug("Tracker timed out: %s:%d", tracker_host, tracker_port)
+            return None
+        log.debug("Announced: %s found %d peers for %s", tracker_host, len(result.peers), info_hash.hex()[:8])
+        return result
+
+
+def enqueue_tracker_search(info_hash: bytes, peer_q: asyncio.Queue):
+    async def on_announcement(announcement: AnnounceResponse):
+        peers = await announcement_to_kademlia_peers(announcement)
+        log.info("Found %d peers from tracker for %s", len(peers), info_hash.hex()[:8])
+        peer_q.put_nowait(peers)
+    TrackerClient.event_controller.add(('search', info_hash, on_announcement))
+
+
+def announcement_to_kademlia_peers(*announcements: AnnounceResponse):
+    peers = [
+        (str(ipaddress.ip_address(peer.address)), peer.port)
+        for announcement in announcements for peer in announcement.peers if peer.port > 1024  # no privileged or 0
+    ]
+    return get_kademlia_peers_from_hosts(peers)
+
+
+class UDPTrackerServerProtocol(asyncio.DatagramProtocol):  # for testing. Not suitable for production
+    def __init__(self):
+        self.transport = None
+        self.known_conns = set()
+        self.peers = {}
+
+    def connection_made(self, transport: asyncio.DatagramTransport) -> None:
+        self.transport = transport
+
+    def add_peer(self, info_hash, ip_address: str, port: int):
+        self.peers.setdefault(info_hash, [])
+        self.peers[info_hash].append(encode_peer(ip_address, port))
+
+    def datagram_received(self, data: bytes, addr: (str, int)) -> None:
+        if len(data) < 16:
+            return
+        action = int.from_bytes(data[8:12], "big", signed=False)
+        if action == 0:
+            req = decode(ConnectRequest, data)
+            connection_id = random.getrandbits(32)
+            self.known_conns.add(connection_id)
+            return self.transport.sendto(encode(ConnectResponse(0, req.transaction_id, connection_id)), addr)
+        elif action == 1:
+            req = decode(AnnounceRequest, data)
+            if req.connection_id not in self.known_conns:
+                resp = encode(ErrorResponse(3, req.transaction_id, b'Connection ID missmatch.\x00'))
+            else:
+                compact_address = encode_peer(addr[0], req.port)
+                if req.event != 3:
+                    self.add_peer(req.info_hash, addr[0], req.port)
+                elif compact_address in self.peers.get(req.info_hash, []):
+                    self.peers[req.info_hash].remove(compact_address)
+                peers = [decode(CompactIPv4Peer, peer) for peer in self.peers[req.info_hash]]
+                resp = encode(AnnounceResponse(1, req.transaction_id, 1700, 0, len(peers), peers))
+            return self.transport.sendto(resp, addr)
+
+
+def encode_peer(ip_address: str, port: int):
+    compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), ip_address.split('.'), bytearray())
+    return compact_ip + port.to_bytes(2, "big", signed=False)

lbry/utils.py

@@ -3,7 +3,9 @@ import codecs
 import datetime
 import random
 import socket
+import time
 import string
+import sys
 import json
 import typing
 import asyncio
@@ -18,8 +20,10 @@ import pkg_resources
 
 import certifi
 import aiohttp
+from prometheus_client import Counter
 from lbry.schema.claim import Claim
 
+
 log = logging.getLogger(__name__)
 
 
@@ -100,10 +104,6 @@ def check_connection(server="lbry.com", port=80, timeout=5) -> bool:
         return False
 
 
-async def async_check_connection(server="lbry.com", port=80, timeout=1) -> bool:
-    return await asyncio.get_event_loop().run_in_executor(None, check_connection, server, port, timeout)
-
-
 def random_string(length=10, chars=string.ascii_lowercase):
     return ''.join([random.choice(chars) for _ in range(length)])
 
@@ -130,21 +130,16 @@ def get_sd_hash(stream_info):
 def json_dumps_pretty(obj, **kwargs):
     return json.dumps(obj, sort_keys=True, indent=2, separators=(',', ': '), **kwargs)
 
-
-def cancel_task(task: typing.Optional[asyncio.Task]):
-    if task and not task.done():
-        task.cancel()
-
-
-def cancel_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
-    for task in tasks:
-        cancel_task(task)
-
-
-def drain_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
-    while tasks:
-        cancel_task(tasks.pop())
-
+try:
+    # the standard contextlib.aclosing() is available in 3.10+
+    from contextlib import aclosing  # pylint: disable=unused-import
+except ImportError:
+    @contextlib.asynccontextmanager
+    async def aclosing(thing):
+        try:
+            yield thing
+        finally:
+            await thing.aclose()
 
 def async_timed_cache(duration: int):
     def wrapper(func):
@@ -155,7 +150,7 @@ def async_timed_cache(duration: int):
         async def _inner(*args, **kwargs) -> typing.Any:
             loop = asyncio.get_running_loop()
             time_now = loop.time()
-            key = tuple([args, tuple([tuple([k, kwargs[k]]) for k in kwargs])])
+            key = (args, tuple(kwargs.items()))
             if key in cache and (time_now - cache[key][1] < duration):
                 return cache[key][0]
             to_cache = await func(*args, **kwargs)
@@ -173,7 +168,7 @@ def cache_concurrent(async_fn):
 
     @functools.wraps(async_fn)
     async def wrapper(*args, **kwargs):
-        key = tuple([args, tuple([tuple([k, kwargs[k]]) for k in kwargs])])
+        key = (args, tuple(kwargs.items()))
         cache[key] = cache.get(key) or asyncio.create_task(async_fn(*args, **kwargs))
         try:
             return await cache[key]
@@ -187,6 +182,8 @@ def cache_concurrent(async_fn):
 async def resolve_host(url: str, port: int, proto: str) -> str:
     if proto not in ['udp', 'tcp']:
         raise Exception("invalid protocol")
+    if url.lower() == 'localhost':
+        return '127.0.0.1'
     try:
         if ipaddress.ip_address(url):
             return url
@@ -201,18 +198,44 @@ async def resolve_host(url: str, port: int, proto: str) -> str:
     ))[0][4][0]
 
 
-class LRUCache:
+class LRUCacheWithMetrics:
     __slots__ = [
         'capacity',
-        'cache'
+        'cache',
+        '_track_metrics',
+        'hits',
+        'misses'
     ]
 
-    def __init__(self, capacity):
+    def __init__(self, capacity: int, metric_name: typing.Optional[str] = None, namespace: str = "daemon_cache"):
         self.capacity = capacity
         self.cache = collections.OrderedDict()
+        if metric_name is None:
+            self._track_metrics = False
+            self.hits = self.misses = None
+        else:
+            self._track_metrics = True
+            try:
+                self.hits = Counter(
+                    f"{metric_name}_cache_hit_count", "Number of cache hits", namespace=namespace
+                )
+                self.misses = Counter(
+                    f"{metric_name}_cache_miss_count", "Number of cache misses", namespace=namespace
+                )
+            except ValueError as err:
+                log.debug("failed to set up prometheus %s_cache_miss_count metric: %s", metric_name, err)
+                self._track_metrics = False
+                self.hits = self.misses = None
 
-    def get(self, key):
-        value = self.cache.pop(key)
+    def get(self, key, default=None):
+        try:
+            value = self.cache.pop(key)
+            if self._track_metrics:
+                self.hits.inc()
+        except KeyError:
+            if self._track_metrics:
+                self.misses.inc()
+            return default
         self.cache[key] = value
         return value
 
@@ -224,22 +247,97 @@ class LRUCache:
                 self.cache.popitem(last=False)
         self.cache[key] = value
 
+    def clear(self):
+        self.cache.clear()
+
+    def pop(self, key):
+        return self.cache.pop(key)
+
+    def __setitem__(self, key, value):
+        return self.set(key, value)
+
+    def __getitem__(self, item):
+        return self.get(item)
+
     def __contains__(self, item) -> bool:
         return item in self.cache
 
+    def __len__(self):
+        return len(self.cache)
+
+    def __delitem__(self, key):
+        self.cache.pop(key)
+
+    def __del__(self):
+        self.clear()
+
+
+class LRUCache:
+    __slots__ = [
+        'capacity',
+        'cache'
+    ]
+
+    def __init__(self, capacity: int):
+        self.capacity = capacity
+        self.cache = collections.OrderedDict()
+
+    def get(self, key, default=None):
+        try:
+            value = self.cache.pop(key)
+        except KeyError:
+            return default
+        self.cache[key] = value
+        return value
+
+    def set(self, key, value):
+        try:
+            self.cache.pop(key)
+        except KeyError:
+            if len(self.cache) >= self.capacity:
+                self.cache.popitem(last=False)
+        self.cache[key] = value
+
+    def items(self):
+        return self.cache.items()
+
+    def clear(self):
+        self.cache.clear()
+
+    def pop(self, key, default=None):
+        return self.cache.pop(key, default)
+
+    def __setitem__(self, key, value):
+        return self.set(key, value)
+
+    def __getitem__(self, item):
+        return self.get(item)
+
+    def __contains__(self, item) -> bool:
+        return item in self.cache
+
+    def __len__(self):
+        return len(self.cache)
+
+    def __delitem__(self, key):
+        self.cache.pop(key)
+
+    def __del__(self):
+        self.clear()
+
 
 def lru_cache_concurrent(cache_size: typing.Optional[int] = None,
-                         override_lru_cache: typing.Optional[LRUCache] = None):
+                         override_lru_cache: typing.Optional[LRUCacheWithMetrics] = None):
     if not cache_size and override_lru_cache is None:
         raise ValueError("invalid cache size")
     concurrent_cache = {}
-    lru_cache = override_lru_cache or LRUCache(cache_size)
+    lru_cache = override_lru_cache if override_lru_cache is not None else LRUCacheWithMetrics(cache_size)
 
     def wrapper(async_fn):
 
         @functools.wraps(async_fn)
         async def _inner(*args, **kwargs):
-            key = tuple([args, tuple([tuple([k, kwargs[k]]) for k in kwargs])])
+            key = (args, tuple(kwargs.items()))
             if key in lru_cache:
                 return lru_cache.get(key)
 
@@ -264,15 +362,125 @@ def get_ssl_context() -> ssl.SSLContext:
 @contextlib.asynccontextmanager
 async def aiohttp_request(method, url, **kwargs) -> typing.AsyncContextManager[aiohttp.ClientResponse]:
     async with aiohttp.ClientSession() as session:
-        async with session.request(method, url, ssl=get_ssl_context(), **kwargs) as response:
+        async with session.request(method, url, **kwargs) as response:
             yield response
 
 
-async def get_external_ip() -> typing.Optional[str]:  # used if upnp is disabled or non-functioning
+# the ipaddress module does not show these subnets as reserved
+CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
+IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
+
+
+def is_valid_public_ipv4(address, allow_localhost: bool = False, allow_lan: bool = False):
+    try:
+        parsed_ip = ipaddress.ip_address(address)
+        if parsed_ip.is_loopback and allow_localhost:
+            return True
+        if allow_lan and parsed_ip.is_private:
+            return True
+        if any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local, parsed_ip.is_loopback,
+                parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private)):
+            return False
+        else:
+            return not any((CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
+                            IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
+    except (ipaddress.AddressValueError, ValueError):
+        return False
+
+
+async def fallback_get_external_ip():  # used if spv servers can't be used for ip detection
     try:
         async with aiohttp_request("get", "https://api.lbry.com/ip") as resp:
             response = await resp.json()
             if response['success']:
-                return response['data']['ip']
+                return response['data']['ip'], None
     except Exception:
-        return
+        return None, None
+
+
+async def _get_external_ip(default_servers) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]:
+    # used if upnp is disabled or non-functioning
+    from lbry.wallet.udp import SPVStatusClientProtocol  # pylint: disable=C0415
+
+    hostname_to_ip = {}
+    ip_to_hostnames = collections.defaultdict(list)
+
+    async def resolve_spv(server, port):
+        try:
+            server_addr = await resolve_host(server, port, 'udp')
+            hostname_to_ip[server] = (server_addr, port)
+            ip_to_hostnames[(server_addr, port)].append(server)
+        except Exception:
+            log.exception("error looking up dns for spv servers")
+
+    # accumulate the dns results
+    await asyncio.gather(*(resolve_spv(server, port) for (server, port) in default_servers))
+
+    loop = asyncio.get_event_loop()
+    pong_responses = asyncio.Queue()
+    connection = SPVStatusClientProtocol(pong_responses)
+    try:
+        await loop.create_datagram_endpoint(lambda: connection, ('0.0.0.0', 0))
+        # could raise OSError if it cant bind
+        randomized_servers = list(ip_to_hostnames.keys())
+        random.shuffle(randomized_servers)
+        for server in randomized_servers:
+            connection.ping(server)
+            try:
+                _, pong = await asyncio.wait_for(pong_responses.get(), 1)
+                if is_valid_public_ipv4(pong.ip_address):
+                    return pong.ip_address, ip_to_hostnames[server][0]
+            except asyncio.TimeoutError:
+                pass
+        return None, None
+    finally:
+        connection.close()
+
+
+async def get_external_ip(default_servers) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]:
+    ip_from_spv_servers = await _get_external_ip(default_servers)
+    if not ip_from_spv_servers[1]:
+        return await fallback_get_external_ip()
+    return ip_from_spv_servers
+
+
+def is_running_from_bundle():
+    # see https://pyinstaller.readthedocs.io/en/stable/runtime-information.html
+    return getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS')
+
+
+class LockWithMetrics(asyncio.Lock):
+    def __init__(self, acquire_metric, held_time_metric):
+        super().__init__()
+        self._acquire_metric = acquire_metric
+        self._lock_held_time_metric = held_time_metric
+        self._lock_acquired_time = None
+
+    async def acquire(self):
+        start = time.perf_counter()
+        try:
+            return await super().acquire()
+        finally:
+            self._lock_acquired_time = time.perf_counter()
+            self._acquire_metric.observe(self._lock_acquired_time - start)
+
+    def release(self):
+        try:
+            return super().release()
+        finally:
+            self._lock_held_time_metric.observe(time.perf_counter() - self._lock_acquired_time)
+
+
+def get_colliding_prefix_bits(first_value: bytes, second_value: bytes):
+    """
+    Calculates the amount of colliding prefix bits between <first_value> and <second_value>.
+    This is given by the amount of bits that are the same until the first different one (via XOR),
+    starting from the most significant bit to the least significant bit.
+    :param first_value: first value to compare, bigger than size.
+    :param second_value: second value to compare, bigger than size.
+    :return: amount of prefix colliding bits.
+    """
+    assert len(first_value) == len(second_value), "length should be the same"
+    size = len(first_value) * 8
+    first_value, second_value = int.from_bytes(first_value, "big"), int.from_bytes(second_value, "big")
+    return size - (first_value ^ second_value).bit_length()

lbry/wallet/__init__.py

@@ -1,17 +1,23 @@
-__node_daemon__ = 'lbrycrdd'
-__node_cli__ = 'lbrycrd-cli'
-__node_bin__ = ''
-__node_url__ = (
-    'https://github.com/lbryio/lbrycrd/releases/download/v0.17.3.2/lbrycrd-linux-1732.zip'
+__lbcd__ = 'lbcd'
+__lbcctl__ = 'lbcctl'
+__lbcwallet__ = 'lbcwallet'
+__lbcd_url__ = (
+    'https://github.com/lbryio/lbcd/releases/download/' +
+    'v0.22.100-rc.0/lbcd_0.22.100-rc.0_TARGET_PLATFORM.tar.gz'
+)
+__lbcwallet_url__ = (
+    'https://github.com/lbryio/lbcwallet/releases/download/' +
+    'v0.13.100-alpha.0/lbcwallet_0.13.100-alpha.0_TARGET_PLATFORM.tar.gz'
 )
 __spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
 
-from .wallet import Wallet, WalletStorage, TimestampedPreferences, ENCRYPT_ON_DISK
-from .manager import WalletManager
-from .network import Network
-from .ledger import Ledger, RegTestLedger, TestNetLedger, BlockHeightEvent
-from .account import Account, AddressManager, SingleKey, HierarchicalDeterministic
-from .transaction import Transaction, Output, Input
-from .script import OutputScript, InputScript
-from .database import SQLiteMixin, Database
-from .header import Headers
+from lbry.wallet.wallet import Wallet, WalletStorage, TimestampedPreferences, ENCRYPT_ON_DISK
+from lbry.wallet.manager import WalletManager
+from lbry.wallet.network import Network
+from lbry.wallet.ledger import Ledger, RegTestLedger, TestNetLedger, BlockHeightEvent
+from lbry.wallet.account import Account, AddressManager, SingleKey, HierarchicalDeterministic, \
+    DeterministicChannelKeyManager
+from lbry.wallet.transaction import Transaction, Output, Input
+from lbry.wallet.script import OutputScript, InputScript
+from lbry.wallet.database import SQLiteMixin, Database
+from lbry.wallet.header import Headers

lbry/wallet/account.py

@@ -5,18 +5,16 @@ import logging
 import typing
 import asyncio
 import random
-from functools import partial
 from hashlib import sha256
 from string import hexdigits
 from typing import Type, Dict, Tuple, Optional, Any, List
 
-import ecdsa
 from lbry.error import InvalidPasswordError
 from lbry.crypto.crypt import aes_encrypt, aes_decrypt
 
-from .bip32 import PrivateKey, PubKey, from_extended_key_string
+from .bip32 import PrivateKey, PublicKey, KeyPath, from_extended_key_string
 from .mnemonic import Mnemonic
-from .constants import COIN, CLAIM_TYPES, TXO_TYPES
+from .constants import COIN, TXO_TYPES
 from .transaction import Transaction, Input, Output
 
 if typing.TYPE_CHECKING:
@@ -35,6 +33,49 @@ def validate_claim_id(claim_id):
         raise Exception("Claim id is not hex encoded")
 
 
+class DeterministicChannelKeyManager:
+
+    def __init__(self, account: 'Account'):
+        self.account = account
+        self.last_known = 0
+        self.cache = {}
+        self._private_key: Optional[PrivateKey] = None
+
+    @property
+    def private_key(self):
+        if self._private_key is None:
+            if self.account.private_key is not None:
+                self._private_key = self.account.private_key.child(KeyPath.CHANNEL)
+        return self._private_key
+
+    def maybe_generate_deterministic_key_for_channel(self, txo):
+        if self.private_key is None:
+            return
+        next_private_key = self.private_key.child(self.last_known)
+        public_key = next_private_key.public_key
+        public_key_bytes = public_key.pubkey_bytes
+        if txo.claim.channel.public_key_bytes == public_key_bytes:
+            self.cache[public_key.address] = next_private_key
+            self.last_known += 1
+
+    async def ensure_cache_primed(self):
+        if self.private_key is not None:
+            await self.generate_next_key()
+
+    async def generate_next_key(self) -> PrivateKey:
+        db = self.account.ledger.db
+        while True:
+            next_private_key = self.private_key.child(self.last_known)
+            public_key = next_private_key.public_key
+            self.cache[public_key.address] = next_private_key
+            if not await db.is_channel_key_used(self.account, public_key):
+                return next_private_key
+            self.last_known += 1
+
+    def get_private_key_from_pubkey_hash(self, pubkey_hash) -> PrivateKey:
+        return self.cache.get(pubkey_hash)
+
+
 class AddressManager:
 
     name: str
@@ -71,6 +112,7 @@ class AddressManager:
 
     def _query_addresses(self, **constraints):
         return self.account.ledger.db.get_addresses(
+            read_only=constraints.pop("read_only", False),
             accounts=[self.account],
             chain=self.chain_number,
             **constraints
@@ -79,7 +121,7 @@ class AddressManager:
     def get_private_key(self, index: int) -> PrivateKey:
         raise NotImplementedError
 
-    def get_public_key(self, index: int) -> PubKey:
+    def get_public_key(self, index: int) -> PublicKey:
         raise NotImplementedError
 
     async def get_max_gap(self):
@@ -96,7 +138,8 @@ class AddressManager:
         return [r['address'] for r in records]
 
     async def get_or_create_usable_address(self) -> str:
-        addresses = await self.get_addresses(only_usable=True, limit=10)
+        async with self.address_generator_lock:
+            addresses = await self.get_addresses(only_usable=True, limit=10)
         if addresses:
             return random.choice(addresses)
         addresses = await self.ensure_address_gap()
@@ -118,8 +161,8 @@ class HierarchicalDeterministic(AddressManager):
     @classmethod
     def from_dict(cls, account: 'Account', d: dict) -> Tuple[AddressManager, AddressManager]:
         return (
-            cls(account, 0, **d.get('receiving', {'gap': 20, 'maximum_uses_per_address': 1})),
-            cls(account, 1, **d.get('change', {'gap': 6, 'maximum_uses_per_address': 1}))
+            cls(account, KeyPath.RECEIVE, **d.get('receiving', {'gap': 20, 'maximum_uses_per_address': 1})),
+            cls(account, KeyPath.CHANGE, **d.get('change', {'gap': 6, 'maximum_uses_per_address': 1}))
         )
 
     def merge(self, d: dict):
@@ -132,7 +175,7 @@ class HierarchicalDeterministic(AddressManager):
     def get_private_key(self, index: int) -> PrivateKey:
         return self.account.private_key.child(self.chain_number).child(index)
 
-    def get_public_key(self, index: int) -> PubKey:
+    def get_public_key(self, index: int) -> PublicKey:
         return self.account.public_key.child(self.chain_number).child(index)
 
     async def get_max_gap(self) -> int:
@@ -192,7 +235,7 @@ class SingleKey(AddressManager):
     @classmethod
     def from_dict(cls, account: 'Account', d: dict) \
             -> Tuple[AddressManager, AddressManager]:
-        same_address_manager = cls(account, account.public_key, 0)
+        same_address_manager = cls(account, account.public_key, KeyPath.RECEIVE)
         return same_address_manager, same_address_manager
 
     def to_dict_instance(self):
@@ -201,7 +244,7 @@ class SingleKey(AddressManager):
     def get_private_key(self, index: int) -> PrivateKey:
         return self.account.private_key
 
-    def get_public_key(self, index: int) -> PubKey:
+    def get_public_key(self, index: int) -> PublicKey:
         return self.account.public_key
 
     async def get_max_gap(self) -> int:
@@ -223,9 +266,6 @@ class SingleKey(AddressManager):
 
 class Account:
 
-    mnemonic_class = Mnemonic
-    private_key_class = PrivateKey
-    public_key_class = PubKey
     address_generators: Dict[str, Type[AddressManager]] = {
         SingleKey.name: SingleKey,
         HierarchicalDeterministic.name: HierarchicalDeterministic,
@@ -233,7 +273,7 @@ class Account:
 
     def __init__(self, ledger: 'Ledger', wallet: 'Wallet', name: str,
                  seed: str, private_key_string: str, encrypted: bool,
-                 private_key: Optional[PrivateKey], public_key: PubKey,
+                 private_key: Optional[PrivateKey], public_key: PublicKey,
                  address_generator: dict, modified_on: float, channel_keys: dict) -> None:
         self.ledger = ledger
         self.wallet = wallet
@@ -244,13 +284,14 @@ class Account:
         self.private_key_string = private_key_string
         self.init_vectors: Dict[str, bytes] = {}
         self.encrypted = encrypted
-        self.private_key = private_key
-        self.public_key = public_key
+        self.private_key: Optional[PrivateKey] = private_key
+        self.public_key: PublicKey = public_key
         generator_name = address_generator.get('name', HierarchicalDeterministic.name)
         self.address_generator = self.address_generators[generator_name]
         self.receiving, self.change = self.address_generator.from_dict(self, address_generator)
-        self.address_managers = {am.chain_number: am for am in {self.receiving, self.change}}
+        self.address_managers = {am.chain_number: am for am in (self.receiving, self.change)}
         self.channel_keys = channel_keys
+        self.deterministic_channel_keys = DeterministicChannelKeyManager(self)
         ledger.add_account(self)
         wallet.add_account(self)
 
@@ -265,19 +306,19 @@ class Account:
                  name: str = None, address_generator: dict = None):
         return cls.from_dict(ledger, wallet, {
             'name': name,
-            'seed': cls.mnemonic_class().make_seed(),
+            'seed': Mnemonic().make_seed(),
             'address_generator': address_generator or {}
         })
 
     @classmethod
     def get_private_key_from_seed(cls, ledger: 'Ledger', seed: str, password: str):
-        return cls.private_key_class.from_seed(
-            ledger, cls.mnemonic_class.mnemonic_to_seed(seed, password or 'lbryum')
+        return PrivateKey.from_seed(
+            ledger, Mnemonic.mnemonic_to_seed(seed, password or 'lbryum')
         )
 
     @classmethod
     def keys_from_dict(cls, ledger: 'Ledger', d: dict) \
-            -> Tuple[str, Optional[PrivateKey], PubKey]:
+            -> Tuple[str, Optional[PrivateKey], PublicKey]:
         seed = d.get('seed', '')
         private_key_string = d.get('private_key', '')
         private_key = None
@@ -310,7 +351,7 @@ class Account:
             private_key=private_key,
             public_key=public_key,
             address_generator=d.get('address_generator', {}),
-            modified_on=d.get('modified_on', time.time()),
+            modified_on=int(d.get('modified_on', time.time())),
             channel_keys=d.get('certificates', {})
         )
 
@@ -342,7 +383,7 @@ class Account:
     def merge(self, d: dict):
         if d.get('modified_on', 0) > self.modified_on:
             self.name = d['name']
-            self.modified_on = d.get('modified_on', time.time())
+            self.modified_on = int(d.get('modified_on', time.time()))
             assert self.address_generator.name == d['address_generator']['name']
             for chain_name in ('change', 'receiving'):
                 if chain_name in d['address_generator']:
@@ -434,9 +475,9 @@ class Account:
             addresses.extend(new_addresses)
         return addresses
 
-    async def get_addresses(self, **constraints) -> List[str]:
-        rows = await self.ledger.db.select_addresses('address', accounts=[self], **constraints)
-        return [r[0] for r in rows]
+    async def get_addresses(self, read_only=False, **constraints) -> List[str]:
+        rows = await self.ledger.db.select_addresses('address', read_only=read_only, accounts=[self], **constraints)
+        return [r['address'] for r in rows]
 
     def get_address_records(self, **constraints):
         return self.ledger.db.get_addresses(accounts=[self], **constraints)
@@ -448,16 +489,16 @@ class Account:
         assert not self.encrypted, "Cannot get private key on encrypted wallet account."
         return self.address_managers[chain].get_private_key(index)
 
-    def get_public_key(self, chain: int, index: int) -> PubKey:
+    def get_public_key(self, chain: int, index: int) -> PublicKey:
         return self.address_managers[chain].get_public_key(index)
 
-    def get_balance(self, confirmations: int = 0, include_claims=False, **constraints):
+    def get_balance(self, confirmations=0, include_claims=False, read_only=False, **constraints):
         if not include_claims:
-            constraints.update({'txo_type__in': (0, TXO_TYPES['purchase'])})
+            constraints.update({'txo_type__in': (TXO_TYPES['other'], TXO_TYPES['purchase'])})
         if confirmations > 0:
             height = self.ledger.headers.height - (confirmations-1)
             constraints.update({'height__lte': height, 'height__gt': 0})
-        return self.ledger.db.get_balance(accounts=[self], **constraints)
+        return self.ledger.db.get_balance(accounts=[self], read_only=read_only, **constraints)
 
     async def get_max_gap(self):
         change_gap = await self.change.get_max_gap()
@@ -467,6 +508,12 @@ class Account:
             'max_receiving_gap': receiving_gap,
         }
 
+    def get_txos(self, **constraints):
+        return self.ledger.get_txos(wallet=self.wallet, accounts=[self], **constraints)
+
+    def get_txo_count(self, **constraints):
+        return self.ledger.get_txo_count(wallet=self.wallet, accounts=[self], **constraints)
+
     def get_utxos(self, **constraints):
         return self.ledger.get_utxos(wallet=self.wallet, accounts=[self], **constraints)
 
@@ -513,16 +560,18 @@ class Account:
 
         return tx
 
-    def add_channel_private_key(self, private_key):
-        public_key_bytes = private_key.get_verifying_key().to_der()
-        channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes)
-        self.channel_keys[channel_pubkey_hash] = private_key.to_pem().decode()
+    async def generate_channel_private_key(self):
+        return await self.deterministic_channel_keys.generate_next_key()
 
-    def get_channel_private_key(self, public_key_bytes):
+    def add_channel_private_key(self, private_key: PrivateKey):
+        self.channel_keys[private_key.address] = private_key.to_pem().decode()
+
+    async def get_channel_private_key(self, public_key_bytes) -> PrivateKey:
         channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes)
         private_key_pem = self.channel_keys.get(channel_pubkey_hash)
         if private_key_pem:
-            return ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=sha256)
+            return PrivateKey.from_pem(self.ledger, private_key_pem)
+        return self.deterministic_channel_keys.get_private_key_from_pubkey_hash(channel_pubkey_hash)
 
     async def maybe_migrate_certificates(self):
         if not self.channel_keys:
@@ -531,11 +580,10 @@ class Account:
         for private_key_pem in self.channel_keys.values():
             if not isinstance(private_key_pem, str):
                 continue
-            if "-----BEGIN EC PRIVATE KEY-----" not in private_key_pem:
+            if not private_key_pem.startswith("-----BEGIN"):
                 continue
-            private_key = ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=sha256)
-            public_key_der = private_key.get_verifying_key().to_der()
-            channel_keys[self.ledger.public_key_to_address(public_key_der)] = private_key_pem
+            private_key = PrivateKey.from_pem(self.ledger, private_key_pem)
+            channel_keys[private_key.address] = private_key_pem
         if self.channel_keys != channel_keys:
             self.channel_keys = channel_keys
             self.wallet.save()
@@ -555,41 +603,24 @@ class Account:
             if gap_changed:
                 self.wallet.save()
 
-    async def get_detailed_balance(self, confirmations=0, reserved_subtotals=False):
-        tips_balance, supports_balance, claims_balance = 0, 0, 0
-        get_total_balance = partial(self.get_balance, confirmations=confirmations, include_claims=True)
-        total = await get_total_balance()
-        if reserved_subtotals:
-            claims_balance = await get_total_balance(txo_type__in=CLAIM_TYPES)
-            for amount, spent, from_me, to_me, height in await self.get_support_summary():
-                if confirmations > 0 and not 0 < height <= self.ledger.headers.height - (confirmations - 1):
-                    continue
-                if not spent and to_me:
-                    if from_me:
-                        supports_balance += amount
-                    else:
-                        tips_balance += amount
-            reserved = claims_balance + supports_balance + tips_balance
-        else:
-            reserved = await self.get_balance(
-                confirmations=confirmations, include_claims=True, txo_type__gt=0
-            )
-        return {
-            'total': total,
-            'available': total - reserved,
-            'reserved': reserved,
-            'reserved_subtotals': {
-                'claims': claims_balance,
-                'supports': supports_balance,
-                'tips': tips_balance
-            } if reserved_subtotals else None
-        }
+    async def get_detailed_balance(self, confirmations=0, read_only=False):
+        constraints = {}
+        if confirmations > 0:
+            height = self.ledger.headers.height - (confirmations-1)
+            constraints.update({'height__lte': height, 'height__gt': 0})
+        return await self.ledger.db.get_detailed_balance(
+            accounts=[self], read_only=read_only, **constraints
+        )
 
-    def get_transaction_history(self, **constraints):
-        return self.ledger.get_transaction_history(wallet=self.wallet, accounts=[self], **constraints)
+    def get_transaction_history(self, read_only=False, **constraints):
+        return self.ledger.get_transaction_history(
+            read_only=read_only, wallet=self.wallet, accounts=[self], **constraints
+        )
 
-    def get_transaction_history_count(self, **constraints):
-        return self.ledger.get_transaction_history_count(wallet=self.wallet, accounts=[self], **constraints)
+    def get_transaction_history_count(self, read_only=False, **constraints):
+        return self.ledger.get_transaction_history_count(
+            read_only=read_only, wallet=self.wallet, accounts=[self], **constraints
+        )
 
     def get_claims(self, **constraints):
         return self.ledger.get_claims(wallet=self.wallet, accounts=[self], **constraints)
@@ -622,7 +653,7 @@ class Account:
         return self.ledger.get_support_count(wallet=self.wallet, accounts=[self], **constraints)
 
     def get_support_summary(self):
-        return self.ledger.db.get_supports_summary(account_id=self.id)
+        return self.ledger.db.get_supports_summary(wallet=self.wallet, accounts=[self])
 
     async def release_all_outputs(self):
         await self.ledger.db.release_all_outputs(self)

lbry/wallet/bip32.py

@@ -1,10 +1,21 @@
-from coincurve import PublicKey, PrivateKey as _PrivateKey
+from asn1crypto.keys import PrivateKeyInfo, ECPrivateKey
+from coincurve import PublicKey as cPublicKey, PrivateKey as cPrivateKey
+from coincurve.utils import (
+    pem_to_der, lib as libsecp256k1, ffi as libsecp256k1_ffi
+)
+from coincurve.ecdsa import CDATA_SIG_LENGTH
 
 from lbry.crypto.hash import hmac_sha512, hash160, double_sha256
 from lbry.crypto.base58 import Base58
 from .util import cachedproperty
 
 
+class KeyPath:
+    RECEIVE = 0
+    CHANGE = 1
+    CHANNEL = 2
+
+
 class DerivationError(Exception):
     """ Raised when an invalid derivation occurs. """
 
@@ -46,9 +57,11 @@ class _KeyBase:
         if len(raw_serkey) != 33:
             raise ValueError('raw_serkey must have length 33')
 
-        return (ver_bytes + bytes((self.depth,))
-                + self.parent_fingerprint() + self.n.to_bytes(4, 'big')
-                + self.chain_code + raw_serkey)
+        return (
+            ver_bytes + bytes((self.depth,))
+            + self.parent_fingerprint() + self.n.to_bytes(4, 'big')
+            + self.chain_code + raw_serkey
+        )
 
     def identifier(self):
         raise NotImplementedError
@@ -69,26 +82,30 @@ class _KeyBase:
         return Base58.encode_check(self.extended_key())
 
 
-class PubKey(_KeyBase):
+class PublicKey(_KeyBase):
     """ A BIP32 public key. """
 
     def __init__(self, ledger, pubkey, chain_code, n, depth, parent=None):
         super().__init__(ledger, chain_code, n, depth, parent)
-        if isinstance(pubkey, PublicKey):
+        if isinstance(pubkey, cPublicKey):
             self.verifying_key = pubkey
         else:
             self.verifying_key = self._verifying_key_from_pubkey(pubkey)
 
+    @classmethod
+    def from_compressed(cls, public_key_bytes, ledger=None) -> 'PublicKey':
+        return cls(ledger, public_key_bytes, bytes((0,)*32), 0, 0)
+
     @classmethod
     def _verifying_key_from_pubkey(cls, pubkey):
-        """ Converts a 33-byte compressed pubkey into an PublicKey object. """
+        """ Converts a 33-byte compressed pubkey into an coincurve.PublicKey object. """
         if not isinstance(pubkey, (bytes, bytearray)):
             raise TypeError('pubkey must be raw bytes')
         if len(pubkey) != 33:
             raise ValueError('pubkey must be 33 bytes')
         if pubkey[0] not in (2, 3):
             raise ValueError('invalid pubkey prefix byte')
-        return PublicKey(pubkey)
+        return cPublicKey(pubkey)
 
     @cachedproperty
     def pubkey_bytes(self):
@@ -103,7 +120,7 @@ class PubKey(_KeyBase):
     def ec_point(self):
         return self.verifying_key.point()
 
-    def child(self, n: int):
+    def child(self, n: int) -> 'PublicKey':
         """ Return the derived child extended pubkey at index N. """
         if not 0 <= n < (1 << 31):
             raise ValueError('invalid BIP32 public key child number')
@@ -111,7 +128,7 @@ class PubKey(_KeyBase):
         msg = self.pubkey_bytes + n.to_bytes(4, 'big')
         L_b, R_b = self._hmac_sha512(msg)  # pylint: disable=invalid-name
         derived_key = self.verifying_key.add(L_b)
-        return PubKey(self.ledger, derived_key, R_b, n, self.depth + 1, self)
+        return PublicKey(self.ledger, derived_key, R_b, n, self.depth + 1, self)
 
     def identifier(self):
         """ Return the key's identifier as 20 bytes. """
@@ -124,6 +141,36 @@ class PubKey(_KeyBase):
             self.pubkey_bytes
         )
 
+    def verify(self, signature, digest) -> bool:
+        """ Verify that a signature is valid for a 32 byte digest. """
+
+        if len(signature) != 64:
+            raise ValueError('Signature must be 64 bytes long.')
+
+        if len(digest) != 32:
+            raise ValueError('Digest must be 32 bytes long.')
+
+        key = self.verifying_key
+
+        raw_signature = libsecp256k1_ffi.new('secp256k1_ecdsa_signature *')
+
+        parsed = libsecp256k1.secp256k1_ecdsa_signature_parse_compact(
+            key.context.ctx, raw_signature, signature
+        )
+        assert parsed == 1
+
+        normalized_signature = libsecp256k1_ffi.new('secp256k1_ecdsa_signature *')
+
+        libsecp256k1.secp256k1_ecdsa_signature_normalize(
+            key.context.ctx, normalized_signature, raw_signature
+        )
+
+        verified = libsecp256k1.secp256k1_ecdsa_verify(
+            key.context.ctx, normalized_signature, digest, key.public_key
+        )
+
+        return bool(verified)
+
 
 class PrivateKey(_KeyBase):
     """A BIP32 private key."""
@@ -132,7 +179,7 @@ class PrivateKey(_KeyBase):
 
     def __init__(self, ledger, privkey, chain_code, n, depth, parent=None):
         super().__init__(ledger, chain_code, n, depth, parent)
-        if isinstance(privkey, _PrivateKey):
+        if isinstance(privkey, cPrivateKey):
             self.signing_key = privkey
         else:
             self.signing_key = self._signing_key_from_privkey(privkey)
@@ -140,7 +187,7 @@ class PrivateKey(_KeyBase):
     @classmethod
     def _signing_key_from_privkey(cls, private_key):
         """ Converts a 32-byte private key into an coincurve.PrivateKey object. """
-        return _PrivateKey.from_int(PrivateKey._private_key_secret_exponent(private_key))
+        return cPrivateKey.from_int(PrivateKey._private_key_secret_exponent(private_key))
 
     @classmethod
     def _private_key_secret_exponent(cls, private_key):
@@ -152,24 +199,40 @@ class PrivateKey(_KeyBase):
         return int.from_bytes(private_key, 'big')
 
     @classmethod
-    def from_seed(cls, ledger, seed):
+    def from_seed(cls, ledger, seed) -> 'PrivateKey':
         # This hard-coded message string seems to be coin-independent...
         hmac = hmac_sha512(b'Bitcoin seed', seed)
         privkey, chain_code = hmac[:32], hmac[32:]
         return cls(ledger, privkey, chain_code, 0, 0)
 
+    @classmethod
+    def from_pem(cls, ledger, pem) -> 'PrivateKey':
+        der = pem_to_der(pem.encode())
+        try:
+            key_int = ECPrivateKey.load(der).native['private_key']
+        except ValueError:
+            key_int = PrivateKeyInfo.load(der).native['private_key']['private_key']
+        private_key = cPrivateKey.from_int(key_int)
+        return cls(ledger, private_key, bytes((0,)*32), 0, 0)
+
+    @classmethod
+    def from_bytes(cls, ledger, key_bytes) -> 'PrivateKey':
+        return cls(ledger, cPrivateKey(key_bytes), bytes((0,)*32), 0, 0)
+
     @cachedproperty
     def private_key_bytes(self):
         """ Return the serialized private key (no leading zero byte). """
         return self.signing_key.secret
 
     @cachedproperty
-    def public_key(self):
+    def public_key(self) -> PublicKey:
         """ Return the corresponding extended public key. """
         verifying_key = self.signing_key.public_key
         parent_pubkey = self.parent.public_key if self.parent else None
-        return PubKey(self.ledger, verifying_key, self.chain_code, self.n, self.depth,
-                      parent_pubkey)
+        return PublicKey(
+            self.ledger, verifying_key, self.chain_code,
+            self.n, self.depth, parent_pubkey
+        )
 
     def ec_point(self):
         return self.public_key.ec_point()
@@ -182,11 +245,12 @@ class PrivateKey(_KeyBase):
         """ Return the private key encoded in Wallet Import Format. """
         return self.ledger.private_key_to_wif(self.private_key_bytes)
 
+    @property
     def address(self):
         """ The public key as a P2PKH address. """
         return self.public_key.address
 
-    def child(self, n):
+    def child(self, n) -> 'PrivateKey':
         """ Return the derived child extended private key at index N."""
         if not 0 <= n < (1 << 32):
             raise ValueError('invalid BIP32 private key child number')
@@ -205,6 +269,28 @@ class PrivateKey(_KeyBase):
         """ Produce a signature for piece of data by double hashing it and signing the hash. """
         return self.signing_key.sign(data, hasher=double_sha256)
 
+    def sign_compact(self, digest):
+        """ Produce a compact signature. """
+        key = self.signing_key
+
+        signature = libsecp256k1_ffi.new('secp256k1_ecdsa_signature *')
+        signed = libsecp256k1.secp256k1_ecdsa_sign(
+            key.context.ctx, signature, digest, key.secret,
+            libsecp256k1_ffi.NULL, libsecp256k1_ffi.NULL
+        )
+
+        if not signed:
+            raise ValueError('The private key was invalid.')
+
+        serialized = libsecp256k1_ffi.new('unsigned char[%d]' % CDATA_SIG_LENGTH)
+        compacted = libsecp256k1.secp256k1_ecdsa_signature_serialize_compact(
+            key.context.ctx, serialized, signature
+        )
+        if compacted != 1:
+            raise ValueError('The signature could not be compacted.')
+
+        return bytes(libsecp256k1_ffi.buffer(serialized, CDATA_SIG_LENGTH))
+
     def identifier(self):
         """Return the key's identifier as 20 bytes."""
         return self.public_key.identifier()
@@ -216,9 +302,12 @@ class PrivateKey(_KeyBase):
             b'\0' + self.private_key_bytes
         )
 
+    def to_pem(self):
+        return self.signing_key.to_pem()
+
 
 def _from_extended_key(ledger, ekey):
-    """Return a PubKey or PrivateKey from an extended key raw bytes."""
+    """Return a PublicKey or PrivateKey from an extended key raw bytes."""
     if not isinstance(ekey, (bytes, bytearray)):
         raise TypeError('extended key must be raw bytes')
     if len(ekey) != 78:
@@ -230,7 +319,7 @@ def _from_extended_key(ledger, ekey):
 
     if ekey[:4] == ledger.extended_public_key_prefix:
         pubkey = ekey[45:]
-        key = PubKey(ledger, pubkey, chain_code, n, depth)
+        key = PublicKey(ledger, pubkey, chain_code, n, depth)
     elif ekey[:4] == ledger.extended_private_key_prefix:
         if ekey[45] != 0:
             raise ValueError('invalid extended private key prefix byte')
@@ -248,6 +337,6 @@ def from_extended_key_string(ledger, ekey_str):
     xpub6BsnM1W2Y7qLMiuhi7f7dbAwQZ5Cz5gYJCRzTNainXzQXYjFwtuQXHd
     3qfi3t3KJtHxshXezfjft93w4UE7BGMtKwhqEHae3ZA7d823DVrL
 
-    return a PubKey or PrivateKey.
+    return a PublicKey or PrivateKey.
     """
     return _from_extended_key(ledger, Base58.decode_check(ekey_str))

lbry/wallet/claim_proofs.py

@@ -15,18 +15,8 @@ def get_hash_for_outpoint(txhash, nout, height_of_last_takeover):
     )
 
 
-def is_proof_valid(claim, proof, root_hash) -> bool:
-    proof_hash = get_hash_for_outpoint(binascii.unhexlify(claim["txId"])[::-1], claim["n"], claim["lastTakeoverHeight"])
-    for p in proof["pairs"]:
-        if p["odd"]:  # odd = 1 = the missing hash is coming from the right side of the binary merkle trie
-            proof_hash = double_sha256(binascii.unhexlify(p["hash"])[::-1] + proof_hash)
-        else:  # even = 0 = left side
-            proof_hash = double_sha256(proof_hash + binascii.unhexlify(p["hash"])[::-1])
-    return root_hash == binascii.hexlify(proof_hash[::-1]).decode("ascii")
-
-
 # noinspection PyPep8
-def verify_proof_old(proof, root_hash, name):
+def verify_proof(proof, root_hash, name):
     previous_computed_hash = None
     reverse_computed_name = ''
     verified_value = False

lbry/wallet/coinselection.py

@@ -5,7 +5,7 @@ from lbry.wallet.transaction import OutputEffectiveAmountEstimator
 
 MAXIMUM_TRIES = 100000
 
-STRATEGIES = []
+STRATEGIES = ['sqlite']  # sqlite coin chooser is in database.py
 
 
 def strategy(method):
@@ -141,7 +141,7 @@ class CoinSelector:
                     _) -> List[OutputEffectiveAmountEstimator]:
         """ Accumulate UTXOs at random until there is enough to cover the target. """
         target = self.target + self.cost_of_change
-        self.random.shuffle(txos, self.random.random)
+        self.random.shuffle(txos, random=self.random.random)  # pylint: disable=deprecated-argument
         selection = []
         amount = 0
         for coin in txos:

lbry/wallet/constants.py

@@ -2,10 +2,12 @@ NULL_HASH32 = b'\x00'*32
 
 CENT = 1000000
 COIN = 100*CENT
+DUST = 1000
 
 TIMEOUT = 30.0
 
 TXO_TYPES = {
+    "other": 0,
     "stream": 1,
     "channel": 2,
     "support": 3,
@@ -14,9 +16,13 @@ TXO_TYPES = {
     "repost": 6,
 }
 
-CLAIM_TYPES = [
-    TXO_TYPES['stream'],
-    TXO_TYPES['channel'],
-    TXO_TYPES['collection'],
-    TXO_TYPES['repost'],
+CLAIM_TYPE_NAMES = [
+    'stream',
+    'channel',
+    'collection',
+    'repost',
+]
+
+CLAIM_TYPES = [
+    TXO_TYPES[name] for name in CLAIM_TYPE_NAMES
 ]

lbry/wallet/header.py

@@ -1,16 +1,18 @@
+import base64
 import os
 import struct
 import asyncio
-import hashlib
 import logging
+import zlib
+from datetime import date
 
 from io import BytesIO
-from contextlib import asynccontextmanager
-from typing import Optional, Iterator, Tuple
+from typing import Optional, Iterator, Tuple, Callable
 from binascii import hexlify, unhexlify
 
 from lbry.crypto.hash import sha512, double_sha256, ripemd160
-from lbry.wallet.util import ArithUint256
+from lbry.wallet.util import ArithUint256, date_to_julian_day
+from .checkpoints import HASHES
 
 
 log = logging.getLogger(__name__)
@@ -32,25 +34,50 @@ class Headers:
     max_target = 0x0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
     genesis_hash = b'9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463'
     target_timespan = 150
-    checkpoint = (600_000, b'100b33ca3d0b86a48f0d6d6f30458a130ecb89d5affefe4afccb134d5a40f4c2')
+    checkpoints = HASHES
+    first_block_timestamp = 1466646588  # block 1, as 0 is off by a lot
+    timestamp_average_offset = 160.6855883050695  # calculated at 733447
 
     validate_difficulty: bool = True
 
     def __init__(self, path) -> None:
-        if path == ':memory:':
-            self.io = BytesIO()
+        self.io = None
         self.path = path
         self._size: Optional[int] = None
+        self.chunk_getter: Optional[Callable] = None
+        self.known_missing_checkpointed_chunks = set()
+        self.check_chunk_lock = asyncio.Lock()
 
     async def open(self):
+        self.io = BytesIO()
         if self.path != ':memory:':
-            if not os.path.exists(self.path):
-                self.io = open(self.path, 'w+b')
-            else:
-                self.io = open(self.path, 'r+b')
+            def _readit():
+                if os.path.exists(self.path):
+                    with open(self.path, 'r+b') as header_file:
+                        self.io.seek(0)
+                        self.io.write(header_file.read())
+            await asyncio.get_event_loop().run_in_executor(None, _readit)
+        bytes_size = self.io.seek(0, os.SEEK_END)
+        self._size = bytes_size // self.header_size
+        max_checkpointed_height = max(self.checkpoints.keys() or [-1]) + 1000
+        if bytes_size % self.header_size:
+            log.warning("Reader file size doesnt match header size. Repairing, might take a while.")
+            await self.repair()
+        else:
+            # try repairing any incomplete write on tip from previous runs (outside of checkpoints, that are ok)
+            await self.repair(start_height=max_checkpointed_height)
+        await self.ensure_checkpointed_size()
+        await self.get_all_missing_headers()
 
     async def close(self):
-        self.io.close()
+        if self.io is not None:
+            def _close():
+                flags = 'r+b' if os.path.exists(self.path) else 'w+b'
+                with open(self.path, flags) as header_file:
+                    header_file.write(self.io.getbuffer())
+            await asyncio.get_event_loop().run_in_executor(None, _close)
+            self.io.close()
+            self.io = None
 
     @staticmethod
     def serialize(header):
@@ -97,23 +124,92 @@ class Headers:
         return new_target
 
     def __len__(self) -> int:
-        if self._size is None:
-            self._size = self.io.seek(0, os.SEEK_END) // self.header_size
         return self._size
 
     def __bool__(self):
         return True
 
-    def __getitem__(self, height) -> dict:
+    async def get(self, height) -> dict:
         if isinstance(height, slice):
             raise NotImplementedError("Slicing of header chain has not been implemented yet.")
+        try:
+            return self.deserialize(height, await self.get_raw_header(height))
+        except struct.error:
+            raise IndexError(f"failed to get {height}, at {len(self)}")
+
+    def estimated_timestamp(self, height, try_real_headers=True):
+        if height <= 0:
+            return
+        if try_real_headers and self.has_header(height):
+            offset = height * self.header_size
+            return struct.unpack('<I', self.io.getbuffer()[offset + 100: offset + 104])[0]
+        return int(self.first_block_timestamp + (height * self.timestamp_average_offset))
+
+    def estimated_julian_day(self, height):
+        return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height, False)))
+
+    async def get_raw_header(self, height) -> bytes:
+        if self.chunk_getter:
+            await self.ensure_chunk_at(height)
         if not 0 <= height <= self.height:
             raise IndexError(f"{height} is out of bounds, current height: {self.height}")
-        return self.deserialize(height, self.get_raw_header(height))
+        return self._read(height)
 
-    def get_raw_header(self, height) -> bytes:
-        self.io.seek(height * self.header_size, os.SEEK_SET)
-        return self.io.read(self.header_size)
+    def _read(self, height, count=1):
+        offset = height * self.header_size
+        return bytes(self.io.getbuffer()[offset: offset + self.header_size * count])
+
+    def chunk_hash(self, start, count):
+        return self.hash_header(self._read(start, count)).decode()
+
+    async def ensure_checkpointed_size(self):
+        max_checkpointed_height = max(self.checkpoints.keys() or [-1])
+        if self.height < max_checkpointed_height:
+            self._write(max_checkpointed_height, bytes([0] * self.header_size * 1000))
+
+    async def ensure_chunk_at(self, height):
+        async with self.check_chunk_lock:
+            if self.has_header(height):
+                log.debug("has header %s", height)
+                return
+            return await self.fetch_chunk(height)
+
+    async def fetch_chunk(self, height):
+        log.info("on-demand fetching height %s", height)
+        start = (height // 1000) * 1000
+        headers = await self.chunk_getter(start)  # pylint: disable=not-callable
+        chunk = (
+            zlib.decompress(base64.b64decode(headers['base64']), wbits=-15, bufsize=600_000)
+        )
+        chunk_hash = self.hash_header(chunk).decode()
+        if self.checkpoints.get(start) == chunk_hash:
+            self._write(start, chunk)
+            if start in self.known_missing_checkpointed_chunks:
+                self.known_missing_checkpointed_chunks.remove(start)
+            return
+        elif start not in self.checkpoints:
+            return  # todo: fixme
+        raise Exception(
+            f"Checkpoint mismatch at height {start}. Expected {self.checkpoints[start]}, but got {chunk_hash} instead."
+        )
+
+    def has_header(self, height):
+        normalized_height = (height // 1000) * 1000
+        if normalized_height in self.checkpoints:
+            return normalized_height not in self.known_missing_checkpointed_chunks
+
+        empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d'
+        all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b'
+        return self.chunk_hash(height, 1) not in (empty, all_zeroes)
+
+    async def get_all_missing_headers(self):
+        # Heavy operation done in one optimized shot
+        for chunk_height, expected_hash in reversed(list(self.checkpoints.items())):
+            if chunk_height in self.known_missing_checkpointed_chunks:
+                continue
+            if self.chunk_hash(chunk_height, 1000) != expected_hash:
+                self.known_missing_checkpointed_chunks.add(chunk_height)
+        return self.known_missing_checkpointed_chunks
 
     @property
     def height(self) -> int:
@@ -123,9 +219,9 @@ class Headers:
     def bytes_size(self):
         return len(self) * self.header_size
 
-    def hash(self, height=None) -> bytes:
+    async def hash(self, height=None) -> bytes:
         return self.hash_header(
-            self.get_raw_header(height if height is not None else self.height)
+            await self.get_raw_header(height if height is not None else self.height)
         )
 
     @staticmethod
@@ -134,44 +230,18 @@ class Headers:
             return b'0' * 64
         return hexlify(double_sha256(header)[::-1])
 
-    @asynccontextmanager
-    async def checkpointed_connector(self):
-        buf = BytesIO()
-        try:
-            yield buf
-        finally:
-            await asyncio.sleep(0)
-            final_height = len(self) + buf.tell() // self.header_size
-            verifiable_bytes = (self.checkpoint[0] - len(self)) * self.header_size if self.checkpoint else 0
-            if verifiable_bytes > 0 and final_height >= self.checkpoint[0]:
-                buf.seek(0)
-                self.io.seek(0)
-                h = hashlib.sha256()
-                h.update(self.io.read())
-                h.update(buf.read(verifiable_bytes))
-                if h.hexdigest().encode() == self.checkpoint[1]:
-                    buf.seek(0)
-                    self._write(len(self), buf.read(verifiable_bytes))
-                    remaining = buf.read()
-                    buf.seek(0)
-                    buf.write(remaining)
-                    buf.truncate()
-                else:
-                    log.warning("Checkpoint mismatch, connecting headers through slow method.")
-            if buf.tell() > 0:
-                await self.connect(len(self), buf.getvalue())
-
     async def connect(self, start: int, headers: bytes) -> int:
         added = 0
         bail = False
         for height, chunk in self._iterate_chunks(start, headers):
             try:
                 # validate_chunk() is CPU bound and reads previous chunks from file system
-                self.validate_chunk(height, chunk)
+                await self.validate_chunk(height, chunk)
             except InvalidHeader as e:
                 bail = True
                 chunk = chunk[:(height-e.height)*self.header_size]
-            added += self._write(height, chunk) if chunk else 0
+            if chunk:
+                added += self._write(height, chunk)
             if bail:
                 break
         return added
@@ -179,20 +249,21 @@ class Headers:
     def _write(self, height, verified_chunk):
         self.io.seek(height * self.header_size, os.SEEK_SET)
         written = self.io.write(verified_chunk) // self.header_size
-        self.io.truncate()
+        # self.io.truncate()
         # .seek()/.write()/.truncate() might also .flush() when needed
         # the goal here is mainly to ensure we're definitely flush()'ing
         self.io.flush()
-        self._size = self.io.tell() // self.header_size
+        self._size = max(self._size or 0, self.io.tell() // self.header_size)
         return written
 
-    def validate_chunk(self, height, chunk):
+    async def validate_chunk(self, height, chunk):
         previous_hash, previous_header, previous_previous_header = None, None, None
         if height > 0:
-            previous_header = self[height-1]
-            previous_hash = self.hash(height-1)
+            raw = await self.get_raw_header(height-1)
+            previous_header = self.deserialize(height-1, raw)
+            previous_hash = self.hash_header(raw)
         if height > 1:
-            previous_previous_header = self[height-2]
+            previous_previous_header = await self.get(height-2)
         chunk_target = self.get_next_chunk_target(height // 2016 - 1)
         for current_hash, current_header in self._iterate_headers(height, chunk):
             block_target = self.get_next_block_target(chunk_target, previous_previous_header, previous_header)
@@ -231,28 +302,30 @@ class Headers:
                     height, f"insufficient proof of work: {proof_of_work.value} vs target {target.value}"
                 )
 
-    async def repair(self):
+    async def repair(self, start_height=0):
         previous_header_hash = fail = None
         batch_size = 36
-        for start_height in range(0, self.height, batch_size):
-            self.io.seek(self.header_size * start_height)
-            headers = self.io.read(self.header_size*batch_size)
+        for height in range(start_height, self.height, batch_size):
+            headers = self._read(height, batch_size)
             if len(headers) % self.header_size != 0:
                 headers = headers[:(len(headers) // self.header_size) * self.header_size]
-            for header_hash, header in self._iterate_headers(start_height, headers):
+            for header_hash, header in self._iterate_headers(height, headers):
                 height = header['block_height']
-                if height:
+                if previous_header_hash:
                     if header['prev_block_hash'] != previous_header_hash:
                         fail = True
-                else:
+                elif height == 0:
                     if header_hash != self.genesis_hash:
                         fail = True
+                else:
+                    # for sanity and clarity, since it is the only way we can end up here
+                    assert start_height > 0 and height == start_height
                 if fail:
                     log.warning("Header file corrupted at height %s, truncating it.", height - 1)
                     self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET)
                     self.io.truncate()
                     self.io.flush()
-                    self._size = None
+                    self._size = self.io.seek(0, os.SEEK_END) // self.header_size
                     return
                 previous_header_hash = header_hash
 
@@ -276,10 +349,6 @@ class Headers:
             header = headers[start:end]
             yield self.hash_header(header), self.deserialize(height+idx, header)
 
-    @property
-    def claim_trie_root(self):
-        return self[self.height]['claim_trie_root']
-
     @staticmethod
     def header_hash_to_pow_hash(header_hash: bytes):
         header_hash_bytes = unhexlify(header_hash)[::-1]
@@ -295,3 +364,4 @@ class UnvalidatedHeaders(Headers):
     validate_difficulty = False
     max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
     genesis_hash = b'6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556'
+    checkpoints = {}

lbry/wallet/manager.py

@@ -3,20 +3,21 @@ import json
 import typing
 import logging
 import asyncio
+
 from binascii import unhexlify
 from decimal import Decimal
 from typing import List, Type, MutableSequence, MutableMapping, Optional
 
-from lbry.error import KeyFeeAboveMaxAllowedError
-from lbry.conf import Config
+from lbry.error import KeyFeeAboveMaxAllowedError, WalletNotLoadedError
+from lbry.conf import Config, NOT_SET
 
-from .dewies import dewies_to_lbc
-from .account import Account
-from .ledger import Ledger, LedgerRegistry
-from .transaction import Transaction, Output
-from .database import Database
-from .wallet import Wallet, WalletStorage, ENCRYPT_ON_DISK
-from .rpc.jsonrpc import CodeMessageError
+from lbry.wallet.dewies import dewies_to_lbc
+from lbry.wallet.account import Account
+from lbry.wallet.ledger import Ledger, LedgerRegistry
+from lbry.wallet.transaction import Transaction, Output
+from lbry.wallet.database import Database
+from lbry.wallet.wallet import Wallet, WalletStorage, ENCRYPT_ON_DISK
+from lbry.wallet.rpc.jsonrpc import CodeMessageError
 
 if typing.TYPE_CHECKING:
     from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
@@ -95,7 +96,7 @@ class WalletManager:
         for wallet in self.wallets:
             if wallet.id == wallet_id:
                 return wallet
-        raise ValueError(f"Couldn't find wallet: {wallet_id}.")
+        raise WalletNotLoadedError(wallet_id)
 
     @staticmethod
     def get_balance(wallet):
@@ -182,9 +183,17 @@ class WalletManager:
 
         ledger_config = {
             'auto_connect': True,
+            'explicit_servers': [],
+            'hub_timeout': config.hub_timeout,
             'default_servers': config.lbryum_servers,
+            'known_hubs': config.known_hubs,
+            'jurisdiction': config.jurisdiction,
+            'concurrent_hub_requests': config.concurrent_hub_requests,
             'data_path': config.wallet_dir,
+            'tx_cache_size': config.transaction_cache_size
         }
+        if 'LBRY_FEE_PER_NAME_CHAR' in os.environ:
+            ledger_config['fee_per_name_char'] = int(os.environ.get('LBRY_FEE_PER_NAME_CHAR'))
 
         wallets_directory = os.path.join(config.wallet_dir, 'wallets')
         if not os.path.exists(wallets_directory):
@@ -194,6 +203,10 @@ class WalletManager:
             os.path.join(wallets_directory, 'default_wallet')
         )
 
+        if Config.lbryum_servers.is_set_to_default(config):
+            with config.update_config() as c:
+                c.lbryum_servers = NOT_SET
+
         manager = cls.from_config({
             'ledgers': {ledger_id: ledger_config},
             'wallets': [
@@ -224,9 +237,16 @@ class WalletManager:
     async def reset(self):
         self.ledger.config = {
             'auto_connect': True,
-            'default_servers': self.config.lbryum_servers,
+            'explicit_servers': [],
+            'default_servers': Config.lbryum_servers.default,
+            'known_hubs': self.config.known_hubs,
+            'jurisdiction': self.config.jurisdiction,
+            'hub_timeout': self.config.hub_timeout,
+            'concurrent_hub_requests': self.config.concurrent_hub_requests,
             'data_path': self.config.wallet_dir,
         }
+        if Config.lbryum_servers.is_set(self.config):
+            self.ledger.config['explicit_servers'] = self.config.lbryum_servers
         await self.ledger.stop()
         await self.ledger.start()
 
@@ -248,26 +268,28 @@ class WalletManager:
             log.warning("Failed to migrate %s receiving addresses!",
                         len(set(receiving_addresses).difference(set(migrated_receiving))))
 
-    def get_best_blockhash(self):
+    async def get_best_blockhash(self):
         if len(self.ledger.headers) <= 0:
             return self.ledger.genesis_hash
-        return self.ledger.headers.hash(self.ledger.headers.height).decode()
+        return (await self.ledger.headers.hash(self.ledger.headers.height)).decode()
 
     def get_unused_address(self):
         return self.default_account.receiving.get_or_create_usable_address()
 
-    async def get_transaction(self, txid):
+    async def get_transaction(self, txid: str):
         tx = await self.db.get_transaction(txid=txid)
-        if not tx:
-            try:
-                raw = await self.ledger.network.get_transaction(txid)
-                height = await self.ledger.network.get_transaction_height(txid)
-            except CodeMessageError as e:
-                if 'No such mempool or blockchain transaction.' in e.message:
-                    return {'success': False, 'code': 404, 'message': 'transaction not found'}
-                return {'success': False, 'code': e.code, 'message': e.message}
-            tx = Transaction(unhexlify(raw))
-            await self.ledger.maybe_verify_transaction(tx, height)
+        if tx:
+            return tx
+        try:
+            raw, merkle = await self.ledger.network.get_transaction_and_merkle(txid)
+        except CodeMessageError as e:
+            if 'No such mempool or blockchain transaction.' in e.message:
+                return {'success': False, 'code': 404, 'message': 'transaction not found'}
+            return {'success': False, 'code': e.code, 'message': e.message}
+        height = merkle.get('block_height')
+        tx = Transaction(unhexlify(raw), height=height)
+        if height and height > 0:
+            await self.ledger.maybe_verify_transaction(tx, height, merkle)
         return tx
 
     async def create_purchase_transaction(
@@ -295,10 +317,4 @@ class WalletManager:
         )
 
     async def broadcast_or_release(self, tx, blocking=False):
-        try:
-            await self.ledger.broadcast(tx)
-            if blocking:
-                await self.ledger.wait(tx, timeout=None)
-        except:
-            await self.ledger.release_tx(tx)
-            raise
+        await self.ledger.broadcast_or_release(tx, blocking=blocking)