initial

.gitignore

@@ -0,0 +1,13 @@
+/.idea
+/.DS_Store
+/build
+/dist
+/.tox
+/.coverage*
+/venv
+scribe.egg-info
+__pycache__
+/tests/.coverage.*
+/.vscode
+/.gitignore
+.*pyc

README.md

@@ -0,0 +1,38 @@
+Scribe maintains a rocksdb database containing the [LBRY blockchain](https://github.com/lbryio/lbrycrd) and provides an interface for python based services that utilize the blockchain data in an ongoing manner. Scribe includes implementations of this interface to provide an electrum server for thin-wallet clients such as lbry-sdk and to maintain an elasticsearch database of claims in the LBRY blockchain.
+
+ * Uses Python 3.7-3.8
+ * Protobuf schema for encoding and decoding metadata stored on the blockchain ([scribe.schema](https://github.com/lbryio/scribe/tree/master/scribe/schema)).
+ * Blockchain processor that maintains an up to date rocksdb database ([scribe.blockchain](https://github.com/lbryio/scribe/tree/master/scribe/blockchain))
+ * Rocksdb based database containing the blockchain data ([scribe.db]((https://github.com/lbryio/scribe/tree/master/scribe/db)))
+ * Interface for python services to implement in order for them maintain a read only view of the blockchain data ([scribe.readers.interface]((https://github.com/lbryio/scribe/tree/master/scribe/db)))
+ * Electrum based server for thin-wallet clients like lbry-sdk ([scribe.readers.hub_server]((https://github.com/lbryio/scribe/tree/master/scribe/db)))
+ * Elasticsearch sync utility to index all the claim metadata in the blockchain into an easily searchable form ([scribe.readers.elastic_sync]((https://github.com/lbryio/scribe/tree/master/scribe/db)))
+
+
+## Installation
+
+Our [releases page](https://github.com/lbryio/scribe/releases) contains pre-built binaries of the latest release, pre-releases, and past releases for macOS and Debian-based Linux.
+Prebuilt [docker images](https://hub.docker.com/r/lbry/scribe/latest-release) are also available.
+
+## Usage
+
+
+## Running from source
+
+Installing from source is also relatively painless. Full instructions are in [INSTALL.md](INSTALL.md)
+
+## Contributing
+
+Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.tech/contribute) link.
+
+## License
+
+This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
+
+## Security
+
+We take security seriously. Please contact security@lbry.com regarding any security issues. [Our PGP key is here](https://lbry.com/faq/pgp-key) if you need it.
+
+## Contact
+
+The primary contact for this project is [@jackrobison](mailto:jackrobison@lbry.com).

scribe/__init__.py

@@ -0,0 +1,5 @@
+__version__ = "0.107.0"
+
+PROMETHEUS_NAMESPACE = "scribe"
+PROTOCOL_MIN = (0, 54, 0)
+PROTOCOL_MAX = (0, 199, 0)

scribe/base58.py

@@ -0,0 +1,111 @@
+import hashlib
+
+
+def sha256(x):
+    """ Simple wrapper of hashlib sha256. """
+    return hashlib.sha256(x).digest()
+
+
+def double_sha256(x):
+    """ SHA-256 of SHA-256, as used extensively in bitcoin. """
+    return sha256(sha256(x))
+
+
+class Base58Error(Exception):
+    """ Exception used for Base58 errors. """
+
+
+_CHARS = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
+
+
+def _iter_encode(be_bytes: bytes):
+    value = int.from_bytes(be_bytes, 'big')
+    while value:
+        value, mod = divmod(value, 58)
+        yield _CHARS[mod]
+    for byte in be_bytes:
+        if byte != 0:
+            break
+        yield '1'
+
+
+def b58_encode(be_bytes: bytes):
+    return ''.join(_iter_encode(be_bytes))[::-1]
+
+
+class Base58:
+    """ Class providing base 58 functionality. """
+
+    chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
+    assert len(chars) == 58
+    char_map = {c: n for n, c in enumerate(chars)}
+
+    @classmethod
+    def char_value(cls, c):
+        val = cls.char_map.get(c)
+        if val is None:
+            raise Base58Error(f'invalid base 58 character "{c}"')
+        return val
+
+    @classmethod
+    def decode(cls, txt):
+        """ Decodes txt into a big-endian bytearray. """
+        if isinstance(txt, memoryview):
+            txt = str(txt)
+
+        if isinstance(txt, bytes):
+            txt = txt.decode()
+
+        if not isinstance(txt, str):
+            raise TypeError(f'a string is required, got {type(txt).__name__}')
+
+        if not txt:
+            raise Base58Error('string cannot be empty')
+
+        value = 0
+        for c in txt:
+            value = value * 58 + cls.char_value(c)
+
+        result = value.to_bytes((value.bit_length() + 7) // 8, 'big')
+
+        # Prepend leading zero bytes if necessary
+        count = 0
+        for c in txt:
+            if c != '1':
+                break
+            count += 1
+        if count:
+            result = bytes((0,)) * count + result
+
+        return result
+
+    @classmethod
+    def _iter_encode(cls, be_bytes: bytes):
+        value = int.from_bytes(be_bytes, 'big')
+        while value:
+            value, mod = divmod(value, 58)
+            yield cls.chars[mod]
+        for byte in be_bytes:
+            if byte != 0:
+                break
+            yield '1'
+
+    @classmethod
+    def encode(cls, be_bytes):
+        return ''.join(cls._iter_encode(be_bytes))[::-1]
+
+    @classmethod
+    def decode_check(cls, txt, hash_fn=double_sha256):
+        """ Decodes a Base58Check-encoded string to a payload. The version prefixes it. """
+        be_bytes = cls.decode(txt)
+        result, check = be_bytes[:-4], be_bytes[-4:]
+        if check != hash_fn(result)[:4]:
+            raise Base58Error(f'invalid base 58 checksum for {txt}')
+        return result
+
+    @classmethod
+    def encode_check(cls, payload, hash_fn=double_sha256):
+        """ Encodes a payload bytearray (which includes the version byte(s))
+            into a Base58Check string."""
+        be_bytes = payload + hash_fn(payload)[:4]
+        return b58_encode(be_bytes)

scribe/bip32.py

@@ -0,0 +1,373 @@
+import sys
+import functools
+import hashlib
+import hmac
+from asn1crypto.keys import PrivateKeyInfo, ECPrivateKey
+from coincurve import PublicKey as cPublicKey, PrivateKey as cPrivateKey
+from coincurve.utils import (
+    pem_to_der, lib as libsecp256k1, ffi as libsecp256k1_ffi
+)
+from coincurve.ecdsa import CDATA_SIG_LENGTH
+from scribe.base58 import Base58
+
+
+if (sys.version_info.major, sys.version_info.minor) > (3, 7):
+    cachedproperty = functools.cached_property
+else:
+    cachedproperty = property
+
+
+def hmac_sha512(key, msg):
+    """ Use SHA-512 to provide an HMAC. """
+    return hmac.new(key, msg, hashlib.sha512).digest()
+
+
+def sha256(x):
+    """ Simple wrapper of hashlib sha256. """
+    return hashlib.sha256(x).digest()
+
+
+def hash160(x):
+    """ RIPEMD-160 of SHA-256.
+        Used to make bitcoin addresses from pubkeys. """
+    return ripemd160(sha256(x))
+
+
+def ripemd160(x):
+    """ Simple wrapper of hashlib ripemd160. """
+    h = hashlib.new('ripemd160')
+    h.update(x)
+    return h.digest()
+
+
+def double_sha256(x):
+    """ SHA-256 of SHA-256, as used extensively in bitcoin. """
+    return sha256(sha256(x))
+
+
+class KeyPath:
+    RECEIVE = 0
+    CHANGE = 1
+    CHANNEL = 2
+
+
+class DerivationError(Exception):
+    """ Raised when an invalid derivation occurs. """
+
+
+class _KeyBase:
+    """ A BIP32 Key, public or private. """
+
+    def __init__(self, ledger, chain_code, n, depth, parent):
+        if not isinstance(chain_code, (bytes, bytearray)):
+            raise TypeError('chain code must be raw bytes')
+        if len(chain_code) != 32:
+            raise ValueError('invalid chain code')
+        if not 0 <= n < 1 << 32:
+            raise ValueError('invalid child number')
+        if not 0 <= depth < 256:
+            raise ValueError('invalid depth')
+        if parent is not None:
+            if not isinstance(parent, type(self)):
+                raise TypeError('parent key has bad type')
+        self.ledger = ledger
+        self.chain_code = chain_code
+        self.n = n
+        self.depth = depth
+        self.parent = parent
+
+    def _hmac_sha512(self, msg):
+        """ Use SHA-512 to provide an HMAC, returned as a pair of 32-byte objects. """
+        hmac = hmac_sha512(self.chain_code, msg)
+        return hmac[:32], hmac[32:]
+
+    def _extended_key(self, ver_bytes, raw_serkey):
+        """ Return the 78-byte extended key given prefix version bytes and serialized key bytes. """
+        if not isinstance(ver_bytes, (bytes, bytearray)):
+            raise TypeError('ver_bytes must be raw bytes')
+        if len(ver_bytes) != 4:
+            raise ValueError('ver_bytes must have length 4')
+        if not isinstance(raw_serkey, (bytes, bytearray)):
+            raise TypeError('raw_serkey must be raw bytes')
+        if len(raw_serkey) != 33:
+            raise ValueError('raw_serkey must have length 33')
+
+        return (
+            ver_bytes + bytes((self.depth,))
+            + self.parent_fingerprint() + self.n.to_bytes(4, 'big')
+            + self.chain_code + raw_serkey
+        )
+
+    def identifier(self):
+        raise NotImplementedError
+
+    def extended_key(self):
+        raise NotImplementedError
+
+    def fingerprint(self):
+        """ Return the key's fingerprint as 4 bytes. """
+        return self.identifier()[:4]
+
+    def parent_fingerprint(self):
+        """ Return the parent key's fingerprint as 4 bytes. """
+        return self.parent.fingerprint() if self.parent else bytes((0,)*4)
+
+    def extended_key_string(self):
+        """ Return an extended key as a base58 string. """
+        return Base58.encode_check(self.extended_key())
+
+
+class PublicKey(_KeyBase):
+    """ A BIP32 public key. """
+
+    def __init__(self, ledger, pubkey, chain_code, n, depth, parent=None):
+        super().__init__(ledger, chain_code, n, depth, parent)
+        if isinstance(pubkey, cPublicKey):
+            self.verifying_key = pubkey
+        else:
+            self.verifying_key = self._verifying_key_from_pubkey(pubkey)
+
+    @classmethod
+    def from_compressed(cls, public_key_bytes, ledger=None) -> 'PublicKey':
+        return cls(ledger, public_key_bytes, bytes((0,)*32), 0, 0)
+
+    @classmethod
+    def _verifying_key_from_pubkey(cls, pubkey):
+        """ Converts a 33-byte compressed pubkey into an coincurve.PublicKey object. """
+        if not isinstance(pubkey, (bytes, bytearray)):
+            raise TypeError('pubkey must be raw bytes')
+        if len(pubkey) != 33:
+            raise ValueError('pubkey must be 33 bytes')
+        if pubkey[0] not in (2, 3):
+            raise ValueError('invalid pubkey prefix byte')
+        return cPublicKey(pubkey)
+
+    @cachedproperty
+    def pubkey_bytes(self):
+        """ Return the compressed public key as 33 bytes. """
+        return self.verifying_key.format(True)
+
+    @cachedproperty
+    def address(self):
+        """ The public key as a P2PKH address. """
+        return self.ledger.public_key_to_address(self.pubkey_bytes)
+
+    def ec_point(self):
+        return self.verifying_key.point()
+
+    def child(self, n: int) -> 'PublicKey':
+        """ Return the derived child extended pubkey at index N. """
+        if not 0 <= n < (1 << 31):
+            raise ValueError('invalid BIP32 public key child number')
+
+        msg = self.pubkey_bytes + n.to_bytes(4, 'big')
+        L_b, R_b = self._hmac_sha512(msg)  # pylint: disable=invalid-name
+        derived_key = self.verifying_key.add(L_b)
+        return PublicKey(self.ledger, derived_key, R_b, n, self.depth + 1, self)
+
+    def identifier(self):
+        """ Return the key's identifier as 20 bytes. """
+        return hash160(self.pubkey_bytes)
+
+    def extended_key(self):
+        """ Return a raw extended public key. """
+        return self._extended_key(
+            self.ledger.extended_public_key_prefix,
+            self.pubkey_bytes
+        )
+
+    def verify(self, signature, digest) -> bool:
+        """ Verify that a signature is valid for a 32 byte digest. """
+
+        if len(signature) != 64:
+            raise ValueError('Signature must be 64 bytes long.')
+
+        if len(digest) != 32:
+            raise ValueError('Digest must be 32 bytes long.')
+
+        key = self.verifying_key
+
+        raw_signature = libsecp256k1_ffi.new('secp256k1_ecdsa_signature *')
+
+        parsed = libsecp256k1.secp256k1_ecdsa_signature_parse_compact(
+            key.context.ctx, raw_signature, signature
+        )
+        assert parsed == 1
+
+        normalized_signature = libsecp256k1_ffi.new('secp256k1_ecdsa_signature *')
+
+        libsecp256k1.secp256k1_ecdsa_signature_normalize(
+            key.context.ctx, normalized_signature, raw_signature
+        )
+
+        verified = libsecp256k1.secp256k1_ecdsa_verify(
+            key.context.ctx, normalized_signature, digest, key.public_key
+        )
+
+        return bool(verified)
+
+
+class PrivateKey(_KeyBase):
+    """A BIP32 private key."""
+
+    HARDENED = 1 << 31
+
+    def __init__(self, ledger, privkey, chain_code, n, depth, parent=None):
+        super().__init__(ledger, chain_code, n, depth, parent)
+        if isinstance(privkey, cPrivateKey):
+            self.signing_key = privkey
+        else:
+            self.signing_key = self._signing_key_from_privkey(privkey)
+
+    @classmethod
+    def _signing_key_from_privkey(cls, private_key):
+        """ Converts a 32-byte private key into an coincurve.PrivateKey object. """
+        return cPrivateKey.from_int(PrivateKey._private_key_secret_exponent(private_key))
+
+    @classmethod
+    def _private_key_secret_exponent(cls, private_key):
+        """ Return the private key as a secret exponent if it is a valid private key. """
+        if not isinstance(private_key, (bytes, bytearray)):
+            raise TypeError('private key must be raw bytes')
+        if len(private_key) != 32:
+            raise ValueError('private key must be 32 bytes')
+        return int.from_bytes(private_key, 'big')
+
+    @classmethod
+    def from_seed(cls, ledger, seed) -> 'PrivateKey':
+        # This hard-coded message string seems to be coin-independent...
+        hmac = hmac_sha512(b'Bitcoin seed', seed)
+        privkey, chain_code = hmac[:32], hmac[32:]
+        return cls(ledger, privkey, chain_code, 0, 0)
+
+    @classmethod
+    def from_pem(cls, ledger, pem) -> 'PrivateKey':
+        der = pem_to_der(pem.encode())
+        try:
+            key_int = ECPrivateKey.load(der).native['private_key']
+        except ValueError:
+            key_int = PrivateKeyInfo.load(der).native['private_key']['private_key']
+        private_key = cPrivateKey.from_int(key_int)
+        return cls(ledger, private_key, bytes((0,)*32), 0, 0)
+
+    @cachedproperty
+    def private_key_bytes(self):
+        """ Return the serialized private key (no leading zero byte). """
+        return self.signing_key.secret
+
+    @cachedproperty
+    def public_key(self) -> PublicKey:
+        """ Return the corresponding extended public key. """
+        verifying_key = self.signing_key.public_key
+        parent_pubkey = self.parent.public_key if self.parent else None
+        return PublicKey(
+            self.ledger, verifying_key, self.chain_code,
+            self.n, self.depth, parent_pubkey
+        )
+
+    def ec_point(self):
+        return self.public_key.ec_point()
+
+    def secret_exponent(self):
+        """ Return the private key as a secret exponent. """
+        return self.signing_key.to_int()
+
+    def wif(self):
+        """ Return the private key encoded in Wallet Import Format. """
+        return self.ledger.private_key_to_wif(self.private_key_bytes)
+
+    @property
+    def address(self):
+        """ The public key as a P2PKH address. """
+        return self.public_key.address
+
+    def child(self, n) -> 'PrivateKey':
+        """ Return the derived child extended private key at index N."""
+        if not 0 <= n < (1 << 32):
+            raise ValueError('invalid BIP32 private key child number')
+
+        if n >= self.HARDENED:
+            serkey = b'\0' + self.private_key_bytes
+        else:
+            serkey = self.public_key.pubkey_bytes
+
+        msg = serkey + n.to_bytes(4, 'big')
+        L_b, R_b = self._hmac_sha512(msg)  # pylint: disable=invalid-name
+        derived_key = self.signing_key.add(L_b)
+        return PrivateKey(self.ledger, derived_key, R_b, n, self.depth + 1, self)
+
+    def sign(self, data):
+        """ Produce a signature for piece of data by double hashing it and signing the hash. """
+        return self.signing_key.sign(data, hasher=double_sha256)
+
+    def sign_compact(self, digest):
+        """ Produce a compact signature. """
+        key = self.signing_key
+
+        signature = libsecp256k1_ffi.new('secp256k1_ecdsa_signature *')
+        signed = libsecp256k1.secp256k1_ecdsa_sign(
+            key.context.ctx, signature, digest, key.secret,
+            libsecp256k1_ffi.NULL, libsecp256k1_ffi.NULL
+        )
+
+        if not signed:
+            raise ValueError('The private key was invalid.')
+
+        serialized = libsecp256k1_ffi.new('unsigned char[%d]' % CDATA_SIG_LENGTH)
+        compacted = libsecp256k1.secp256k1_ecdsa_signature_serialize_compact(
+            key.context.ctx, serialized, signature
+        )
+        if compacted != 1:
+            raise ValueError('The signature could not be compacted.')
+
+        return bytes(libsecp256k1_ffi.buffer(serialized, CDATA_SIG_LENGTH))
+
+    def identifier(self):
+        """Return the key's identifier as 20 bytes."""
+        return self.public_key.identifier()
+
+    def extended_key(self):
+        """Return a raw extended private key."""
+        return self._extended_key(
+            self.ledger.extended_private_key_prefix,
+            b'\0' + self.private_key_bytes
+        )
+
+    def to_pem(self):
+        return self.signing_key.to_pem()
+
+
+def _from_extended_key(ledger, ekey):
+    """Return a PublicKey or PrivateKey from an extended key raw bytes."""
+    if not isinstance(ekey, (bytes, bytearray)):
+        raise TypeError('extended key must be raw bytes')
+    if len(ekey) != 78:
+        raise ValueError('extended key must have length 78')
+
+    depth = ekey[4]
+    n = int.from_bytes(ekey[9:13], 'big')
+    chain_code = ekey[13:45]
+
+    if ekey[:4] == ledger.extended_public_key_prefix:
+        pubkey = ekey[45:]
+        key = PublicKey(ledger, pubkey, chain_code, n, depth)
+    elif ekey[:4] == ledger.extended_private_key_prefix:
+        if ekey[45] != 0:
+            raise ValueError('invalid extended private key prefix byte')
+        privkey = ekey[46:]
+        key = PrivateKey(ledger, privkey, chain_code, n, depth)
+    else:
+        raise ValueError('version bytes unrecognised')
+
+    return key
+
+
+def from_extended_key_string(ledger, ekey_str):
+    """Given an extended key string, such as
+
+    xpub6BsnM1W2Y7qLMiuhi7f7dbAwQZ5Cz5gYJCRzTNainXzQXYjFwtuQXHd
+    3qfi3t3KJtHxshXezfjft93w4UE7BGMtKwhqEHae3ZA7d823DVrL
+
+    return a PublicKey or PrivateKey.
+    """
+    return _from_extended_key(ledger, Base58.decode_check(ekey_str))

scribe/blockchain/__init__.py

@@ -0,0 +1 @@
+from .network import LBCTestNet, LBCRegTest, LBCMainNet

scribe/blockchain/daemon.py

@@ -0,0 +1,328 @@
+import asyncio
+import itertools
+import json
+import time
+import logging
+from functools import wraps
+
+import aiohttp
+from prometheus_client import Gauge, Histogram
+from scribe import PROMETHEUS_NAMESPACE
+from scribe.common import LRUCacheWithMetrics, RPCError, DaemonError, WarmingUpError, WorkQueueFullError
+
+
+log = logging.getLogger(__name__)
+
+
+NAMESPACE = f"{PROMETHEUS_NAMESPACE}_blockchain"
+METHOD_NOT_FOUND = -32601
+
+
+def handles_errors(decorated_function):
+    @wraps(decorated_function)
+    async def wrapper(*args, **kwargs):
+        try:
+            return await decorated_function(*args, **kwargs)
+        except DaemonError as daemon_error:
+            raise RPCError(1, daemon_error.args[0])
+    return wrapper
+
+
+class LBCDaemon:
+    """Handles connections to a daemon at the given URL."""
+
+    WARMING_UP = -28
+    id_counter = itertools.count()
+
+    lbrycrd_request_time_metric = Histogram(
+        "lbrycrd_request", "lbrycrd requests count", namespace=NAMESPACE, labelnames=("method",)
+    )
+    lbrycrd_pending_count_metric = Gauge(
+        "lbrycrd_pending_count", "Number of lbrycrd rpcs that are in flight", namespace=NAMESPACE,
+        labelnames=("method",)
+    )
+
+    def __init__(self, coin, url, max_workqueue=10, init_retry=0.25,
+                 max_retry=4.0):
+        self.coin = coin
+        self.logger = logging.getLogger(__name__)
+        self.set_url(url)
+        # Limit concurrent RPC calls to this number.
+        # See DEFAULT_HTTP_WORKQUEUE in bitcoind, which is typically 16
+        self.workqueue_semaphore = asyncio.Semaphore(value=max_workqueue)
+        self.init_retry = init_retry
+        self.max_retry = max_retry
+        self._height = None
+        self.available_rpcs = {}
+        self.connector = aiohttp.TCPConnector(ssl=False)
+        self._block_hash_cache = LRUCacheWithMetrics(100000)
+        self._block_cache = LRUCacheWithMetrics(2 ** 13, metric_name='block', namespace=NAMESPACE)
+
+    async def close(self):
+        if self.connector:
+            await self.connector.close()
+            self.connector = None
+
+    def set_url(self, url):
+        """Set the URLS to the given list, and switch to the first one."""
+        urls = url.split(',')
+        urls = [self.coin.sanitize_url(url) for url in urls]
+        for n, url in enumerate(urls):
+            status = '' if n else ' (current)'
+            logged_url = self.logged_url(url)
+            self.logger.info(f'daemon #{n + 1} at {logged_url}{status}')
+        self.url_index = 0
+        self.urls = urls
+
+    def current_url(self):
+        """Returns the current daemon URL."""
+        return self.urls[self.url_index]
+
+    def logged_url(self, url=None):
+        """The host and port part, for logging."""
+        url = url or self.current_url()
+        return url[url.rindex('@') + 1:]
+
+    def failover(self):
+        """Call to fail-over to the next daemon URL.
+
+        Returns False if there is only one, otherwise True.
+        """
+        if len(self.urls) > 1:
+            self.url_index = (self.url_index + 1) % len(self.urls)
+            self.logger.info(f'failing over to {self.logged_url()}')
+            return True
+        return False
+
+    def client_session(self):
+        """An aiohttp client session."""
+        return aiohttp.ClientSession(connector=self.connector, connector_owner=False)
+
+    async def _send_data(self, data):
+        if not self.connector:
+            raise asyncio.CancelledError('Tried to send request during shutdown.')
+        async with self.workqueue_semaphore:
+            async with self.client_session() as session:
+                async with session.post(self.current_url(), data=data) as resp:
+                    kind = resp.headers.get('Content-Type', None)
+                    if kind == 'application/json':
+                        return await resp.json()
+                    # bitcoind's HTTP protocol "handling" is a bad joke
+                    text = await resp.text()
+                    if 'Work queue depth exceeded' in text:
+                        raise WorkQueueFullError
+                    text = text.strip() or resp.reason
+                    self.logger.error(text)
+                    raise DaemonError(text)
+
+    async def _send(self, payload, processor):
+        """Send a payload to be converted to JSON.
+
+        Handles temporary connection issues.  Daemon response errors
+        are raise through DaemonError.
+        """
+
+        def log_error(error):
+            nonlocal last_error_log, retry
+            now = time.time()
+            if now - last_error_log > 60:
+                last_error_log = now
+                self.logger.error(f'{error}  Retrying occasionally...')
+            if retry == self.max_retry and self.failover():
+                retry = 0
+
+        on_good_message = None
+        last_error_log = 0
+        data = json.dumps(payload)
+        retry = self.init_retry
+        methods = tuple(
+            [payload['method']] if isinstance(payload, dict) else [request['method'] for request in payload]
+        )
+        while True:
+            try:
+                for method in methods:
+                    self.lbrycrd_pending_count_metric.labels(method=method).inc()
+                result = await self._send_data(data)
+                result = processor(result)
+                if on_good_message:
+                    self.logger.info(on_good_message)
+                return result
+            except asyncio.TimeoutError:
+                log_error('timeout error.')
+            except aiohttp.ServerDisconnectedError:
+                log_error('disconnected.')
+                on_good_message = 'connection restored'
+            except aiohttp.ClientConnectionError:
+                log_error('connection problem - is your daemon running?')
+                on_good_message = 'connection restored'
+            except aiohttp.ClientError as e:
+                log_error(f'daemon error: {e}')
+                on_good_message = 'running normally'
+            except WarmingUpError:
+                log_error('starting up checking blocks.')
+                on_good_message = 'running normally'
+            except WorkQueueFullError:
+                log_error('work queue full.')
+                on_good_message = 'running normally'
+            finally:
+                for method in methods:
+                    self.lbrycrd_pending_count_metric.labels(method=method).dec()
+            await asyncio.sleep(retry)
+            retry = max(min(self.max_retry, retry * 2), self.init_retry)
+
+    async def _send_single(self, method, params=None):
+        """Send a single request to the daemon."""
+
+        start = time.perf_counter()
+
+        def processor(result):
+            err = result['error']
+            if not err:
+                return result['result']
+            if err.get('code') == self.WARMING_UP:
+                raise WarmingUpError
+            raise DaemonError(err)
+
+        payload = {'method': method, 'id': next(self.id_counter)}
+        if params:
+            payload['params'] = params
+        result = await self._send(payload, processor)
+        self.lbrycrd_request_time_metric.labels(method=method).observe(time.perf_counter() - start)
+        return result
+
+    async def _send_vector(self, method, params_iterable, replace_errs=False):
+        """Send several requests of the same method.
+
+        The result will be an array of the same length as params_iterable.
+        If replace_errs is true, any item with an error is returned as None,
+        otherwise an exception is raised."""
+
+        start = time.perf_counter()
+
+        def processor(result):
+            errs = [item['error'] for item in result if item['error']]
+            if any(err.get('code') == self.WARMING_UP for err in errs):
+                raise WarmingUpError
+            if not errs or replace_errs:
+                return [item['result'] for item in result]
+            raise DaemonError(errs)
+
+        payload = [{'method': method, 'params': p, 'id': next(self.id_counter)}
+                   for p in params_iterable]
+        result = []
+        if payload:
+            result = await self._send(payload, processor)
+        self.lbrycrd_request_time_metric.labels(method=method).observe(time.perf_counter() - start)
+        return result
+
+    async def _is_rpc_available(self, method):
+        """Return whether given RPC method is available in the daemon.
+
+        Results are cached and the daemon will generally not be queried with
+        the same method more than once."""
+        available = self.available_rpcs.get(method)
+        if available is None:
+            available = True
+            try:
+                await self._send_single(method)
+            except DaemonError as e:
+                err = e.args[0]
+                error_code = err.get("code")
+                available = error_code != METHOD_NOT_FOUND
+            self.available_rpcs[method] = available
+        return available
+
+    async def block_hex_hashes(self, first, count):
+        """Return the hex hashes of count block starting at height first."""
+        if first + count < (self.cached_height() or 0) - 200:
+            return await self._cached_block_hex_hashes(first, count)
+        params_iterable = ((h, ) for h in range(first, first + count))
+        return await self._send_vector('getblockhash', params_iterable)
+
+    async def _cached_block_hex_hashes(self, first, count):
+        """Return the hex hashes of count block starting at height first."""
+        cached = self._block_hash_cache.get((first, count))
+        if cached:
+            return cached
+        params_iterable = ((h, ) for h in range(first, first + count))
+        self._block_hash_cache[(first, count)] = await self._send_vector('getblockhash', params_iterable)
+        return self._block_hash_cache[(first, count)]
+
+    async def deserialised_block(self, hex_hash):
+        """Return the deserialised block with the given hex hash."""
+        if hex_hash not in self._block_cache:
+            block = await self._send_single('getblock', (hex_hash, 1))
+            self._block_cache[hex_hash] = block
+            return block
+        return self._block_cache[hex_hash]
+
+    async def raw_blocks(self, hex_hashes):
+        """Return the raw binary blocks with the given hex hashes."""
+        params_iterable = ((h, 0) for h in hex_hashes)
+        blocks = await self._send_vector('getblock', params_iterable)
+        # Convert hex string to bytes
+        return [bytes.fromhex(block) for block in blocks]
+
+    async def mempool_hashes(self):
+        """Update our record of the daemon's mempool hashes."""
+        return await self._send_single('getrawmempool')
+
+    async def estimatefee(self, block_count):
+        """Return the fee estimate for the block count.  Units are whole
+        currency units per KB, e.g. 0.00000995, or -1 if no estimate
+        is available.
+        """
+        args = (block_count, )
+        if await self._is_rpc_available('estimatesmartfee'):
+            estimate = await self._send_single('estimatesmartfee', args)
+            return estimate.get('feerate', -1)
+        return await self._send_single('estimatefee', args)
+
+    async def getnetworkinfo(self):
+        """Return the result of the 'getnetworkinfo' RPC call."""
+        return await self._send_single('getnetworkinfo')
+
+    async def relayfee(self):
+        """The minimum fee a low-priority tx must pay in order to be accepted
+        to the daemon's memory pool."""
+        network_info = await self.getnetworkinfo()
+        return network_info['relayfee']
+
+    async def getrawtransactions(self, hex_hashes, replace_errs=True):
+        """Return the serialized raw transactions with the given hashes.
+
+        Replaces errors with None by default."""
+        params_iterable = ((hex_hash, 0) for hex_hash in hex_hashes)
+        txs = await self._send_vector('getrawtransaction', params_iterable,
+                                      replace_errs=replace_errs)
+        # Convert hex strings to bytes
+        return [bytes.fromhex(tx) if tx else None for tx in txs]
+
+    async def broadcast_transaction(self, raw_tx):
+        """Broadcast a transaction to the network."""
+        return await self._send_single('sendrawtransaction', (raw_tx, ))
+
+    async def height(self):
+        """Query the daemon for its current height."""
+        self._height = await self._send_single('getblockcount')
+        return self._height
+
+    def cached_height(self):
+        """Return the cached daemon height.
+
+        If the daemon has not been queried yet this returns None."""
+        return self._height
+
+    @handles_errors
+    async def getrawtransaction(self, hex_hash, verbose=False):
+        return await self._send_single('getrawtransaction', (hex_hash, int(verbose)))
+
+    @handles_errors
+    async def getclaimsforname(self, name):
+        '''Given a name, retrieves all claims matching that name.'''
+        return await self._send_single('getclaimsforname', (name,))
+
+    @handles_errors
+    async def getbestblockhash(self):
+        '''Given a name, retrieves all claims matching that name.'''
+        return await self._send_single('getbestblockhash')

scribe/blockchain/network.py

@@ -0,0 +1,300 @@
+import re
+import struct
+import typing
+from typing import List
+from hashlib import sha256
+from decimal import Decimal
+from scribe.base58 import Base58
+from scribe.bip32 import PublicKey
+from scribe.common import hash160, hash_to_hex_str, double_sha256
+from scribe.blockchain.transaction import TxOutput, TxInput, Block
+from scribe.blockchain.transaction.deserializer import Deserializer
+from scribe.blockchain.transaction.script import OpCodes, P2PKH_script, P2SH_script, txo_script_parser
+
+
+HASHX_LEN = 11
+
+
+class CoinError(Exception):
+    """Exception raised for coin-related errors."""
+
+
+ENCODE_CHECK = Base58.encode_check
+DECODE_CHECK = Base58.decode_check
+
+
+class LBCMainNet:
+    NAME = "LBRY"
+    SHORTNAME = "LBC"
+    NET = "mainnet"
+    ENCODE_CHECK = Base58.encode_check
+    DECODE_CHECK = Base58.decode_check
+    DESERIALIZER = Deserializer
+    BASIC_HEADER_SIZE = 112
+    CHUNK_SIZE = 96
+    XPUB_VERBYTES = bytes.fromhex("0488b21e")
+    XPRV_VERBYTES = bytes.fromhex("0488ade4")
+    P2PKH_VERBYTE = bytes.fromhex("55")
+    P2SH_VERBYTES = bytes.fromhex("7A")
+    WIF_BYTE = bytes.fromhex("1C")
+    GENESIS_HASH = '9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463'
+    RPC_PORT = 9245
+    REORG_LIMIT = 200
+    RPC_URL_REGEX = re.compile('.+@(\\[[0-9a-fA-F:]+\\]|[^:]+)(:[0-9]+)?')
+    VALUE_PER_COIN = 100000000
+
+    # Peer discovery
+    PEER_DEFAULT_PORTS = {'t': '50001', 's': '50002'}
+    PEERS: List[str] = []
+    # claimtrie/takeover params
+    nOriginalClaimExpirationTime = 262974
+    nExtendedClaimExpirationTime = 2102400
+    nExtendedClaimExpirationForkHeight = 400155
+    nNormalizedNameForkHeight = 539940  # targeting 21 March 2019
+    nMinTakeoverWorkaroundHeight = 496850
+    nMaxTakeoverWorkaroundHeight = 658300  # targeting 30 Oct 2019
+    nWitnessForkHeight = 680770  # targeting 11 Dec 2019
+    nAllClaimsInMerkleForkHeight = 658310  # targeting 30 Oct 2019
+    proportionalDelayFactor = 32
+    maxTakeoverDelay = 4032
+
+    @classmethod
+    def sanitize_url(cls, url):
+        # Remove surrounding ws and trailing /s
+        url = url.strip().rstrip('/')
+        match = cls.RPC_URL_REGEX.match(url)
+        if not match:
+            raise CoinError(f'invalid daemon URL: "{url}"')
+        if match.groups()[1] is None:
+            url += f':{cls.RPC_PORT:d}'
+        if not url.startswith('http://') and not url.startswith('https://'):
+            url = 'http://' + url
+        return url + '/'
+
+    @classmethod
+    def address_to_hashX(cls, address):
+        """Return a hashX given a coin address."""
+        return cls.hashX_from_script(cls.pay_to_address_script(address))
+
+    @classmethod
+    def P2PKH_address_from_hash160(cls, hash160_bytes):
+        """Return a P2PKH address given a public key."""
+        assert len(hash160_bytes) == 20
+        return ENCODE_CHECK(cls.P2PKH_VERBYTE + hash160_bytes)
+
+    @classmethod
+    def P2PKH_address_from_pubkey(cls, pubkey):
+        """Return a coin address given a public key."""
+        return cls.P2PKH_address_from_hash160(hash160(pubkey))
+
+    @classmethod
+    def P2SH_address_from_hash160(cls, hash160_bytes):
+        """Return a coin address given a hash160."""
+        assert len(hash160_bytes) == 20
+        return ENCODE_CHECK(cls.P2SH_VERBYTES + hash160_bytes)
+
+    @classmethod
+    def hash160_to_P2PKH_script(cls, hash160_bytes):
+        return P2PKH_script(hash160_bytes)
+
+    @classmethod
+    def hash160_to_P2PKH_hashX(cls, hash160_bytes):
+        return cls.hashX_from_script(P2PKH_script(hash160_bytes))
+
+    @classmethod
+    def pay_to_address_script(cls, address):
+        """Return a pubkey script that pays to a pubkey hash.
+
+        Pass the address (either P2PKH or P2SH) in base58 form.
+        """
+        raw = DECODE_CHECK(address)
+
+        # Require version byte(s) plus hash160.
+        verlen = len(raw) - 20
+        if verlen > 0:
+            verbyte, hash160_bytes = raw[:verlen], raw[verlen:]
+            if verbyte == cls.P2PKH_VERBYTE:
+                return P2PKH_script(hash160_bytes)
+            if verbyte in cls.P2SH_VERBYTES:
+                return P2SH_script(hash160_bytes)
+
+        raise CoinError(f'invalid address: {address}')
+
+    @classmethod
+    def privkey_WIF(cls, privkey_bytes, compressed):
+        """Return the private key encoded in Wallet Import Format."""
+        payload = bytearray(cls.WIF_BYTE) + privkey_bytes
+        if compressed:
+            payload.append(0x01)
+        return cls.ENCODE_CHECK(payload)
+
+    @classmethod
+    def header_hash(cls, header):
+        """Given a header return hash"""
+        return double_sha256(header)
+
+    @classmethod
+    def header_prevhash(cls, header):
+        """Given a header return previous hash"""
+        return header[4:36]
+
+    @classmethod
+    def static_header_offset(cls, height):
+        """Given a header height return its offset in the headers file.
+
+        If header sizes change at some point, this is the only code
+        that needs updating."""
+        return height * cls.BASIC_HEADER_SIZE
+
+    @classmethod
+    def static_header_len(cls, height):
+        """Given a header height return its length."""
+        return (cls.static_header_offset(height + 1)
+                - cls.static_header_offset(height))
+
+    @classmethod
+    def block_header(cls, block, height):
+        """Returns the block header given a block and its height."""
+        return block[:cls.static_header_len(height)]
+
+    @classmethod
+    def block(cls, raw_block, height):
+        """Return a Block namedtuple given a raw block and its height."""
+        header = cls.block_header(raw_block, height)
+        txs = Deserializer(raw_block, start=len(header)).read_tx_block()
+        return Block(raw_block, header, txs)
+
+    @classmethod
+    def transaction(cls, raw_tx: bytes):
+        """Return a Block namedtuple given a raw block and its height."""
+        return Deserializer(raw_tx).read_tx()
+
+    @classmethod
+    def decimal_value(cls, value):
+        """Return the number of standard coin units as a Decimal given a
+        quantity of smallest units.
+
+        For example 1 BTC is returned for 100 million satoshis.
+        """
+        return Decimal(value) / cls.VALUE_PER_COIN
+
+    @classmethod
+    def genesis_block(cls, block):
+        '''Check the Genesis block is the right one for this coin.
+
+        Return the block less its unspendable coinbase.
+        '''
+        header = cls.block_header(block, 0)
+        header_hex_hash = hash_to_hex_str(cls.header_hash(header))
+        if header_hex_hash != cls.GENESIS_HASH:
+            raise CoinError(f'genesis block has hash {header_hex_hash} expected {cls.GENESIS_HASH}')
+
+        return block
+
+    @classmethod
+    def electrum_header(cls, header, height):
+        version, = struct.unpack('<I', header[:4])
+        timestamp, bits, nonce = struct.unpack('<III', header[100:112])
+        return {
+            'version': version,
+            'prev_block_hash': hash_to_hex_str(header[4:36]),
+            'merkle_root': hash_to_hex_str(header[36:68]),
+            'claim_trie_root': hash_to_hex_str(header[68:100]),
+            'timestamp': timestamp,
+            'bits': bits,
+            'nonce': nonce,
+            'block_height': height,
+            }
+
+    @classmethod
+    def claim_address_handler(cls, txo: TxOutput) -> typing.Optional[str]:
+        '''Parse a claim script, returns the address
+        '''
+        if txo.pubkey_hash:
+            return cls.P2PKH_address_from_hash160(txo.pubkey_hash)
+        elif txo.script_hash:
+            return cls.P2SH_address_from_hash160(txo.script_hash)
+        elif txo.pubkey:
+            return cls.P2PKH_address_from_pubkey(txo.pubkey)
+
+    @classmethod
+    def hashX_from_txo(cls, txo: 'TxOutput'):
+        address = cls.claim_address_handler(txo)
+        if address:
+            script = cls.pay_to_address_script(address)
+        else:
+            script = txo.pk_script
+        return sha256(script).digest()[:HASHX_LEN]
+
+    @classmethod
+    def hashX_from_script(cls, script: bytes):
+        '''
+        Overrides electrumx hashX from script by extracting addresses from claim scripts.
+        '''
+        if script and script[0] == OpCodes.OP_RETURN or not script:
+            return None
+        if script[0] in [
+            OpCodes.OP_CLAIM_NAME,
+            OpCodes.OP_UPDATE_CLAIM,
+            OpCodes.OP_SUPPORT_CLAIM,
+        ]:
+            decoded = txo_script_parser(script)
+            if not decoded:
+                return
+            claim, support, pubkey_hash, script_hash, pubkey = decoded
+            if pubkey_hash:
+                return cls.address_to_hashX(cls.P2PKH_address_from_hash160(pubkey_hash))
+            elif script_hash:
+                return cls.address_to_hashX(cls.P2SH_address_from_hash160(script_hash))
+            elif pubkey:
+                return cls.address_to_hashX(cls.P2PKH_address_from_pubkey(pubkey))
+        else:
+            return sha256(script).digest()[:HASHX_LEN]
+
+    @classmethod
+    def get_expiration_height(cls, last_updated_height: int, extended: bool = False) -> int:
+        if extended:
+            return last_updated_height + cls.nExtendedClaimExpirationTime
+        if last_updated_height < cls.nExtendedClaimExpirationForkHeight:
+            return last_updated_height + cls.nOriginalClaimExpirationTime
+        return last_updated_height + cls.nExtendedClaimExpirationTime
+
+    @classmethod
+    def get_delay_for_name(cls, blocks_of_continuous_ownership: int) -> int:
+        return min(blocks_of_continuous_ownership // cls.proportionalDelayFactor, cls.maxTakeoverDelay)
+
+    @classmethod
+    def verify_signed_metadata(cls, public_key_bytes: bytes, txo: TxOutput, first_input: TxInput):
+        m = txo.metadata
+        if m.unsigned_payload:
+            pieces = (Base58.decode(cls.claim_address_handler(txo)), m.unsigned_payload, m.signing_channel_hash[::-1])
+        else:
+            pieces = (first_input.prev_hash + first_input.prev_idx.to_bytes(4, byteorder='little'),
+                      m.signing_channel_hash, m.to_message_bytes())
+        return PublicKey.from_compressed(public_key_bytes).verify(
+            m.signature, sha256(b''.join(pieces)).digest()
+        )
+
+
+class LBCRegTest(LBCMainNet):
+    NET = "regtest"
+    GENESIS_HASH = '6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556'
+    XPUB_VERBYTES = bytes.fromhex('043587cf')
+    XPRV_VERBYTES = bytes.fromhex('04358394')
+
+    P2PKH_VERBYTE = bytes.fromhex("6f")
+    P2SH_VERBYTES = bytes.fromhex("c4")
+
+    nOriginalClaimExpirationTime = 500
+    nExtendedClaimExpirationTime = 600
+    nExtendedClaimExpirationForkHeight = 800
+    nNormalizedNameForkHeight = 250
+    nMinTakeoverWorkaroundHeight = -1
+    nMaxTakeoverWorkaroundHeight = -1
+    nWitnessForkHeight = 150
+    nAllClaimsInMerkleForkHeight = 350
+
+
+class LBCTestNet(LBCRegTest):
+    NET = "testnet"
+    GENESIS_HASH = '9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463'

scribe/blockchain/prefetcher.py

@@ -0,0 +1,128 @@
+import asyncio
+import logging
+import typing
+if typing.TYPE_CHECKING:
+    from scribe.blockchain.network import LBCMainNet
+    from scribe.blockchain.daemon import LBCDaemon
+
+
+def chunks(items, size):
+    """Break up items, an iterable, into chunks of length size."""
+    for i in range(0, len(items), size):
+        yield items[i: i + size]
+
+
+class Prefetcher:
+    """Prefetches blocks (in the forward direction only)."""
+
+    def __init__(self, daemon: 'LBCDaemon', coin: 'LBCMainNet', blocks_event: asyncio.Event):
+        self.logger = logging.getLogger(__name__)
+        self.daemon = daemon
+        self.coin = coin
+        self.blocks_event = blocks_event
+        self.blocks = []
+        self.caught_up = False
+        # Access to fetched_height should be protected by the semaphore
+        self.fetched_height = None
+        self.semaphore = asyncio.Semaphore()
+        self.refill_event = asyncio.Event()
+        # The prefetched block cache size.  The min cache size has
+        # little effect on sync time.
+        self.cache_size = 0
+        self.min_cache_size = 10 * 1024 * 1024
+        # This makes the first fetch be 10 blocks
+        self.ave_size = self.min_cache_size // 10
+        self.polling_delay = 0.5
+
+    async def main_loop(self, bp_height):
+        """Loop forever polling for more blocks."""
+        await self.reset_height(bp_height)
+        try:
+            while True:
+                # Sleep a while if there is nothing to prefetch
+                await self.refill_event.wait()
+                if not await self._prefetch_blocks():
+                    await asyncio.sleep(self.polling_delay)
+        except Exception as e:
+            if not isinstance(e, asyncio.CancelledError):
+                self.logger.exception("block fetcher loop crashed")
+            raise
+        finally:
+            self.logger.info("block pre-fetcher is shutting down")
+
+    def get_prefetched_blocks(self):
+        """Called by block processor when it is processing queued blocks."""
+        blocks = self.blocks
+        self.blocks = []
+        self.cache_size = 0
+        self.refill_event.set()
+        return blocks
+
+    async def reset_height(self, height):
+        """Reset to prefetch blocks from the block processor's height.
+
+        Used in blockchain reorganisations.  This coroutine can be
+        called asynchronously to the _prefetch_blocks coroutine so we
+        must synchronize with a semaphore.
+        """
+        async with self.semaphore:
+            self.blocks.clear()
+            self.cache_size = 0
+            self.fetched_height = height
+            self.refill_event.set()
+
+        daemon_height = await self.daemon.height()
+        behind = daemon_height - height
+        if behind > 0:
+            self.logger.info(f'catching up to daemon height {daemon_height:,d} '
+                             f'({behind:,d} blocks behind)')
+        else:
+            self.logger.info(f'caught up to daemon height {daemon_height:,d}')
+
+    async def _prefetch_blocks(self):
+        """Prefetch some blocks and put them on the queue.
+
+        Repeats until the queue is full or caught up.
+        """
+        daemon = self.daemon
+        daemon_height = await daemon.height()
+        async with self.semaphore:
+            while self.cache_size < self.min_cache_size:
+                # Try and catch up all blocks but limit to room in cache.
+                # Constrain fetch count to between 0 and 500 regardless;
+                # testnet can be lumpy.
+                cache_room = self.min_cache_size // self.ave_size
+                count = min(daemon_height - self.fetched_height, cache_room)
+                count = min(500, max(count, 0))
+                if not count:
+                    self.caught_up = True
+                    return False
+
+                first = self.fetched_height + 1
+                hex_hashes = await daemon.block_hex_hashes(first, count)
+                if self.caught_up:
+                    self.logger.info('new block height {:,d} hash {}'
+                                     .format(first + count-1, hex_hashes[-1]))
+                blocks = await daemon.raw_blocks(hex_hashes)
+
+                assert count == len(blocks)
+
+                # Special handling for genesis block
+                if first == 0:
+                    blocks[0] = self.coin.genesis_block(blocks[0])
+                    self.logger.info(f'verified genesis block with hash {hex_hashes[0]}')
+
+                # Update our recent average block size estimate
+                size = sum(len(block) for block in blocks)
+                if count >= 10:
+                    self.ave_size = size // count
+                else:
+                    self.ave_size = (size + (10 - count) * self.ave_size) // 10
+
+                self.blocks.extend(blocks)
+                self.cache_size += size
+                self.fetched_height += count
+                self.blocks_event.set()
+
+        self.refill_event.clear()
+        return True

scribe/blockchain/transaction/__init__.py

@@ -0,0 +1,148 @@
+import sys
+import functools
+import typing
+from dataclasses import dataclass
+from struct import Struct
+from scribe.schema.claim import Claim
+
+if (sys.version_info.major, sys.version_info.minor) > (3, 7):
+    cachedproperty = functools.cached_property
+else:
+    cachedproperty = property
+
+
+struct_le_i = Struct('<i')
+struct_le_q = Struct('<q')
+struct_le_H = Struct('<H')
+struct_le_I = Struct('<I')
+struct_le_Q = Struct('<Q')
+struct_be_H = Struct('>H')
+struct_be_I = Struct('>I')
+structB = Struct('B')
+
+unpack_le_int32_from = struct_le_i.unpack_from
+unpack_le_int64_from = struct_le_q.unpack_from
+unpack_le_uint16_from = struct_le_H.unpack_from
+unpack_le_uint32_from = struct_le_I.unpack_from
+unpack_le_uint64_from = struct_le_Q.unpack_from
+unpack_be_uint16_from = struct_be_H.unpack_from
+unpack_be_uint32_from = struct_be_I.unpack_from
+
+pack_le_int32 = struct_le_i.pack
+pack_le_int64 = struct_le_q.pack
+pack_le_uint16 = struct_le_H.pack
+pack_le_uint32 = struct_le_I.pack
+pack_le_uint64 = struct_le_q.pack
+pack_be_uint64 = lambda x: x.to_bytes(8, byteorder='big')
+pack_be_uint16 = lambda x: x.to_bytes(2, byteorder='big')
+pack_be_uint32 = struct_be_I.pack
+pack_byte = structB.pack
+
+
+def pack_varint(n):
+    if n < 253:
+        return pack_byte(n)
+    if n < 65536:
+        return pack_byte(253) + pack_le_uint16(n)
+    if n < 4294967296:
+        return pack_byte(254) + pack_le_uint32(n)
+    return pack_byte(255) + pack_le_uint64(n)
+
+
+def pack_varbytes(data):
+    return pack_varint(len(data)) + data
+
+
+class NameClaim(typing.NamedTuple):
+    name: bytes
+    value: bytes
+
+
+class ClaimUpdate(typing.NamedTuple):
+    name: bytes
+    claim_hash: bytes
+    value: typing.Optional[bytes] = None
+
+
+class ClaimSupport(typing.NamedTuple):
+    name: bytes
+    claim_hash: bytes
+    value: typing.Optional[bytes] = None
+
+
+ZERO = bytes(32)
+MINUS_1 = 4294967295
+
+
+class Tx(typing.NamedTuple):
+    version: int
+    inputs: typing.List['TxInput']
+    outputs: typing.List['TxOutput']
+    locktime: int
+    raw: bytes
+    marker: typing.Optional[int] = None
+    flag: typing.Optional[int] = None
+    witness: typing.Optional[typing.List[typing.List[bytes]]] = None
+
+
+class TxInput(typing.NamedTuple):
+    prev_hash: bytes
+    prev_idx: int
+    script: bytes
+    sequence: int
+
+    def __str__(self):
+        return f"TxInput({self.prev_hash[::-1].hex()}, {self.prev_idx:d}, script={self.script.hex()}, " \
+               f"sequence={self.sequence:d})"
+
+    def is_generation(self):
+        """Test if an input is generation/coinbase like"""
+        return self.prev_idx == MINUS_1 and self.prev_hash == ZERO
+
+    def serialize(self):
+        return b''.join((
+            self.prev_hash,
+            pack_le_uint32(self.prev_idx),
+            pack_varbytes(self.script),
+            pack_le_uint32(self.sequence),
+        ))
+
+
+@dataclass
+class TxOutput:
+    nout: int
+    value: int
+    pk_script: bytes
+    claim: typing.Optional[typing.Union[NameClaim, ClaimUpdate]]  # TODO: fix this being mutable, it shouldn't be
+    support: typing.Optional[ClaimSupport]
+    pubkey_hash: typing.Optional[bytes]
+    script_hash: typing.Optional[bytes]
+    pubkey: typing.Optional[bytes]
+
+    @property
+    def is_claim(self):
+        return isinstance(self.claim, NameClaim)
+
+    @property
+    def is_update(self):
+        return isinstance(self.claim, ClaimUpdate)
+
+    @cachedproperty
+    def metadata(self) -> typing.Optional[Claim]:
+        return None if not (self.claim or self.support).value else Claim.from_bytes((self.claim or self.support).value)
+
+    @property
+    def is_support(self):
+        return self.support is not None
+
+    def serialize(self):
+        return b''.join((
+            pack_le_int64(self.value),
+            pack_varbytes(self.pk_script),
+        ))
+
+
+class Block(typing.NamedTuple):
+    raw: bytes
+    header: bytes
+    transactions: typing.List[Tx]

scribe/blockchain/transaction/deserializer.py

@@ -0,0 +1,163 @@
+from scribe.common import double_sha256
+from scribe.blockchain.transaction import (
+    unpack_le_int32_from, unpack_le_int64_from, unpack_le_uint16_from,
+    unpack_le_uint32_from, unpack_le_uint64_from, Tx, TxInput, TxOutput
+)
+from scribe.blockchain.transaction.script import txo_script_parser
+
+
+class Deserializer:
+    """Deserializes blocks into transactions.
+
+    External entry points are read_tx(), read_tx_and_hash(),
+    read_tx_and_vsize() and read_block().
+
+    This code is performance sensitive as it is executed 100s of
+    millions of times during sync.
+    """
+
+    TX_HASH_FN = staticmethod(double_sha256)
+
+    def __init__(self, binary, start=0):
+        assert isinstance(binary, bytes), f"type {type(binary)} is not 'bytes'"
+        self.binary = binary
+        self.binary_length = len(binary)
+        self.cursor = start
+        self.flags = 0
+
+    def _read_witness(self, fields):
+        read_witness_field = self._read_witness_field
+        return [read_witness_field() for i in range(fields)]
+
+    def _read_witness_field(self):
+        read_varbytes = self._read_varbytes
+        return [read_varbytes() for i in range(self._read_varint())]
+
+    def _read_tx_parts(self):
+        """Return a (deserialized TX, tx_hash, vsize) tuple."""
+        start = self.cursor
+        marker = self.binary[self.cursor + 4]
+        if marker:
+            tx = Tx(
+                self._read_le_int32(),  # version
+                self._read_inputs(),  # inputs
+                self._read_outputs(),  # outputs
+                self._read_le_uint32(),  # locktime
+                self.binary[start:self.cursor],
+            )
+            tx_hash = self.TX_HASH_FN(self.binary[start:self.cursor])
+            return tx, tx_hash, self.binary_length
+
+        # Ugh, this is nasty.
+        version = self._read_le_int32()
+        orig_ser = self.binary[start:self.cursor]
+
+        marker = self._read_byte()
+        flag = self._read_byte()
+
+        start = self.cursor
+        inputs = self._read_inputs()
+        outputs = self._read_outputs()
+        orig_ser += self.binary[start:self.cursor]
+
+        base_size = self.cursor - start
+        witness = self._read_witness(len(inputs))
+
+        start = self.cursor
+        locktime = self._read_le_uint32()
+        orig_ser += self.binary[start:self.cursor]
+        vsize = (3 * base_size + self.binary_length) // 4
+
+        return Tx(version, inputs, outputs, locktime, orig_ser, marker, flag, witness), self.TX_HASH_FN(orig_ser), vsize
+
+    def read_tx(self):
+        return self._read_tx_parts()[0]
+
+    def read_tx_and_hash(self):
+        tx, tx_hash, vsize = self._read_tx_parts()
+        return tx, tx_hash
+
+    def read_tx_and_vsize(self):
+        tx, tx_hash, vsize = self._read_tx_parts()
+        return tx, vsize
+
+    def read_tx_block(self):
+        """Returns a list of (deserialized_tx, tx_hash) pairs."""
+        read = self.read_tx_and_hash
+        # Some coins have excess data beyond the end of the transactions
+        return [read() for _ in range(self._read_varint())]
+
+    def _read_inputs(self):
+        read_input = self._read_input
+        return [read_input() for i in range(self._read_varint())]
+
+    def _read_input(self):
+        return TxInput(
+            self._read_nbytes(32),   # prev_hash
+            self._read_le_uint32(),  # prev_idx
+            self._read_varbytes(),   # script
+            self._read_le_uint32()   # sequence
+        )
+
+    def _read_outputs(self):
+        read_output = self._read_output
+        return [read_output(n) for n in range(self._read_varint())]
+
+    def _read_output(self, n):
+        value = self._read_le_int64()
+        script = self._read_varbytes()  # pk_script
+        decoded = txo_script_parser(script)
+        claim = support = pubkey_hash = script_hash = pubkey = None
+        if decoded:
+            claim, support, pubkey_hash, script_hash, pubkey = decoded
+        return TxOutput(n, value, script, claim, support, pubkey_hash, script_hash, pubkey)
+
+    def _read_byte(self):
+        cursor = self.cursor
+        self.cursor += 1
+        return self.binary[cursor]
+
+    def _read_nbytes(self, n):
+        cursor = self.cursor
+        self.cursor = end = cursor + n
+        assert self.binary_length >= end
+        return self.binary[cursor:end]
+
+    def _read_varbytes(self):
+        return self._read_nbytes(self._read_varint())
+
+    def _read_varint(self):
+        n = self.binary[self.cursor]
+        self.cursor += 1
+        if n < 253:
+            return n
+        if n == 253:
+            return self._read_le_uint16()
+        if n == 254:
+            return self._read_le_uint32()
+        return self._read_le_uint64()
+
+    def _read_le_int32(self):
+        result, = unpack_le_int32_from(self.binary, self.cursor)
+        self.cursor += 4
+        return result
+
+    def _read_le_int64(self):
+        result, = unpack_le_int64_from(self.binary, self.cursor)
+        self.cursor += 8
+        return result
+
+    def _read_le_uint16(self):
+        result, = unpack_le_uint16_from(self.binary, self.cursor)
+        self.cursor += 2
+        return result
+
+    def _read_le_uint32(self):
+        result, = unpack_le_uint32_from(self.binary, self.cursor)
+        self.cursor += 4
+        return result
+
+    def _read_le_uint64(self):
+        result, = unpack_le_uint64_from(self.binary, self.cursor)
+        self.cursor += 8
+        return result

scribe/blockchain/transaction/script.py

@@ -0,0 +1,298 @@
+import typing
+from scribe.blockchain.transaction import NameClaim, ClaimUpdate, ClaimSupport
+from scribe.blockchain.transaction import unpack_le_uint16_from, unpack_le_uint32_from, pack_le_uint16, pack_le_uint32
+
+
+class _OpCodes(typing.NamedTuple):
+    def whatis(self, value: int):
+        try:
+            return self._fields[self.index(value)]
+        except (ValueError, IndexError):
+            return -1
+
+    OP_PUSHDATA1: int = 0x4c
+    OP_PUSHDATA2: int = 0x4d
+    OP_PUSHDATA4: int = 0x4e
+    OP_1NEGATE: int = 0x4f
+    OP_RESERVED: int = 0x50
+    OP_1: int = 0x51
+    OP_2: int = 0x52
+    OP_3: int = 0x53
+    OP_4: int = 0x54
+    OP_5: int = 0x55
+    OP_6: int = 0x56
+    OP_7: int = 0x57
+    OP_8: int = 0x58
+    OP_9: int = 0x59
+    OP_10: int = 0x5a
+    OP_11: int = 0x5b
+    OP_12: int = 0x5c
+    OP_13: int = 0x5d
+    OP_14: int = 0x5e
+    OP_15: int = 0x5f
+    OP_16: int = 0x60
+    OP_NOP: int = 0x61
+    OP_VER: int = 0x62
+    OP_IF: int = 0x63
+    OP_NOTIF: int = 0x64
+    OP_VERIF: int = 0x65
+    OP_VERNOTIF: int = 0x66
+    OP_ELSE: int = 0x67
+    OP_ENDIF: int = 0x68
+    OP_VERIFY: int = 0x69
+    OP_RETURN: int = 0x6a
+    OP_TOALTSTACK: int = 0x6b
+    OP_FROMALTSTACK: int = 0x6c
+    OP_2DROP: int = 0x6d
+    OP_2DUP: int = 0x6e
+    OP_3DUP: int = 0x6f
+    OP_2OVER: int = 0x70
+    OP_2ROT: int = 0x71
+    OP_2SWAP: int = 0x72
+    OP_IFDUP: int = 0x73
+    OP_DEPTH: int = 0x74
+    OP_DROP: int = 0x75
+    OP_DUP: int = 0x76
+    OP_NIP: int = 0x77
+    OP_OVER: int = 0x78
+    OP_PICK: int = 0x79
+    OP_ROLL: int = 0x7a
+    OP_ROT: int = 0x7b
+    OP_SWAP: int = 0x7c
+    OP_TUCK: int = 0x7d
+    OP_CAT: int = 0x7e
+    OP_SUBSTR: int = 0x7f
+    OP_LEFT: int = 0x80
+    OP_RIGHT: int = 0x81
+    OP_SIZE: int = 0x82
+    OP_INVERT: int = 0x83
+    OP_AND: int = 0x84
+    OP_OR: int = 0x85
+    OP_XOR: int = 0x86
+    OP_EQUAL: int = 0x87
+    OP_EQUALVERIFY: int = 0x88
+    OP_RESERVED1: int = 0x89
+    OP_RESERVED2: int = 0x8a
+    OP_1ADD: int = 0x8b
+    OP_1SUB: int = 0x8c
+    OP_2MUL: int = 0x8d
+    OP_2DIV: int = 0x8e
+    OP_NEGATE: int = 0x8f
+    OP_ABS: int = 0x90
+    OP_NOT: int = 0x91
+    OP_0NOTEQUAL: int = 0x92
+    OP_ADD: int = 0x93
+    OP_SUB: int = 0x94
+    OP_MUL: int = 0x95
+    OP_DIV: int = 0x96
+    OP_MOD: int = 0x97
+    OP_LSHIFT: int = 0x98
+    OP_RSHIFT: int = 0x99
+    OP_BOOLAND: int = 0x9a
+    OP_BOOLOR: int = 0x9b
+    OP_NUMEQUAL: int = 0x9c
+    OP_NUMEQUALVERIFY: int = 0x9d
+    OP_NUMNOTEQUAL: int = 0x9e
+    OP_LESSTHAN: int = 0x9f
+    OP_GREATERTHAN: int = 0xa0
+    OP_LESSTHANOREQUAL: int = 0xa1
+    OP_GREATERTHANOREQUAL: int = 0xa2
+    OP_MIN: int = 0xa3
+    OP_MAX: int = 0xa4
+    OP_WITHIN: int = 0xa5
+    OP_RIPEMD160: int = 0xa6
+    OP_SHA1: int = 0xa7
+    OP_SHA256: int = 0xa8
+    OP_HASH160: int = 0xa9
+    OP_HASH256: int = 0xaa
+    OP_CODESEPARATOR: int = 0xab
+    OP_CHECKSIG: int = 0xac
+    OP_CHECKSIGVERIFY: int = 0xad
+    OP_CHECKMULTISIG: int = 0xae
+    OP_CHECKMULTISIGVERIFY: int = 0xaf
+    OP_NOP1: int = 0xb0
+    OP_CHECKLOCKTIMEVERIFY: int = 0xb1
+    OP_CHECKSEQUENCEVERIFY: int = 0xb2
+    OP_NOP4: int = 0xb3
+    OP_NOP5: int = 0xb4
+    OP_CLAIM_NAME: int = 0xb5
+    OP_SUPPORT_CLAIM: int = 0xb6
+    OP_UPDATE_CLAIM: int = 0xb7
+    OP_NOP9: int = 0xb8
+    OP_NOP10: int = 0xb9
+
+
+OpCodes = _OpCodes()
+
+
+# Paranoia to make it hard to create bad scripts
+assert OpCodes.OP_DUP == 0x76
+assert OpCodes.OP_HASH160 == 0xa9
+assert OpCodes.OP_EQUAL == 0x87
+assert OpCodes.OP_EQUALVERIFY == 0x88
+assert OpCodes.OP_CHECKSIG == 0xac
+assert OpCodes.OP_CHECKMULTISIG == 0xae
+
+assert OpCodes.OP_CLAIM_NAME == 0xb5
+assert OpCodes.OP_SUPPORT_CLAIM == 0xb6
+assert OpCodes.OP_UPDATE_CLAIM == 0xb7
+
+
+def P2SH_script(hash160_bytes: bytes):
+    return bytes([OpCodes.OP_HASH160]) + script_push_data(hash160_bytes) + bytes([OpCodes.OP_EQUAL])
+
+
+def P2PKH_script(hash160_bytes: bytes):
+    return (bytes([OpCodes.OP_DUP, OpCodes.OP_HASH160])
+            + script_push_data(hash160_bytes)
+            + bytes([OpCodes.OP_EQUALVERIFY, OpCodes.OP_CHECKSIG]))
+
+
+def script_push_data(data: bytes):
+    n = len(data)
+    if n < OpCodes.OP_PUSHDATA1:
+        return bytes([n]) + data
+    if n < 256:
+        return bytes([OpCodes.OP_PUSHDATA1, n]) + data
+    if n < 65536:
+        return bytes([OpCodes.OP_PUSHDATA2]) + pack_le_uint16(n) + data
+    return bytes([OpCodes.OP_PUSHDATA4]) + pack_le_uint32(n) + data
+
+
+def script_GetOp(script_bytes: bytes):
+    i = 0
+    while i < len(script_bytes):
+        vch = None
+        opcode = script_bytes[i]
+        i += 1
+        if opcode <= OpCodes.OP_PUSHDATA4:
+            n_size = opcode
+            if opcode == OpCodes.OP_PUSHDATA1:
+                n_size = script_bytes[i]
+                i += 1
+            elif opcode == OpCodes.OP_PUSHDATA2:
+                (n_size,) = unpack_le_uint16_from(script_bytes, i)
+                i += 2
+            elif opcode == OpCodes.OP_PUSHDATA4:
+                (n_size,) = unpack_le_uint32_from(script_bytes, i)
+                i += 4
+            if i + n_size > len(script_bytes):
+                vch = b"_INVALID_" + script_bytes[i:]
+                i = len(script_bytes)
+            else:
+                vch = script_bytes[i:i + n_size]
+                i += n_size
+        yield opcode, vch, i
+
+
+_SCRIPT_TEMPLATES = (
+    # claim related templates
+    (OpCodes.OP_CLAIM_NAME, -1, -1, OpCodes.OP_2DROP, OpCodes.OP_DROP),
+    (OpCodes.OP_UPDATE_CLAIM, -1, -1, OpCodes.OP_2DROP, OpCodes.OP_DROP),
+    (OpCodes.OP_UPDATE_CLAIM, -1, -1, -1, OpCodes.OP_2DROP, OpCodes.OP_2DROP),
+    (OpCodes.OP_SUPPORT_CLAIM, -1, -1, OpCodes.OP_2DROP, OpCodes.OP_DROP),
+    (OpCodes.OP_SUPPORT_CLAIM, -1, -1, -1, OpCodes.OP_2DROP, OpCodes.OP_2DROP),
+
+    # receive script templates
+    (OpCodes.OP_DUP, OpCodes.OP_HASH160, -1, OpCodes.OP_EQUALVERIFY, OpCodes.OP_CHECKSIG),
+    (OpCodes.OP_HASH160, -1, OpCodes.OP_EQUAL),
+    (-1, OpCodes.OP_CHECKSIG)
+)
+_CLAIM_TEMPLATE = 0
+_UPDATE_NO_DATA_TEMPLATE = 1
+_UPDATE_TEMPLATE = 2
+_SUPPORT_TEMPLATE = 3
+_SUPPORT_WITH_DATA_TEMPLATE = 4
+
+_TO_ADDRESS_TEMPLATE = 5
+_TO_P2SH_TEMPLATE = 6
+_TO_PUBKEY_TEMPLATE = 7
+
+
+def txo_script_parser(script: bytes):
+    template = None
+    template_idx = None
+    values = []
+    receive_values = []
+    finished_decoding_claim = False
+    receive_cur = 0
+    claim, support, pubkey_hash, pubkey, script_hash = None, None, None, None, None
+    for cur, (op, data, _) in enumerate(script_GetOp(script)):
+        if finished_decoding_claim:  # we're decoding the receiving part of the script (the last part)
+            if receive_cur == 0:
+                if op == OpCodes.OP_DUP:
+                    template_idx = _TO_ADDRESS_TEMPLATE
+                elif op == OpCodes.OP_HASH160:
+                    template_idx = _TO_P2SH_TEMPLATE
+                elif op == -1:
+                    template_idx = _TO_PUBKEY_TEMPLATE
+                else:
+                    break  # return the decoded part
+                template = _SCRIPT_TEMPLATES[template_idx]
+            expected = template[receive_cur]
+            if expected == -1 and data is None:  # if data data is expected make sure it's there
+                # print("\texpected data", OpCodes.whatis(op), data)
+                return
+            elif expected == -1 and data:
+                receive_values.append(data)
+            elif op != expected:
+                # print("\top mismatch")
+                return
+            receive_cur += 1
+            continue
+
+        if cur == 0:  # initialize the template
+            if op == OpCodes.OP_CLAIM_NAME:
+                template_idx = _CLAIM_TEMPLATE
+            elif op == OpCodes.OP_UPDATE_CLAIM:
+                template_idx = _UPDATE_NO_DATA_TEMPLATE
+            elif op == OpCodes.OP_SUPPORT_CLAIM:
+                template_idx = _SUPPORT_TEMPLATE  # could be a support w/ data
+            elif op == OpCodes.OP_DUP:
+                template_idx = _TO_ADDRESS_TEMPLATE
+            elif op == OpCodes.OP_HASH160:
+                template_idx = _TO_P2SH_TEMPLATE
+            elif op == -1:
+                template_idx = _TO_PUBKEY_TEMPLATE
+            else:
+                return
+            template = _SCRIPT_TEMPLATES[template_idx]
+        elif cur == 3 and template_idx == _SUPPORT_TEMPLATE and data:
+            template_idx = _SUPPORT_WITH_DATA_TEMPLATE
+            template = _SCRIPT_TEMPLATES[template_idx]
+        elif cur == 3 and template_idx == _UPDATE_NO_DATA_TEMPLATE and data:
+            template_idx = _UPDATE_TEMPLATE
+            template = _SCRIPT_TEMPLATES[template_idx]
+
+        if cur >= len(template):
+            return
+
+        expected = template[cur]
+
+        if expected == -1 and data is None:  # if data data is expected make sure it's there
+            # print("\texpected data", OpCodes.whatis(op), data)
+            return
+        elif expected == -1 and data:
+            if template_idx in (_TO_ADDRESS_TEMPLATE, _TO_P2SH_TEMPLATE, _TO_ADDRESS_TEMPLATE):
+                receive_values.append(data)
+            else:
+                values.append(data)
+        elif op != expected:
+            # print("\top mismatch")
+            return
+        if cur + 1 == len(template):
+            finished_decoding_claim = True
+            if template_idx == _CLAIM_TEMPLATE:
+                claim = NameClaim(*values)
+            elif template_idx in (_UPDATE_NO_DATA_TEMPLATE, _UPDATE_TEMPLATE):
+                claim = ClaimUpdate(*values)
+            elif template_idx in (_SUPPORT_TEMPLATE, _SUPPORT_WITH_DATA_TEMPLATE):
+                support = ClaimSupport(*values)
+
+    if template_idx == _TO_ADDRESS_TEMPLATE:
+        pubkey_hash = receive_values[0]
+    elif template_idx == _TO_P2SH_TEMPLATE:
+        script_hash = receive_values[0]
+    elif template_idx == _TO_PUBKEY_TEMPLATE:
+        pubkey = receive_values[0]
+    return claim, support, pubkey_hash, script_hash, pubkey

scribe/build_info.py

@@ -0,0 +1,4 @@
+# don't touch this. CI server changes this during build/deployment
+BUILD = "dev"
+COMMIT_HASH = "none"
+DOCKER_TAG = "none"

scribe/cli.py

@@ -0,0 +1,63 @@
+import logging
+import traceback
+import argparse
+from scribe.env import Env
+from scribe.blockchain.block_processor import BlockProcessor
+from scribe.readers import BlockchainReaderServer, ElasticWriter
+
+
+def get_arg_parser(name):
+    parser = argparse.ArgumentParser(
+        prog=name
+    )
+    Env.contribute_to_arg_parser(parser)
+    return parser
+
+
+def setup_logging():
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-4s %(name)s:%(lineno)d: %(message)s")
+    logging.getLogger('aiohttp').setLevel(logging.WARNING)
+    logging.getLogger('elasticsearch').setLevel(logging.WARNING)
+
+
+def run_writer_forever():
+    setup_logging()
+    args = get_arg_parser('scribe').parse_args()
+    try:
+        block_processor = BlockProcessor(Env.from_arg_parser(args))
+        block_processor.run()
+    except Exception:
+        traceback.print_exc()
+        logging.critical('scribe terminated abnormally')
+    else:
+        logging.info('scribe terminated normally')
+
+
+def run_server_forever():
+    setup_logging()
+    args = get_arg_parser('scribe-hub').parse_args()
+
+    try:
+        server = BlockchainReaderServer(Env.from_arg_parser(args))
+        server.run()
+    except Exception:
+        traceback.print_exc()
+        logging.critical('hub terminated abnormally')
+    else:
+        logging.info('hub terminated normally')
+
+
+def run_es_sync_forever():
+    setup_logging()
+    parser = get_arg_parser('scribe-elastic-sync')
+    parser.add_argument('--reindex', type=bool, default=False)
+    args = parser.parse_args()
+
+    try:
+        server = ElasticWriter(Env.from_arg_parser(args))
+        server.run(args.reindex)
+    except Exception:
+        traceback.print_exc()
+        logging.critical('es sync terminated abnormally')
+    else:
+        logging.info('es sync terminated normally')

scribe/common.py

@@ -0,0 +1,362 @@
+import hashlib
+import hmac
+import ipaddress
+import logging
+import typing
+import collections
+from asyncio import get_event_loop, Event
+from prometheus_client import Counter
+
+log = logging.getLogger(__name__)
+
+
+_sha256 = hashlib.sha256
+_sha512 = hashlib.sha512
+_new_hash = hashlib.new
+_new_hmac = hmac.new
+HASHX_LEN = 11
+CLAIM_HASH_LEN = 20
+
+
+# class cachedproperty:
+#     def __init__(self, f):
+#         self.f = f
+#
+#     def __get__(self, obj, type):
+#         obj = obj or type
+#         value = self.f(obj)
+#         setattr(obj, self.f.__name__, value)
+#         return value
+
+
+def formatted_time(t, sep=' '):
+    """Return a number of seconds as a string in days, hours, mins and
+    maybe secs."""
+    t = int(t)
+    fmts = (('{:d}d', 86400), ('{:02d}h', 3600), ('{:02d}m', 60))
+    parts = []
+    for fmt, n in fmts:
+        val = t // n
+        if parts or val:
+            parts.append(fmt.format(val))
+        t %= n
+    if len(parts) < 3:
+        parts.append(f'{t:02d}s')
+    return sep.join(parts)
+
+
+def protocol_tuple(s):
+    """Converts a protocol version number, such as "1.0" to a tuple (1, 0).
+
+    If the version number is bad, (0, ) indicating version 0 is returned."""
+    try:
+        return tuple(int(part) for part in s.split('.'))
+    except Exception:
+        return (0, )
+
+
+def version_string(ptuple):
+    """Convert a version tuple such as (1, 2) to "1.2".
+    There is always at least one dot, so (1, ) becomes "1.0"."""
+    while len(ptuple) < 2:
+        ptuple += (0, )
+    return '.'.join(str(p) for p in ptuple)
+
+
+def protocol_version(client_req, min_tuple, max_tuple):
+    """Given a client's protocol version string, return a pair of
+    protocol tuples:
+           (negotiated version, client min request)
+    If the request is unsupported, the negotiated protocol tuple is
+    None.
+    """
+    if client_req is None:
+        client_min = client_max = min_tuple
+    else:
+        if isinstance(client_req, list) and len(client_req) == 2:
+            client_min, client_max = client_req
+        else:
+            client_min = client_max = client_req
+        client_min = protocol_tuple(client_min)
+        client_max = protocol_tuple(client_max)
+
+    result = min(client_max, max_tuple)
+    if result < max(client_min, min_tuple) or result == (0, ):
+        result = None
+
+    return result, client_min
+
+
+class LRUCacheWithMetrics:
+    __slots__ = [
+        'capacity',
+        'cache',
+        '_track_metrics',
+        'hits',
+        'misses'
+    ]
+
+    def __init__(self, capacity: int, metric_name: typing.Optional[str] = None, namespace: str = "daemon_cache"):
+        self.capacity = capacity
+        self.cache = collections.OrderedDict()
+        if metric_name is None:
+            self._track_metrics = False
+            self.hits = self.misses = None
+        else:
+            self._track_metrics = True
+            try:
+                self.hits = Counter(
+                    f"{metric_name}_cache_hit_count", "Number of cache hits", namespace=namespace
+                )
+                self.misses = Counter(
+                    f"{metric_name}_cache_miss_count", "Number of cache misses", namespace=namespace
+                )
+            except ValueError as err:
+                log.debug("failed to set up prometheus %s_cache_miss_count metric: %s", metric_name, err)
+                self._track_metrics = False
+                self.hits = self.misses = None
+
+    def get(self, key, default=None):
+        try:
+            value = self.cache.pop(key)
+            if self._track_metrics:
+                self.hits.inc()
+        except KeyError:
+            if self._track_metrics:
+                self.misses.inc()
+            return default
+        self.cache[key] = value
+        return value
+
+    def set(self, key, value):
+        try:
+            self.cache.pop(key)
+        except KeyError:
+            if len(self.cache) >= self.capacity:
+                self.cache.popitem(last=False)
+        self.cache[key] = value
+
+    def clear(self):
+        self.cache.clear()
+
+    def pop(self, key):
+        return self.cache.pop(key)
+
+    def __setitem__(self, key, value):
+        return self.set(key, value)
+
+    def __getitem__(self, item):
+        return self.get(item)
+
+    def __contains__(self, item) -> bool:
+        return item in self.cache
+
+    def __len__(self):
+        return len(self.cache)
+
+    def __delitem__(self, key):
+        self.cache.pop(key)
+
+    def __del__(self):
+        self.clear()
+
+
+class LRUCache:
+    __slots__ = [
+        'capacity',
+        'cache'
+    ]
+
+    def __init__(self, capacity: int):
+        self.capacity = capacity
+        self.cache = collections.OrderedDict()
+
+    def get(self, key, default=None):
+        try:
+            value = self.cache.pop(key)
+        except KeyError:
+            return default
+        self.cache[key] = value
+        return value
+
+    def set(self, key, value):
+        try:
+            self.cache.pop(key)
+        except KeyError:
+            if len(self.cache) >= self.capacity:
+                self.cache.popitem(last=False)
+        self.cache[key] = value
+
+    def items(self):
+        return self.cache.items()
+
+    def clear(self):
+        self.cache.clear()
+
+    def pop(self, key, default=None):
+        return self.cache.pop(key, default)
+
+    def __setitem__(self, key, value):
+        return self.set(key, value)
+
+    def __getitem__(self, item):
+        return self.get(item)
+
+    def __contains__(self, item) -> bool:
+        return item in self.cache
+
+    def __len__(self):
+        return len(self.cache)
+
+    def __delitem__(self, key):
+        self.cache.pop(key)
+
+    def __del__(self):
+        self.clear()
+
+
+# the ipaddress module does not show these subnets as reserved
+CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
+IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
+
+
+def is_valid_public_ipv4(address, allow_localhost: bool = False, allow_lan: bool = False):
+    try:
+        parsed_ip = ipaddress.ip_address(address)
+        if parsed_ip.is_loopback and allow_localhost:
+            return True
+        if allow_lan and parsed_ip.is_private:
+            return True
+        if any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local, parsed_ip.is_loopback,
+                parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private)):
+            return False
+        else:
+            return not any((CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
+                            IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
+    except (ipaddress.AddressValueError, ValueError):
+        return False
+
+
+def sha256(x):
+    """Simple wrapper of hashlib sha256."""
+    return _sha256(x).digest()
+
+
+def ripemd160(x):
+    """Simple wrapper of hashlib ripemd160."""
+    h = _new_hash('ripemd160')
+    h.update(x)
+    return h.digest()
+
+
+def double_sha256(x):
+    """SHA-256 of SHA-256, as used extensively in bitcoin."""
+    return sha256(sha256(x))
+
+
+def hmac_sha512(key, msg):
+    """Use SHA-512 to provide an HMAC."""
+    return _new_hmac(key, msg, _sha512).digest()
+
+
+def hash160(x):
+    """RIPEMD-160 of SHA-256. Used to make bitcoin addresses from pubkeys."""
+    return ripemd160(sha256(x))
+
+
+def hash_to_hex_str(x: bytes) -> str:
+    """Convert a big-endian binary hash to displayed hex string.
+
+    Display form of a binary hash is reversed and converted to hex.
+    """
+    return x[::-1].hex()
+
+
+def hex_str_to_hash(x: str) -> bytes:
+    """Convert a displayed hex string to a binary hash."""
+    return bytes.fromhex(x)[::-1]
+
+
+
+INVALID_REQUEST = -32600
+INVALID_ARGS = -32602
+
+
+class CodeMessageError(Exception):
+
+    @property
+    def code(self):
+        return self.args[0]
+
+    @property
+    def message(self):
+        return self.args[1]
+
+    def __eq__(self, other):
+        return (isinstance(other, self.__class__) and
+                self.code == other.code and self.message == other.message)
+
+    def __hash__(self):
+        # overridden to make the exception hashable
+        # see https://bugs.python.org/issue28603
+        return hash((self.code, self.message))
+
+    @classmethod
+    def invalid_args(cls, message):
+        return cls(INVALID_ARGS, message)
+
+    @classmethod
+    def invalid_request(cls, message):
+        return cls(INVALID_REQUEST, message)
+
+    @classmethod
+    def empty_batch(cls):
+        return cls.invalid_request('batch is empty')
+
+
+class RPCError(CodeMessageError):
+    pass
+
+
+
+class DaemonError(Exception):
+    """Raised when the daemon returns an error in its results."""
+
+
+class WarmingUpError(Exception):
+    """Internal - when the daemon is warming up."""
+
+
+class WorkQueueFullError(Exception):
+    """Internal - when the daemon's work queue is full."""
+
+
+class TaskGroup:
+    def __init__(self, loop=None):
+        self._loop = loop or get_event_loop()
+        self._tasks = set()
+        self.done = Event()
+        self.started = Event()
+
+    def __len__(self):
+        return len(self._tasks)
+
+    def add(self, coro):
+        task = self._loop.create_task(coro)
+        self._tasks.add(task)
+        self.started.set()
+        self.done.clear()
+        task.add_done_callback(self._remove)
+        return task
+
+    def _remove(self, task):
+        self._tasks.remove(task)
+        if len(self._tasks) < 1:
+            self.done.set()
+            self.started.clear()
+
+    def cancel(self):
+        for task in self._tasks:
+            task.cancel()
+        self.done.set()
+        self.started.clear()

scribe/db/__init__.py

@@ -0,0 +1 @@
+from .db import HubDB

scribe/db/common.py

@@ -0,0 +1,526 @@
+import typing
+import enum
+from typing import Optional
+from scribe.error import ResolveCensoredError
+
+
+@enum.unique
+class DB_PREFIXES(enum.Enum):
+    claim_to_support = b'K'
+    support_to_claim = b'L'
+
+    claim_to_txo = b'E'
+    txo_to_claim = b'G'
+
+    claim_to_channel = b'I'
+    channel_to_claim = b'J'
+
+    claim_short_id_prefix = b'F'
+    effective_amount = b'D'
+    claim_expiration = b'O'
+
+    claim_takeover = b'P'
+    pending_activation = b'Q'
+    activated_claim_and_support = b'R'
+    active_amount = b'S'
+
+    repost = b'V'
+    reposted_claim = b'W'
+
+    undo = b'M'
+    touched_or_deleted = b'Y'
+
+    tx = b'B'
+    block_hash = b'C'
+    header = b'H'
+    tx_num = b'N'
+    tx_count = b'T'
+    tx_hash = b'X'
+    utxo = b'u'
+    hashx_utxo = b'h'
+    hashx_history = b'x'
+    db_state = b's'
+    channel_count = b'Z'
+    support_amount = b'a'
+    block_tx = b'b'
+    trending_notifications = b'c'
+    mempool_tx = b'd'
+    touched_hashX = b'e'
+
+
+COLUMN_SETTINGS = {}  # this is updated by the PrefixRow metaclass
+
+
+CLAIM_TYPES = {
+    'stream': 1,
+    'channel': 2,
+    'repost': 3,
+    'collection': 4,
+}
+
+STREAM_TYPES = {
+    'video': 1,
+    'audio': 2,
+    'image': 3,
+    'document': 4,
+    'binary': 5,
+    'model': 6,
+}
+
+# 9/21/2020
+MOST_USED_TAGS = {
+    "gaming",
+    "people & blogs",
+    "entertainment",
+    "music",
+    "pop culture",
+    "education",
+    "technology",
+    "blockchain",
+    "news",
+    "funny",
+    "science & technology",
+    "learning",
+    "gameplay",
+    "news & politics",
+    "comedy",
+    "bitcoin",
+    "beliefs",
+    "nature",
+    "art",
+    "economics",
+    "film & animation",
+    "lets play",
+    "games",
+    "sports",
+    "howto & style",
+    "game",
+    "cryptocurrency",
+    "playstation 4",
+    "automotive",
+    "crypto",
+    "mature",
+    "sony interactive entertainment",
+    "walkthrough",
+    "tutorial",
+    "video game",
+    "weapons",
+    "playthrough",
+    "pc",
+    "anime",
+    "how to",
+    "btc",
+    "fun",
+    "ethereum",
+    "food",
+    "travel & events",
+    "minecraft",
+    "science",
+    "autos & vehicles",
+    "play",
+    "politics",
+    "commentary",
+    "twitch",
+    "ps4live",
+    "love",
+    "ps4",
+    "nonprofits & activism",
+    "ps4share",
+    "fortnite",
+    "xbox",
+    "porn",
+    "video games",
+    "trump",
+    "español",
+    "money",
+    "music video",
+    "nintendo",
+    "movie",
+    "coronavirus",
+    "donald trump",
+    "steam",
+    "trailer",
+    "android",
+    "podcast",
+    "xbox one",
+    "survival",
+    "audio",
+    "linux",
+    "travel",
+    "funny moments",
+    "litecoin",
+    "animation",
+    "gamer",
+    "lets",
+    "playstation",
+    "bitcoin news",
+    "history",
+    "xxx",
+    "fox news",
+    "dance",
+    "god",
+    "adventure",
+    "liberal",
+    "2020",
+    "horror",
+    "government",
+    "freedom",
+    "reaction",
+    "meme",
+    "photography",
+    "truth",
+    "health",
+    "lbry",
+    "family",
+    "online",
+    "eth",
+    "crypto news",
+    "diy",
+    "trading",
+    "gold",
+    "memes",
+    "world",
+    "space",
+    "lol",
+    "covid-19",
+    "rpg",
+    "humor",
+    "democrat",
+    "film",
+    "call of duty",
+    "tech",
+    "religion",
+    "conspiracy",
+    "rap",
+    "cnn",
+    "hangoutsonair",
+    "unboxing",
+    "fiction",
+    "conservative",
+    "cars",
+    "hoa",
+    "epic",
+    "programming",
+    "progressive",
+    "cryptocurrency news",
+    "classical",
+    "jesus",
+    "movies",
+    "book",
+    "ps3",
+    "republican",
+    "fitness",
+    "books",
+    "multiplayer",
+    "animals",
+    "pokemon",
+    "bitcoin price",
+    "facebook",
+    "sharefactory",
+    "criptomonedas",
+    "cod",
+    "bible",
+    "business",
+    "stream",
+    "comics",
+    "how",
+    "fail",
+    "nsfw",
+    "new music",
+    "satire",
+    "pets & animals",
+    "computer",
+    "classical music",
+    "indie",
+    "musica",
+    "msnbc",
+    "fps",
+    "mod",
+    "sport",
+    "sony",
+    "ripple",
+    "auto",
+    "rock",
+    "marvel",
+    "complete",
+    "mining",
+    "political",
+    "mobile",
+    "pubg",
+    "hip hop",
+    "flat earth",
+    "xbox 360",
+    "reviews",
+    "vlogging",
+    "latest news",
+    "hack",
+    "tarot",
+    "iphone",
+    "media",
+    "cute",
+    "christian",
+    "free speech",
+    "trap",
+    "war",
+    "remix",
+    "ios",
+    "xrp",
+    "spirituality",
+    "song",
+    "league of legends",
+    "cat"
+}
+
+MATURE_TAGS = [
+    'nsfw', 'porn', 'xxx', 'mature', 'adult', 'sex'
+]
+
+
+def normalize_tag(tag):
+    return tag.replace(" ", "_").replace("&", "and").replace("-", "_")
+
+
+COMMON_TAGS = {
+    tag: normalize_tag(tag) for tag in list(MOST_USED_TAGS)
+}
+
+INDEXED_LANGUAGES = [
+  'none',
+  'en',
+  'aa',
+  'ab',
+  'ae',
+  'af',
+  'ak',
+  'am',
+  'an',
+  'ar',
+  'as',
+  'av',
+  'ay',
+  'az',
+  'ba',
+  'be',
+  'bg',
+  'bh',
+  'bi',
+  'bm',
+  'bn',
+  'bo',
+  'br',
+  'bs',
+  'ca',
+  'ce',
+  'ch',
+  'co',
+  'cr',
+  'cs',
+  'cu',
+  'cv',
+  'cy',
+  'da',
+  'de',
+  'dv',
+  'dz',
+  'ee',
+  'el',
+  'eo',
+  'es',
+  'et',
+  'eu',
+  'fa',
+  'ff',
+  'fi',
+  'fj',
+  'fo',
+  'fr',
+  'fy',
+  'ga',
+  'gd',
+  'gl',
+  'gn',
+  'gu',
+  'gv',
+  'ha',
+  'he',
+  'hi',
+  'ho',
+  'hr',
+  'ht',
+  'hu',
+  'hy',
+  'hz',
+  'ia',
+  'id',
+  'ie',
+  'ig',
+  'ii',
+  'ik',
+  'io',
+  'is',
+  'it',
+  'iu',
+  'ja',
+  'jv',
+  'ka',
+  'kg',
+  'ki',
+  'kj',
+  'kk',
+  'kl',
+  'km',
+  'kn',
+  'ko',
+  'kr',
+  'ks',
+  'ku',
+  'kv',
+  'kw',
+  'ky',
+  'la',
+  'lb',
+  'lg',
+  'li',
+  'ln',
+  'lo',
+  'lt',
+  'lu',
+  'lv',
+  'mg',
+  'mh',
+  'mi',
+  'mk',
+  'ml',
+  'mn',
+  'mr',
+  'ms',
+  'mt',
+  'my',
+  'na',
+  'nb',
+  'nd',
+  'ne',
+  'ng',
+  'nl',
+  'nn',
+  'no',
+  'nr',
+  'nv',
+  'ny',
+  'oc',
+  'oj',
+  'om',
+  'or',
+  'os',
+  'pa',
+  'pi',
+  'pl',
+  'ps',
+  'pt',
+  'qu',
+  'rm',
+  'rn',
+  'ro',
+  'ru',
+  'rw',
+  'sa',
+  'sc',
+  'sd',
+  'se',
+  'sg',
+  'si',
+  'sk',
+  'sl',
+  'sm',
+  'sn',
+  'so',
+  'sq',
+  'sr',
+  'ss',
+  'st',
+  'su',
+  'sv',
+  'sw',
+  'ta',
+  'te',
+  'tg',
+  'th',
+  'ti',
+  'tk',
+  'tl',
+  'tn',
+  'to',
+  'tr',
+  'ts',
+  'tt',
+  'tw',
+  'ty',
+  'ug',
+  'uk',
+  'ur',
+  'uz',
+  've',
+  'vi',
+  'vo',
+  'wa',
+  'wo',
+  'xh',
+  'yi',
+  'yo',
+  'za',
+  'zh',
+  'zu'
+]
+
+
+class ResolveResult(typing.NamedTuple):
+    name: str
+    normalized_name: str
+    claim_hash: bytes
+    tx_num: int
+    position: int
+    tx_hash: bytes
+    height: int
+    amount: int
+    short_url: str
+    is_controlling: bool
+    canonical_url: str
+    creation_height: int
+    activation_height: int
+    expiration_height: int
+    effective_amount: int
+    support_amount: int
+    reposted: int
+    last_takeover_height: typing.Optional[int]
+    claims_in_channel: typing.Optional[int]
+    channel_hash: typing.Optional[bytes]
+    reposted_claim_hash: typing.Optional[bytes]
+    signature_valid: typing.Optional[bool]
+
+
+class TrendingNotification(typing.NamedTuple):
+    height: int
+    prev_amount: int
+    new_amount: int
+
+
+class UTXO(typing.NamedTuple):
+    tx_num: int
+    tx_pos: int
+    tx_hash: bytes
+    height: int
+    value: int
+
+
+OptionalResolveResultOrError = Optional[typing.Union[ResolveResult, ResolveCensoredError, LookupError, ValueError]]
+
+
+class ExpandedResolveResult(typing.NamedTuple):
+    stream: OptionalResolveResultOrError
+    channel: OptionalResolveResultOrError
+    repost: OptionalResolveResultOrError
+    reposted_channel: OptionalResolveResultOrError
+
+
+class DBError(Exception):
+    """Raised on general DB errors generally indicating corruption."""

scribe/db/interface.py

@@ -0,0 +1,273 @@
+import struct
+import typing
+import rocksdb
+from typing import Optional
+from scribe.db.common import DB_PREFIXES, COLUMN_SETTINGS
+from scribe.db.revertable import RevertableOpStack, RevertablePut, RevertableDelete
+
+
+ROW_TYPES = {}
+
+
+class PrefixRowType(type):
+    def __new__(cls, name, bases, kwargs):
+        klass = super().__new__(cls, name, bases, kwargs)
+        if name != "PrefixRow":
+            ROW_TYPES[klass.prefix] = klass
+            cache_size = klass.cache_size
+            COLUMN_SETTINGS[klass.prefix] = {
+                'cache_size': cache_size,
+            }
+        return klass
+
+
+class PrefixRow(metaclass=PrefixRowType):
+    prefix: bytes
+    key_struct: struct.Struct
+    value_struct: struct.Struct
+    key_part_lambdas = []
+    cache_size: int = 1024 * 1024 * 64
+
+    def __init__(self, db: 'rocksdb.DB', op_stack: RevertableOpStack):
+        self._db = db
+        self._op_stack = op_stack
+        self._column_family = self._db.get_column_family(self.prefix)
+        if not self._column_family.is_valid:
+            raise RuntimeError('column family is not valid')
+
+    def iterate(self, prefix=None, start=None, stop=None, reverse: bool = False, include_key: bool = True,
+                include_value: bool = True, fill_cache: bool = True, deserialize_key: bool = True,
+                deserialize_value: bool = True):
+        if not prefix and not start and not stop:
+            prefix = ()
+        if prefix is not None:
+            prefix = self.pack_partial_key(*prefix)
+            if stop is None:
+                try:
+                   stop = (int.from_bytes(prefix, byteorder='big') + 1).to_bytes(len(prefix), byteorder='big')
+                except OverflowError:
+                    stop = (int.from_bytes(prefix, byteorder='big') + 1).to_bytes(len(prefix) + 1, byteorder='big')
+            else:
+                stop = self.pack_partial_key(*stop)
+        else:
+            if start is not None:
+                start = self.pack_partial_key(*start)
+            if stop is not None:
+                stop = self.pack_partial_key(*stop)
+
+        if deserialize_key:
+            key_getter = lambda k: self.unpack_key(k)
+        else:
+            key_getter = lambda k: k
+        if deserialize_value:
+            value_getter = lambda v: self.unpack_value(v)
+        else:
+            value_getter = lambda v: v
+
+        it = self._db.iterator(
+            start or prefix, self._column_family, iterate_lower_bound=(start or prefix),
+            iterate_upper_bound=stop, reverse=reverse, include_key=include_key,
+            include_value=include_value, fill_cache=fill_cache, prefix_same_as_start=False
+        )
+
+        if include_key and include_value:
+            for k, v in it:
+                yield key_getter(k[1]), value_getter(v)
+        elif include_key:
+            for k in it:
+                yield key_getter(k[1])
+        elif include_value:
+            for v in it:
+                yield value_getter(v)
+        else:
+            for _ in it:
+                yield None
+
+    def get(self, *key_args, fill_cache=True, deserialize_value=True):
+        v = self._db.get((self._column_family, self.pack_key(*key_args)), fill_cache=fill_cache)
+        if v:
+            return v if not deserialize_value else self.unpack_value(v)
+
+    def get_pending(self, *key_args, fill_cache=True, deserialize_value=True):
+        packed_key = self.pack_key(*key_args)
+        last_op = self._op_stack.get_last_op_for_key(packed_key)
+        if last_op:
+            if last_op.is_put:
+                return last_op.value if not deserialize_value else self.unpack_value(last_op.value)
+            else:  # it's a delete
+                return
+        v = self._db.get((self._column_family, packed_key), fill_cache=fill_cache)
+        if v:
+            return v if not deserialize_value else self.unpack_value(v)
+
+    def stage_put(self, key_args=(), value_args=()):
+        self._op_stack.append_op(RevertablePut(self.pack_key(*key_args), self.pack_value(*value_args)))
+
+    def stage_delete(self, key_args=(), value_args=()):
+        self._op_stack.append_op(RevertableDelete(self.pack_key(*key_args), self.pack_value(*value_args)))
+
+    @classmethod
+    def pack_partial_key(cls, *args) -> bytes:
+        return cls.prefix + cls.key_part_lambdas[len(args)](*args)
+
+    @classmethod
+    def pack_key(cls, *args) -> bytes:
+        return cls.prefix + cls.key_struct.pack(*args)
+
+    @classmethod
+    def pack_value(cls, *args) -> bytes:
+        return cls.value_struct.pack(*args)
+
+    @classmethod
+    def unpack_key(cls, key: bytes):
+        assert key[:1] == cls.prefix, f"prefix should be {cls.prefix}, got {key[:1]}"
+        return cls.key_struct.unpack(key[1:])
+
+    @classmethod
+    def unpack_value(cls, data: bytes):
+        return cls.value_struct.unpack(data)
+
+    @classmethod
+    def unpack_item(cls, key: bytes, value: bytes):
+        return cls.unpack_key(key), cls.unpack_value(value)
+
+    def estimate_num_keys(self) -> int:
+        return int(self._db.get_property(b'rocksdb.estimate-num-keys', self._column_family).decode())
+
+
+class BasePrefixDB:
+    """
+    Base class for a revertable rocksdb database (a rocksdb db where each set of applied changes can be undone)
+    """
+    UNDO_KEY_STRUCT = struct.Struct(b'>Q32s')
+    PARTIAL_UNDO_KEY_STRUCT = struct.Struct(b'>Q')
+
+    def __init__(self, path, max_open_files=64, secondary_path='', max_undo_depth: int = 200, unsafe_prefixes=None):
+        column_family_options = {}
+        for prefix in DB_PREFIXES:
+            settings = COLUMN_SETTINGS[prefix.value]
+            column_family_options[prefix.value] = rocksdb.ColumnFamilyOptions()
+            column_family_options[prefix.value].table_factory = rocksdb.BlockBasedTableFactory(
+                block_cache=rocksdb.LRUCache(settings['cache_size']),
+            )
+        self.column_families: typing.Dict[bytes, 'rocksdb.ColumnFamilyHandle'] = {}
+        options = rocksdb.Options(
+            create_if_missing=True, use_fsync=False, target_file_size_base=33554432,
+            max_open_files=max_open_files if not secondary_path else -1, create_missing_column_families=True
+        )
+        self._db = rocksdb.DB(
+            path, options, secondary_name=secondary_path, column_families=column_family_options
+        )
+        for prefix in DB_PREFIXES:
+            cf = self._db.get_column_family(prefix.value)
+            if cf is None and not secondary_path:
+                self._db.create_column_family(prefix.value, column_family_options[prefix.value])
+                cf = self._db.get_column_family(prefix.value)
+            self.column_families[prefix.value] = cf
+
+        self._op_stack = RevertableOpStack(self.get, unsafe_prefixes=unsafe_prefixes)
+        self._max_undo_depth = max_undo_depth
+
+    def unsafe_commit(self):
+        """
+        Write staged changes to the database without keeping undo information
+        Changes written cannot be undone
+        """
+        try:
+            if not len(self._op_stack):
+                return
+            with self._db.write_batch(sync=True) as batch:
+                batch_put = batch.put
+                batch_delete = batch.delete
+                get_column_family = self.column_families.__getitem__
+                for staged_change in self._op_stack:
+                    column_family = get_column_family(DB_PREFIXES(staged_change.key[:1]).value)
+                    if staged_change.is_put:
+                        batch_put((column_family, staged_change.key), staged_change.value)
+                    else:
+                        batch_delete((column_family, staged_change.key))
+        finally:
+            self._op_stack.clear()
+
+    def commit(self, height: int, block_hash: bytes):
+        """
+        Write changes for a block height to the database and keep undo information so that the changes can be reverted
+        """
+        undo_ops = self._op_stack.get_undo_ops()
+        delete_undos = []
+        if height > self._max_undo_depth:
+            delete_undos.extend(self._db.iterator(
+                start=DB_PREFIXES.undo.value + self.PARTIAL_UNDO_KEY_STRUCT.pack(0),
+                iterate_upper_bound=DB_PREFIXES.undo.value + self.PARTIAL_UNDO_KEY_STRUCT.pack(height - self._max_undo_depth),
+                include_value=False
+            ))
+        try:
+            undo_c_f = self.column_families[DB_PREFIXES.undo.value]
+            with self._db.write_batch(sync=True) as batch:
+                batch_put = batch.put
+                batch_delete = batch.delete
+                get_column_family = self.column_families.__getitem__
+                for staged_change in self._op_stack:
+                    column_family = get_column_family(DB_PREFIXES(staged_change.key[:1]).value)
+                    if staged_change.is_put:
+                        batch_put((column_family, staged_change.key), staged_change.value)
+                    else:
+                        batch_delete((column_family, staged_change.key))
+                for undo_to_delete in delete_undos:
+                    batch_delete((undo_c_f, undo_to_delete))
+                batch_put((undo_c_f, DB_PREFIXES.undo.value + self.UNDO_KEY_STRUCT.pack(height, block_hash)), undo_ops)
+        finally:
+            self._op_stack.clear()
+
+    def rollback(self, height: int, block_hash: bytes):
+        """
+        Revert changes for a block height
+        """
+        undo_key = DB_PREFIXES.undo.value + self.UNDO_KEY_STRUCT.pack(height, block_hash)
+        undo_c_f = self.column_families[DB_PREFIXES.undo.value]
+        undo_info = self._db.get((undo_c_f, undo_key))
+        self._op_stack.apply_packed_undo_ops(undo_info)
+        try:
+            with self._db.write_batch(sync=True) as batch:
+                batch_put = batch.put
+                batch_delete = batch.delete
+                get_column_family = self.column_families.__getitem__
+                for staged_change in self._op_stack:
+                    column_family = get_column_family(DB_PREFIXES(staged_change.key[:1]).value)
+                    if staged_change.is_put:
+                        batch_put((column_family, staged_change.key), staged_change.value)
+                    else:
+                        batch_delete((column_family, staged_change.key))
+                # batch_delete(undo_key)
+        finally:
+            self._op_stack.clear()
+
+    def get(self, key: bytes, fill_cache: bool = True) -> Optional[bytes]:
+        cf = self.column_families[key[:1]]
+        return self._db.get((cf, key), fill_cache=fill_cache)
+
+    def iterator(self, start: bytes, column_family: 'rocksdb.ColumnFamilyHandle' = None,
+                 iterate_lower_bound: bytes = None, iterate_upper_bound: bytes = None,
+                 reverse: bool = False, include_key: bool = True, include_value: bool = True,
+                 fill_cache: bool = True, prefix_same_as_start: bool = False, auto_prefix_mode: bool = True):
+        return self._db.iterator(
+            start=start, column_family=column_family, iterate_lower_bound=iterate_lower_bound,
+            iterate_upper_bound=iterate_upper_bound, reverse=reverse, include_key=include_key,
+            include_value=include_value, fill_cache=fill_cache, prefix_same_as_start=prefix_same_as_start,
+            auto_prefix_mode=auto_prefix_mode
+        )
+
+    def close(self):
+        self._db.close()
+
+    def try_catch_up_with_primary(self):
+        self._db.try_catch_up_with_primary()
+
+    def stage_raw_put(self, key: bytes, value: bytes):
+        self._op_stack.append_op(RevertablePut(key, value))
+
+    def stage_raw_delete(self, key: bytes, value: bytes):
+        self._op_stack.append_op(RevertableDelete(key, value))
+
+    def estimate_num_keys(self, column_family: 'rocksdb.ColumnFamilyHandle' = None):
+        return int(self._db.get_property(b'rocksdb.estimate-num-keys', column_family).decode())

scribe/db/merkle.py

@@ -0,0 +1,258 @@
+# Copyright (c) 2018, Neil Booth
+#
+# All rights reserved.
+#
+# The MIT License (MIT)
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# and warranty status of this software.
+
+"""Merkle trees, branches, proofs and roots."""
+
+from asyncio import Event
+from math import ceil, log
+
+from scribe.common import double_sha256
+
+
+class Merkle:
+    """Perform merkle tree calculations on binary hashes using a given hash
+    function.
+
+    If the hash count is not even, the final hash is repeated when
+    calculating the next merkle layer up the tree.
+    """
+
+    def __init__(self, hash_func=double_sha256):
+        self.hash_func = hash_func
+
+    @staticmethod
+    def tree_depth(hash_count):
+        return Merkle.branch_length(hash_count) + 1
+
+    @staticmethod
+    def branch_length(hash_count):
+        """Return the length of a merkle branch given the number of hashes."""
+        if not isinstance(hash_count, int):
+            raise TypeError('hash_count must be an integer')
+        if hash_count < 1:
+            raise ValueError('hash_count must be at least 1')
+        return ceil(log(hash_count, 2))
+
+    @staticmethod
+    def branch_and_root(hashes, index, length=None, hash_func=double_sha256):
+        """Return a (merkle branch, merkle_root) pair given hashes, and the
+        index of one of those hashes.
+        """
+        hashes = list(hashes)
+        if not isinstance(index, int):
+            raise TypeError('index must be an integer')
+        # This also asserts hashes is not empty
+        if not 0 <= index < len(hashes):
+            raise ValueError(f"index '{index}/{len(hashes)}' out of range")
+        natural_length = Merkle.branch_length(len(hashes))
+        if length is None:
+            length = natural_length
+        else:
+            if not isinstance(length, int):
+                raise TypeError('length must be an integer')
+            if length < natural_length:
+                raise ValueError('length out of range')
+
+        branch = []
+        for _ in range(length):
+            if len(hashes) & 1:
+                hashes.append(hashes[-1])
+            branch.append(hashes[index ^ 1])
+            index >>= 1
+            hashes = [hash_func(hashes[n] + hashes[n + 1])
+                      for n in range(0, len(hashes), 2)]
+
+        return branch, hashes[0]
+
+    @staticmethod
+    def root(hashes, length=None):
+        """Return the merkle root of a non-empty iterable of binary hashes."""
+        branch, root = Merkle.branch_and_root(hashes, 0, length)
+        return root
+
+    # @staticmethod
+    # def root_from_proof(hash, branch, index, hash_func=double_sha256):
+    #     """Return the merkle root given a hash, a merkle branch to it, and
+    #     its index in the hashes array.
+    #
+    #     branch is an iterable sorted deepest to shallowest.  If the
+    #     returned root is the expected value then the merkle proof is
+    #     verified.
+    #
+    #     The caller should have confirmed the length of the branch with
+    #     branch_length().  Unfortunately this is not easily done for
+    #     bitcoin transactions as the number of transactions in a block
+    #     is unknown to an SPV client.
+    #     """
+    #     for elt in branch:
+    #         if index & 1:
+    #             hash = hash_func(elt + hash)
+    #         else:
+    #             hash = hash_func(hash + elt)
+    #         index >>= 1
+    #     if index:
+    #         raise ValueError('index out of range for branch')
+    #     return hash
+
+    @staticmethod
+    def level(hashes, depth_higher):
+        """Return a level of the merkle tree of hashes the given depth
+        higher than the bottom row of the original tree."""
+        size = 1 << depth_higher
+        root = Merkle.root
+        return [root(hashes[n: n + size], depth_higher)
+                for n in range(0, len(hashes), size)]
+
+    @staticmethod
+    def branch_and_root_from_level(level, leaf_hashes, index,
+                                   depth_higher):
+        """Return a (merkle branch, merkle_root) pair when a merkle-tree has a
+        level cached.
+
+        To maximally reduce the amount of data hashed in computing a
+        markle branch, cache a tree of depth N at level N // 2.
+
+        level is a list of hashes in the middle of the tree (returned
+        by level())
+
+        leaf_hashes are the leaves needed to calculate a partial branch
+        up to level.
+
+        depth_higher is how much higher level is than the leaves of the tree
+
+        index is the index in the full list of hashes of the hash whose
+        merkle branch we want.
+        """
+        if not isinstance(level, list):
+            raise TypeError("level must be a list")
+        if not isinstance(leaf_hashes, list):
+            raise TypeError("leaf_hashes must be a list")
+        leaf_index = (index >> depth_higher) << depth_higher
+        leaf_branch, leaf_root = Merkle.branch_and_root(
+            leaf_hashes, index - leaf_index, depth_higher)
+        index >>= depth_higher
+        level_branch, root = Merkle.branch_and_root(level, index)
+        # Check last so that we know index is in-range
+        if leaf_root != level[index]:
+            raise ValueError('leaf hashes inconsistent with level')
+        return leaf_branch + level_branch, root
+
+
+class MerkleCache:
+    """A cache to calculate merkle branches efficiently."""
+
+    def __init__(self, merkle, source_func):
+        """Initialise a cache hashes taken from source_func:
+
+           async def source_func(index, count):
+              ...
+        """
+        self.merkle = merkle
+        self.source_func = source_func
+        self.length = 0
+        self.depth_higher = 0
+        self.initialized = Event()
+
+    def _segment_length(self):
+        return 1 << self.depth_higher
+
+    def _leaf_start(self, index):
+        """Given a level's depth higher and a hash index, return the leaf
+        index and leaf hash count needed to calculate a merkle branch.
+        """
+        depth_higher = self.depth_higher
+        return (index >> depth_higher) << depth_higher
+
+    def _level(self, hashes):
+        return self.merkle.level(hashes, self.depth_higher)
+
+    async def _extend_to(self, length):
+        """Extend the length of the cache if necessary."""
+        if length <= self.length:
+            return
+        # Start from the beginning of any final partial segment.
+        # Retain the value of depth_higher; in practice this is fine
+        start = self._leaf_start(self.length)
+        hashes = await self.source_func(start, length - start)
+        self.level[start >> self.depth_higher:] = self._level(hashes)
+        self.length = length
+
+    async def _level_for(self, length):
+        """Return a (level_length, final_hash) pair for a truncation
+        of the hashes to the given length."""
+        if length == self.length:
+            return self.level
+        level = self.level[:length >> self.depth_higher]
+        leaf_start = self._leaf_start(length)
+        count = min(self._segment_length(), length - leaf_start)
+        hashes = await self.source_func(leaf_start, count)
+        level += self._level(hashes)
+        return level
+
+    async def initialize(self, length):
+        """Call to initialize the cache to a source of given length."""
+        self.length = length
+        self.depth_higher = self.merkle.tree_depth(length) // 2
+        self.level = self._level(await self.source_func(0, length))
+        self.initialized.set()
+
+    def truncate(self, length):
+        """Truncate the cache so it covers no more than length underlying
+        hashes."""
+        if not isinstance(length, int):
+            raise TypeError('length must be an integer')
+        if length <= 0:
+            raise ValueError('length must be positive')
+        if length >= self.length:
+            return
+        length = self._leaf_start(length)
+        self.length = length
+        self.level[length >> self.depth_higher:] = []
+
+    async def branch_and_root(self, length, index):
+        """Return a merkle branch and root.  Length is the number of
+        hashes used to calculate the merkle root, index is the position
+        of the hash to calculate the branch of.
+
+        index must be less than length, which must be at least 1."""
+        if not isinstance(length, int):
+            raise TypeError('length must be an integer')
+        if not isinstance(index, int):
+            raise TypeError('index must be an integer')
+        if length <= 0:
+            raise ValueError('length must be positive')
+        if index >= length:
+            raise ValueError('index must be less than length')
+        await self.initialized.wait()
+        await self._extend_to(length)
+        leaf_start = self._leaf_start(index)
+        count = min(self._segment_length(), length - leaf_start)
+        leaf_hashes = await self.source_func(leaf_start, count)
+        if length < self._segment_length():
+            return self.merkle.branch_and_root(leaf_hashes, index)
+        level = await self._level_for(length)
+        return self.merkle.branch_and_root_from_level(
+            level, leaf_hashes, index, self.depth_higher)

scribe/db/revertable.py

@@ -0,0 +1,175 @@
+import struct
+import logging
+from string import printable
+from collections import defaultdict
+from typing import Tuple, Iterable, Callable, Optional
+from scribe.db.common import DB_PREFIXES
+
+_OP_STRUCT = struct.Struct('>BLL')
+log = logging.getLogger(__name__)
+
+
+class RevertableOp:
+    __slots__ = [
+        'key',
+        'value',
+    ]
+    is_put = 0
+
+    def __init__(self, key: bytes, value: bytes):
+        self.key = key
+        self.value = value
+
+    @property
+    def is_delete(self) -> bool:
+        return not self.is_put
+
+    def invert(self) -> 'RevertableOp':
+        raise NotImplementedError()
+
+    def pack(self) -> bytes:
+        """
+        Serialize to bytes
+        """
+        return struct.pack(
+            f'>BLL{len(self.key)}s{len(self.value)}s', int(self.is_put), len(self.key), len(self.value), self.key,
+            self.value
+        )
+
+    @classmethod
+    def unpack(cls, packed: bytes) -> Tuple['RevertableOp', bytes]:
+        """
+        Deserialize from bytes
+
+        :param packed: bytes containing at least one packed revertable op
+        :return: tuple of the deserialized op (a put or a delete) and the remaining serialized bytes
+        """
+        is_put, key_len, val_len = _OP_STRUCT.unpack(packed[:9])
+        key = packed[9:9 + key_len]
+        value = packed[9 + key_len:9 + key_len + val_len]
+        if is_put == 1:
+            return RevertablePut(key, value), packed[9 + key_len + val_len:]
+        return RevertableDelete(key, value), packed[9 + key_len + val_len:]
+
+    def __eq__(self, other: 'RevertableOp') -> bool:
+        return (self.is_put, self.key, self.value) == (other.is_put, other.key, other.value)
+
+    def __repr__(self) -> str:
+        return str(self)
+
+    def __str__(self) -> str:
+        from scribe.db.prefixes import auto_decode_item
+        k, v = auto_decode_item(self.key, self.value)
+        key = ''.join(c if c in printable else '.' for c in str(k))
+        val = ''.join(c if c in printable else '.' for c in str(v))
+        return f"{'PUT' if self.is_put else 'DELETE'} {DB_PREFIXES(self.key[:1]).name}: {key} | {val}"
+
+
+class RevertableDelete(RevertableOp):
+    def invert(self):
+        return RevertablePut(self.key, self.value)
+
+
+class RevertablePut(RevertableOp):
+    is_put = True
+
+    def invert(self):
+        return RevertableDelete(self.key, self.value)
+
+
+class OpStackIntegrity(Exception):
+    pass
+
+
+class RevertableOpStack:
+    def __init__(self, get_fn: Callable[[bytes], Optional[bytes]], unsafe_prefixes=None):
+        """
+        This represents a sequence of revertable puts and deletes to a key-value database that checks for integrity
+        violations when applying the puts and deletes. The integrity checks assure that keys that do not exist
+        are not deleted, and that when keys are deleted the current value is correctly known so that the delete
+        may be undone. When putting values, the integrity checks assure that existing values are not overwritten
+        without first being deleted. Updates are performed by applying a delete op for the old value and a put op
+        for the new value.
+
+        :param get_fn: getter function from an object implementing `KeyValueStorage`
+        :param unsafe_prefixes: optional set of prefixes to ignore integrity errors for, violations are still logged
+        """
+        self._get = get_fn
+        self._items = defaultdict(list)
+        self._unsafe_prefixes = unsafe_prefixes or set()
+
+    def append_op(self, op: RevertableOp):
+        """
+        Apply a put or delete op, checking that it introduces no integrity errors
+        """
+
+        inverted = op.invert()
+        if self._items[op.key] and inverted == self._items[op.key][-1]:
+            self._items[op.key].pop()  # if the new op is the inverse of the last op, we can safely null both
+            return
+        elif self._items[op.key] and self._items[op.key][-1] == op:  # duplicate of last op
+            return  # raise an error?
+        stored_val = self._get(op.key)
+        has_stored_val = stored_val is not None
+        delete_stored_op = None if not has_stored_val else RevertableDelete(op.key, stored_val)
+        will_delete_existing_stored = False if delete_stored_op is None else (delete_stored_op in self._items[op.key])
+        try:
+            if op.is_put and has_stored_val and not will_delete_existing_stored:
+                raise OpStackIntegrity(
+                    f"db op tries to add on top of existing key without deleting first: {op}"
+                )
+            elif op.is_delete and has_stored_val and stored_val != op.value and not will_delete_existing_stored:
+                # there is a value and we're not deleting it in this op
+                # check that a delete for the stored value is in the stack
+                raise OpStackIntegrity(f"db op tries to delete with incorrect existing value {op}")
+            elif op.is_delete and not has_stored_val:
+                raise OpStackIntegrity(f"db op tries to delete nonexistent key: {op}")
+            elif op.is_delete and stored_val != op.value:
+                raise OpStackIntegrity(f"db op tries to delete with incorrect value: {op}")
+        except OpStackIntegrity as err:
+            if op.key[:1] in self._unsafe_prefixes:
+                log.debug(f"skipping over integrity error: {err}")
+            else:
+                raise err
+        self._items[op.key].append(op)
+
+    def extend_ops(self, ops: Iterable[RevertableOp]):
+        """
+        Apply a sequence of put or delete ops, checking that they introduce no integrity errors
+        """
+        for op in ops:
+            self.append_op(op)
+
+    def clear(self):
+        self._items.clear()
+
+    def __len__(self):
+        return sum(map(len, self._items.values()))
+
+    def __iter__(self):
+        for key, ops in self._items.items():
+            for op in ops:
+                yield op
+
+    def __reversed__(self):
+        for key, ops in self._items.items():
+            for op in reversed(ops):
+                yield op
+
+    def get_undo_ops(self) -> bytes:
+        """
+        Get the serialized bytes to undo all of the changes made by the pending ops
+        """
+        return b''.join(op.invert().pack() for op in reversed(self))
+
+    def apply_packed_undo_ops(self, packed: bytes):
+        """
+        Unpack and apply a sequence of undo ops from serialized undo bytes
+        """
+        while packed:
+            op, packed = RevertableOp.unpack(packed)
+            self.append_op(op)
+
+    def get_last_op_for_key(self, key: bytes) -> Optional[RevertableOp]:
+        if key in self._items and self._items[key]:
+            return self._items[key][-1]

scribe/elasticsearch/__init__.py

@@ -0,0 +1,2 @@
+from .search import SearchIndex
+from .notifier_protocol import ElasticNotifierClientProtocol

scribe/elasticsearch/constants.py

@@ -0,0 +1,100 @@
+INDEX_DEFAULT_SETTINGS = {
+    "settings":
+        {"analysis":
+            {"analyzer": {
+                "default": {"tokenizer": "whitespace", "filter": ["lowercase", "porter_stem"]}}},
+            "index":
+                {"refresh_interval": -1,
+                 "number_of_shards": 1,
+                 "number_of_replicas": 0,
+                 "sort": {
+                     "field": ["trending_score", "release_time"],
+                     "order": ["desc", "desc"]
+                 }}
+        },
+    "mappings": {
+        "properties": {
+            "claim_id": {
+                "fields": {
+                    "keyword": {
+                        "ignore_above": 256,
+                        "type": "keyword"
+                    }
+                },
+                "type": "text",
+                "index_prefixes": {
+                    "min_chars": 1,
+                    "max_chars": 10
+                }
+            },
+            "sd_hash": {
+                "fields": {
+                    "keyword": {
+                        "ignore_above": 96,
+                        "type": "keyword"
+                    }
+                },
+                "type": "text",
+                "index_prefixes": {
+                    "min_chars": 1,
+                    "max_chars": 4
+                }
+            },
+            "height": {"type": "integer"},
+            "claim_type": {"type": "byte"},
+            "censor_type": {"type": "byte"},
+            "trending_score": {"type": "double"},
+            "release_time": {"type": "long"}
+        }
+    }
+}
+
+FIELDS = {
+    '_id',
+    'claim_id', 'claim_type', 'claim_name', 'normalized_name',
+    'tx_id', 'tx_nout', 'tx_position',
+    'short_url', 'canonical_url',
+    'is_controlling', 'last_take_over_height',
+    'public_key_bytes', 'public_key_id', 'claims_in_channel',
+    'channel_id', 'signature', 'signature_digest', 'is_signature_valid',
+    'amount', 'effective_amount', 'support_amount',
+    'fee_amount', 'fee_currency',
+    'height', 'creation_height', 'activation_height', 'expiration_height',
+    'stream_type', 'media_type', 'censor_type',
+    'title', 'author', 'description',
+    'timestamp', 'creation_timestamp',
+    'duration', 'release_time',
+    'tags', 'languages', 'has_source', 'reposted_claim_type',
+    'reposted_claim_id', 'repost_count', 'sd_hash',
+    'trending_score', 'tx_num'
+}
+
+TEXT_FIELDS = {'author', 'canonical_url', 'channel_id', 'description', 'claim_id', 'censoring_channel_id',
+               'media_type', 'normalized_name', 'public_key_bytes', 'public_key_id', 'short_url', 'signature',
+               'claim_name', 'signature_digest', 'title', 'tx_id', 'fee_currency', 'reposted_claim_id',
+               'tags', 'sd_hash'}
+
+RANGE_FIELDS = {
+    'height', 'creation_height', 'activation_height', 'expiration_height',
+    'timestamp', 'creation_timestamp', 'duration', 'release_time', 'fee_amount',
+    'tx_position', 'repost_count', 'limit_claims_per_channel',
+    'amount', 'effective_amount', 'support_amount',
+    'trending_score', 'censor_type', 'tx_num'
+}
+
+ALL_FIELDS = RANGE_FIELDS | TEXT_FIELDS | FIELDS
+
+REPLACEMENTS = {
+    'claim_name': 'normalized_name',
+    'name': 'normalized_name',
+    'txid': 'tx_id',
+    'nout': 'tx_nout',
+    'trending_group': 'trending_score',
+    'trending_mixed': 'trending_score',
+    'trending_global': 'trending_score',
+    'trending_local': 'trending_score',
+    'reposted': 'repost_count',
+    'stream_types': 'stream_type',
+    'media_types': 'media_type',
+    'valid_channel_signature': 'is_signature_valid'
+}

scribe/elasticsearch/fast_ar_trending.py

@@ -0,0 +1,117 @@
+FAST_AR_TRENDING_SCRIPT = """
+double softenLBC(double lbc) { return (Math.pow(lbc, 1.0 / 3.0)); }
+
+double logsumexp(double x, double y)
+{
+    double top;
+    if(x > y)
+        top = x;
+    else
+        top = y;
+    double result = top + Math.log(Math.exp(x-top) + Math.exp(y-top));
+    return(result);
+}
+
+double logdiffexp(double big, double small)
+{
+    return big + Math.log(1.0 - Math.exp(small - big));
+}
+
+double squash(double x)
+{
+    if(x < 0.0)
+return -Math.log(1.0 - x);
+    else
+return Math.log(x + 1.0);
+}
+
+double unsquash(double x)
+{
+    if(x < 0.0)
+        return 1.0 - Math.exp(-x);
+    else
+        return Math.exp(x) - 1.0;
+}
+
+double log_to_squash(double x)
+{
+    return logsumexp(x, 0.0);
+}
+
+double squash_to_log(double x)
+{
+    //assert x > 0.0;
+    return logdiffexp(x, 0.0);
+}
+
+double squashed_add(double x, double y)
+{
+    // squash(unsquash(x) + unsquash(y)) but avoiding overflow.
+    // Cases where the signs are the same
+    if (x < 0.0 && y < 0.0)
+        return -logsumexp(-x, logdiffexp(-y, 0.0));
+    if (x >= 0.0 && y >= 0.0)
+        return logsumexp(x, logdiffexp(y, 0.0));
+    // Where the signs differ
+    if (x >= 0.0 && y < 0.0)
+        if (Math.abs(x) >= Math.abs(y))
+            return logsumexp(0.0, logdiffexp(x, -y));
+        else
+            return -logsumexp(0.0, logdiffexp(-y, x));
+    if (x < 0.0 && y >= 0.0)
+    {
+        // Addition is commutative, hooray for new math
+        return squashed_add(y, x);
+    }
+    return 0.0;
+}
+
+double squashed_multiply(double x, double y)
+{
+    // squash(unsquash(x)*unsquash(y)) but avoiding overflow.
+    int sign;
+    if(x*y >= 0.0)
+sign = 1;
+    else
+sign = -1;
+    return sign*logsumexp(squash_to_log(Math.abs(x))
+    + squash_to_log(Math.abs(y)), 0.0);
+}
+
+// Squashed inflated units
+double inflateUnits(int height) {
+    double timescale = 576.0; // Half life of 400 = e-folding time of a day
+      // by coincidence, so may as well go with it
+    return log_to_squash(height / timescale);
+}
+
+double spikePower(double newAmount) {
+    if (newAmount < 50.0) {
+        return(0.5);
+    } else if (newAmount < 85.0) {
+        return(newAmount / 100.0);
+    } else {
+        return(0.85);
+    }
+}
+
+double spikeMass(double oldAmount, double newAmount) {
+    double softenedChange = softenLBC(Math.abs(newAmount - oldAmount));
+    double changeInSoftened = Math.abs(softenLBC(newAmount) - softenLBC(oldAmount));
+    double power = spikePower(newAmount);
+    if (oldAmount > newAmount) {
+        -1.0 * Math.pow(changeInSoftened, power) * Math.pow(softenedChange, 1.0 - power)
+    } else {
+        Math.pow(changeInSoftened, power) * Math.pow(softenedChange, 1.0 - power)
+    }
+}
+
+for (i in params.src.changes) {
+    double units = inflateUnits(i.height);
+    if (ctx._source.trending_score == null) {
+        ctx._source.trending_score = 0.0;
+    }
+    double bigSpike = squashed_multiply(units, squash(spikeMass(i.prev_amount, i.new_amount)));
+    ctx._source.trending_score = squashed_add(ctx._source.trending_score, bigSpike);
+}
+"""

scribe/elasticsearch/notifier_protocol.py

@@ -0,0 +1,55 @@
+import typing
+import struct
+import asyncio
+import logging
+
+
+log = logging.getLogger(__name__)
+
+
+class ElasticNotifierProtocol(asyncio.Protocol):
+    """notifies the reader when ES has written updates"""
+
+    def __init__(self, listeners):
+        self._listeners = listeners
+        self.transport: typing.Optional[asyncio.Transport] = None
+
+    def connection_made(self, transport):
+        self.transport = transport
+        self._listeners.append(self)
+        log.info("got es notifier connection")
+
+    def connection_lost(self, exc) -> None:
+        self._listeners.remove(self)
+        self.transport = None
+
+    def send_height(self, height: int, block_hash: bytes):
+        log.info("notify es update '%s'", height)
+        self.transport.write(struct.pack(b'>Q32s', height, block_hash))
+
+
+class ElasticNotifierClientProtocol(asyncio.Protocol):
+    """notifies the reader when ES has written updates"""
+
+    def __init__(self, notifications: asyncio.Queue):
+        self.notifications = notifications
+        self.transport: typing.Optional[asyncio.Transport] = None
+
+    def close(self):
+        if self.transport and not self.transport.is_closing():
+            self.transport.close()
+
+    def connection_made(self, transport):
+        self.transport = transport
+        log.info("connected to es notifier")
+
+    def connection_lost(self, exc) -> None:
+        self.transport = None
+
+    def data_received(self, data: bytes) -> None:
+        try:
+            height, block_hash = struct.unpack(b'>Q32s', data)
+        except:
+            log.exception("failed to decode %s", (data or b'').hex())
+            raise
+        self.notifications.put_nowait((height, block_hash))

scribe/elasticsearch/search.py

@@ -0,0 +1,870 @@
+import logging
+import time
+import asyncio
+import struct
+from binascii import unhexlify
+from collections import Counter, deque
+from decimal import Decimal
+from operator import itemgetter
+from typing import Optional, List, Iterable
+
+from elasticsearch import AsyncElasticsearch, NotFoundError, ConnectionError
+from elasticsearch.helpers import async_streaming_bulk
+from scribe.schema.result import Outputs, Censor
+from scribe.schema.tags import clean_tags
+from scribe.schema.url import normalize_name
+from scribe.error import TooManyClaimSearchParametersError
+from scribe.common import LRUCache
+from scribe.db.common import CLAIM_TYPES, STREAM_TYPES
+from scribe.elasticsearch.constants import INDEX_DEFAULT_SETTINGS, REPLACEMENTS, FIELDS, TEXT_FIELDS, \
+    RANGE_FIELDS, ALL_FIELDS
+from scribe.db.common import ResolveResult
+
+
+def expand_query(**kwargs):
+    if "amount_order" in kwargs:
+        kwargs["limit"] = 1
+        kwargs["order_by"] = "effective_amount"
+        kwargs["offset"] = int(kwargs["amount_order"]) - 1
+    if 'name' in kwargs:
+        kwargs['name'] = normalize_name(kwargs.pop('name'))
+    if kwargs.get('is_controlling') is False:
+        kwargs.pop('is_controlling')
+    query = {'must': [], 'must_not': []}
+    collapse = None
+    if 'fee_currency' in kwargs and kwargs['fee_currency'] is not None:
+        kwargs['fee_currency'] = kwargs['fee_currency'].upper()
+    for key, value in kwargs.items():
+        key = key.replace('claim.', '')
+        many = key.endswith('__in') or isinstance(value, list)
+        if many and len(value) > 2048:
+            raise TooManyClaimSearchParametersError(key, 2048)
+        if many:
+            key = key.replace('__in', '')
+            value = list(filter(None, value))
+        if value is None or isinstance(value, list) and len(value) == 0:
+            continue
+        key = REPLACEMENTS.get(key, key)
+        if key in FIELDS:
+            partial_id = False
+            if key == 'claim_type':
+                if isinstance(value, str):
+                    value = CLAIM_TYPES[value]
+                else:
+                    value = [CLAIM_TYPES[claim_type] for claim_type in value]
+            elif key == 'stream_type':
+                value = [STREAM_TYPES[value]] if isinstance(value, str) else list(map(STREAM_TYPES.get, value))
+            if key == '_id':
+                if isinstance(value, Iterable):
+                    value = [item[::-1].hex() for item in value]
+                else:
+                    value = value[::-1].hex()
+            if not many and key in ('_id', 'claim_id') and len(value) < 20:
+                partial_id = True
+            if key in ('signature_valid', 'has_source'):
+                continue  # handled later
+            if key in TEXT_FIELDS:
+                key += '.keyword'
+            ops = {'<=': 'lte', '>=': 'gte', '<': 'lt', '>': 'gt'}
+            if partial_id:
+                query['must'].append({"prefix": {"claim_id": value}})
+            elif key in RANGE_FIELDS and isinstance(value, str) and value[0] in ops:
+                operator_length = 2 if value[:2] in ops else 1
+                operator, value = value[:operator_length], value[operator_length:]
+                if key == 'fee_amount':
+                    value = str(Decimal(value)*1000)
+                query['must'].append({"range": {key: {ops[operator]: value}}})
+            elif many:
+                query['must'].append({"terms": {key: value}})
+            else:
+                if key == 'fee_amount':
+                    value = str(Decimal(value)*1000)
+                query['must'].append({"term": {key: {"value": value}}})
+        elif key == 'not_channel_ids':
+            for channel_id in value:
+                query['must_not'].append({"term": {'channel_id.keyword': channel_id}})
+                query['must_not'].append({"term": {'_id': channel_id}})
+        elif key == 'channel_ids':
+            query['must'].append({"terms": {'channel_id.keyword': value}})
+        elif key == 'claim_ids':
+            query['must'].append({"terms": {'claim_id.keyword': value}})
+        elif key == 'media_types':
+            query['must'].append({"terms": {'media_type.keyword': value}})
+        elif key == 'any_languages':
+            query['must'].append({"terms": {'languages': clean_tags(value)}})
+        elif key == 'any_languages':
+            query['must'].append({"terms": {'languages': value}})
+        elif key == 'all_languages':
+            query['must'].extend([{"term": {'languages': tag}} for tag in value])
+        elif key == 'any_tags':
+            query['must'].append({"terms": {'tags.keyword': clean_tags(value)}})
+        elif key == 'all_tags':
+            query['must'].extend([{"term": {'tags.keyword': tag}} for tag in clean_tags(value)])
+        elif key == 'not_tags':
+            query['must_not'].extend([{"term": {'tags.keyword': tag}} for tag in clean_tags(value)])
+        elif key == 'not_claim_id':
+            query['must_not'].extend([{"term": {'claim_id.keyword': cid}} for cid in value])
+        elif key == 'limit_claims_per_channel':
+            collapse = ('channel_id.keyword', value)
+    if kwargs.get('has_channel_signature'):
+        query['must'].append({"exists": {"field": "signature"}})
+        if 'signature_valid' in kwargs:
+            query['must'].append({"term": {"is_signature_valid": bool(kwargs["signature_valid"])}})
+    elif 'signature_valid' in kwargs:
+        query.setdefault('should', [])
+        query["minimum_should_match"] = 1
+        query['should'].append({"bool": {"must_not": {"exists": {"field": "signature"}}}})
+        query['should'].append({"term": {"is_signature_valid": bool(kwargs["signature_valid"])}})
+    if 'has_source' in kwargs:
+        query.setdefault('should', [])
+        query["minimum_should_match"] = 1
+        is_stream_or_repost = {"terms": {"claim_type": [CLAIM_TYPES['stream'], CLAIM_TYPES['repost']]}}
+        query['should'].append(
+            {"bool": {"must": [{"match": {"has_source": kwargs['has_source']}}, is_stream_or_repost]}})
+        query['should'].append({"bool": {"must_not": [is_stream_or_repost]}})
+        query['should'].append({"bool": {"must": [{"term": {"reposted_claim_type": CLAIM_TYPES['channel']}}]}})
+    if kwargs.get('text'):
+        query['must'].append(
+                    {"simple_query_string":
+                         {"query": kwargs["text"], "fields": [
+                             "claim_name^4", "channel_name^8", "title^1", "description^.5", "author^1", "tags^.5"
+                         ]}})
+    query = {
+        "_source": {"excludes": ["description", "title"]},
+        'query': {'bool': query},
+        "sort": [],
+    }
+    if "limit" in kwargs:
+        query["size"] = kwargs["limit"]
+    if 'offset' in kwargs:
+        query["from"] = kwargs["offset"]
+    if 'order_by' in kwargs:
+        if isinstance(kwargs["order_by"], str):
+            kwargs["order_by"] = [kwargs["order_by"]]
+        for value in kwargs['order_by']:
+            if 'trending_group' in value:
+                # fixme: trending_mixed is 0 for all records on variable decay, making sort slow.
+                continue
+            is_asc = value.startswith('^')
+            value = value[1:] if is_asc else value
+            value = REPLACEMENTS.get(value, value)
+            if value in TEXT_FIELDS:
+                value += '.keyword'
+            query['sort'].append({value: "asc" if is_asc else "desc"})
+    if collapse:
+        query["collapse"] = {
+            "field": collapse[0],
+            "inner_hits": {
+                "name": collapse[0],
+                "size": collapse[1],
+                "sort": query["sort"]
+            }
+        }
+    return query
+
+
+
+class ChannelResolution(str):
+    @classmethod
+    def lookup_error(cls, url):
+        return LookupError(f'Could not find channel in "{url}".')
+
+
+class StreamResolution(str):
+    @classmethod
+    def lookup_error(cls, url):
+        return LookupError(f'Could not find claim at "{url}".')
+
+
+class IndexVersionMismatch(Exception):
+    def __init__(self, got_version, expected_version):
+        self.got_version = got_version
+        self.expected_version = expected_version
+
+
+class SearchIndex:
+    VERSION = 1
+
+    def __init__(self, index_prefix: str, search_timeout=3.0, elastic_host='localhost', elastic_port=9200):
+        self.search_timeout = search_timeout
+        self.sync_timeout = 600  # wont hit that 99% of the time, but can hit on a fresh import
+        self.search_client: Optional[AsyncElasticsearch] = None
+        self.sync_client: Optional[AsyncElasticsearch] = None
+        self.index = index_prefix + 'claims'
+        self.logger = logging.getLogger(__name__)
+        self.claim_cache = LRUCache(2 ** 15)
+        self.search_cache = LRUCache(2 ** 17)
+        self._elastic_host = elastic_host
+        self._elastic_port = elastic_port
+
+    async def get_index_version(self) -> int:
+        try:
+            template = await self.sync_client.indices.get_template(self.index)
+            return template[self.index]['version']
+        except NotFoundError:
+            return 0
+
+    async def set_index_version(self, version):
+        await self.sync_client.indices.put_template(
+            self.index, body={'version': version, 'index_patterns': ['ignored']}, ignore=400
+        )
+
+    async def start(self) -> bool:
+        if self.sync_client:
+            return False
+        hosts = [{'host': self._elastic_host, 'port': self._elastic_port}]
+        self.sync_client = AsyncElasticsearch(hosts, timeout=self.sync_timeout)
+        self.search_client = AsyncElasticsearch(hosts, timeout=self.search_timeout)
+        while True:
+            try:
+                await self.sync_client.cluster.health(wait_for_status='yellow')
+                break
+            except ConnectionError:
+                self.logger.warning("Failed to connect to Elasticsearch. Waiting for it!")
+                await asyncio.sleep(1)
+
+        res = await self.sync_client.indices.create(self.index, INDEX_DEFAULT_SETTINGS, ignore=400)
+        acked = res.get('acknowledged', False)
+        if acked:
+            await self.set_index_version(self.VERSION)
+            return acked
+        index_version = await self.get_index_version()
+        if index_version != self.VERSION:
+            self.logger.error("es search index has an incompatible version: %s vs %s", index_version, self.VERSION)
+            raise IndexVersionMismatch(index_version, self.VERSION)
+        await self.sync_client.indices.refresh(self.index)
+        return acked
+
+    async def stop(self):
+        clients = [c for c in (self.sync_client, self.search_client) if c is not None]
+        self.sync_client, self.search_client = None, None
+        if clients:
+            await asyncio.gather(*(client.close() for client in clients))
+
+    def delete_index(self):
+        return self.sync_client.indices.delete(self.index, ignore_unavailable=True)
+
+    async def _consume_claim_producer(self, claim_producer):
+        count = 0
+        async for op, doc in claim_producer:
+            if op == 'delete':
+                yield {
+                    '_index': self.index,
+                    '_op_type': 'delete',
+                    '_id': doc
+                }
+            else:
+                yield {
+                    'doc': {key: value for key, value in doc.items() if key in ALL_FIELDS},
+                    '_id': doc['claim_id'],
+                    '_index': self.index,
+                    '_op_type': 'update',
+                    'doc_as_upsert': True
+                }
+            count += 1
+            if count % 100 == 0:
+                self.logger.info("Indexing in progress, %d claims.", count)
+        if count:
+            self.logger.info("Indexing done for %d claims.", count)
+        else:
+            self.logger.debug("Indexing done for %d claims.", count)
+
+    async def claim_consumer(self, claim_producer):
+        touched = set()
+        async for ok, item in async_streaming_bulk(self.sync_client, self._consume_claim_producer(claim_producer),
+                                                   raise_on_error=False):
+            if not ok:
+                self.logger.warning("indexing failed for an item: %s", item)
+            else:
+                item = item.popitem()[1]
+                touched.add(item['_id'])
+        await self.sync_client.indices.refresh(self.index)
+        self.logger.debug("Indexing done.")
+
+    def update_filter_query(self, censor_type, blockdict, channels=False):
+        blockdict = {blocked.hex(): blocker.hex() for blocked, blocker in blockdict.items()}
+        if channels:
+            update = expand_query(channel_id__in=list(blockdict.keys()), censor_type=f"<{censor_type}")
+        else:
+            update = expand_query(claim_id__in=list(blockdict.keys()), censor_type=f"<{censor_type}")
+        key = 'channel_id' if channels else 'claim_id'
+        update['script'] = {
+            "source": f"ctx._source.censor_type={censor_type}; "
+                      f"ctx._source.censoring_channel_id=params[ctx._source.{key}];",
+            "lang": "painless",
+            "params": blockdict
+        }
+        return update
+
+    async def update_trending_score(self, params):
+        update_trending_score_script = """
+        double softenLBC(double lbc) { return (Math.pow(lbc, 1.0 / 3.0)); }
+
+        double logsumexp(double x, double y)
+        {
+            double top;
+            if(x > y)
+                top = x;
+            else
+                top = y;
+            double result = top + Math.log(Math.exp(x-top) + Math.exp(y-top));
+            return(result);
+        }
+
+        double logdiffexp(double big, double small)
+        {
+            return big + Math.log(1.0 - Math.exp(small - big));
+        }
+
+        double squash(double x)
+        {
+            if(x < 0.0)
+                return -Math.log(1.0 - x);
+            else
+                return Math.log(x + 1.0);
+        }
+
+        double unsquash(double x)
+        {
+            if(x < 0.0)
+                return 1.0 - Math.exp(-x);
+            else
+                return Math.exp(x) - 1.0;
+        }
+
+        double log_to_squash(double x)
+        {
+            return logsumexp(x, 0.0);
+        }
+
+        double squash_to_log(double x)
+        {
+            //assert x > 0.0;
+            return logdiffexp(x, 0.0);
+        }
+
+        double squashed_add(double x, double y)
+        {
+            // squash(unsquash(x) + unsquash(y)) but avoiding overflow.
+            // Cases where the signs are the same
+            if (x < 0.0 && y < 0.0)
+                return -logsumexp(-x, logdiffexp(-y, 0.0));
+            if (x >= 0.0 && y >= 0.0)
+                return logsumexp(x, logdiffexp(y, 0.0));
+            // Where the signs differ
+            if (x >= 0.0 && y < 0.0)
+                if (Math.abs(x) >= Math.abs(y))
+                    return logsumexp(0.0, logdiffexp(x, -y));
+                else
+                    return -logsumexp(0.0, logdiffexp(-y, x));
+            if (x < 0.0 && y >= 0.0)
+            {
+                // Addition is commutative, hooray for new math
+                return squashed_add(y, x);
+            }
+            return 0.0;
+        }
+
+        double squashed_multiply(double x, double y)
+        {
+            // squash(unsquash(x)*unsquash(y)) but avoiding overflow.
+            int sign;
+            if(x*y >= 0.0)
+                sign = 1;
+            else
+                sign = -1;
+            return sign*logsumexp(squash_to_log(Math.abs(x))
+                            + squash_to_log(Math.abs(y)), 0.0);
+        }
+
+        // Squashed inflated units
+        double inflateUnits(int height) {
+            double timescale = 576.0; // Half life of 400 = e-folding time of a day
+                                      // by coincidence, so may as well go with it
+            return log_to_squash(height / timescale);
+        }
+
+        double spikePower(double newAmount) {
+            if (newAmount < 50.0) {
+                return(0.5);
+            } else if (newAmount < 85.0) {
+                return(newAmount / 100.0);
+            } else {
+                return(0.85);
+            }
+        }
+
+        double spikeMass(double oldAmount, double newAmount) {
+            double softenedChange = softenLBC(Math.abs(newAmount - oldAmount));
+            double changeInSoftened = Math.abs(softenLBC(newAmount) - softenLBC(oldAmount));
+            double power = spikePower(newAmount);
+            if (oldAmount > newAmount) {
+                -1.0 * Math.pow(changeInSoftened, power) * Math.pow(softenedChange, 1.0 - power)
+            } else {
+                Math.pow(changeInSoftened, power) * Math.pow(softenedChange, 1.0 - power)
+            }
+        }
+        for (i in params.src.changes) {
+            double units = inflateUnits(i.height);
+            if (ctx._source.trending_score == null) {
+                ctx._source.trending_score = 0.0;
+            }
+            double bigSpike = squashed_multiply(units, squash(spikeMass(i.prev_amount, i.new_amount)));
+            ctx._source.trending_score = squashed_add(ctx._source.trending_score, bigSpike);
+        }
+        """
+        start = time.perf_counter()
+
+        def producer():
+            for claim_id, claim_updates in params.items():
+                yield {
+                    '_id': claim_id,
+                    '_index': self.index,
+                    '_op_type': 'update',
+                    'script': {
+                        'lang': 'painless',
+                        'source': update_trending_score_script,
+                        'params': {'src': {
+                            'changes': [
+                                {
+                                    'height': p.height,
+                                    'prev_amount': p.prev_amount / 1E8,
+                                    'new_amount': p.new_amount / 1E8,
+                                } for p in claim_updates
+                            ]
+                        }}
+                    },
+                }
+        if not params:
+            return
+        async for ok, item in async_streaming_bulk(self.sync_client, producer(), raise_on_error=False):
+            if not ok:
+                self.logger.warning("updating trending failed for an item: %s", item)
+        await self.sync_client.indices.refresh(self.index)
+        self.logger.info("updated trending scores in %ims", int((time.perf_counter() - start) * 1000))
+
+    async def apply_filters(self, blocked_streams, blocked_channels, filtered_streams, filtered_channels):
+        if filtered_streams:
+            await self.sync_client.update_by_query(
+                self.index, body=self.update_filter_query(Censor.SEARCH, filtered_streams), slices=4)
+            await self.sync_client.indices.refresh(self.index)
+        if filtered_channels:
+            await self.sync_client.update_by_query(
+                self.index, body=self.update_filter_query(Censor.SEARCH, filtered_channels), slices=4)
+            await self.sync_client.indices.refresh(self.index)
+            await self.sync_client.update_by_query(
+                self.index, body=self.update_filter_query(Censor.SEARCH, filtered_channels, True), slices=4)
+            await self.sync_client.indices.refresh(self.index)
+        if blocked_streams:
+            await self.sync_client.update_by_query(
+                self.index, body=self.update_filter_query(Censor.RESOLVE, blocked_streams), slices=4)
+            await self.sync_client.indices.refresh(self.index)
+        if blocked_channels:
+            await self.sync_client.update_by_query(
+                self.index, body=self.update_filter_query(Censor.RESOLVE, blocked_channels), slices=4)
+            await self.sync_client.indices.refresh(self.index)
+            await self.sync_client.update_by_query(
+                self.index, body=self.update_filter_query(Censor.RESOLVE, blocked_channels, True), slices=4)
+            await self.sync_client.indices.refresh(self.index)
+        self.clear_caches()
+
+    def clear_caches(self):
+        self.search_cache.clear()
+        self.claim_cache.clear()
+
+    async def cached_search(self, kwargs):
+        total_referenced = []
+        cache_item = ResultCacheItem.from_cache(str(kwargs), self.search_cache)
+        if cache_item.result is not None:
+            return cache_item.result
+        async with cache_item.lock:
+            if cache_item.result:
+                return cache_item.result
+            censor = Censor(Censor.SEARCH)
+            if kwargs.get('no_totals'):
+                response, offset, total = await self.search(**kwargs, censor_type=Censor.NOT_CENSORED)
+            else:
+                response, offset, total = await self.search(**kwargs)
+            censor.apply(response)
+            total_referenced.extend(response)
+
+            if censor.censored:
+                response, _, _ = await self.search(**kwargs, censor_type=Censor.NOT_CENSORED)
+                total_referenced.extend(response)
+            response = [
+                ResolveResult(
+                    name=r['claim_name'],
+                    normalized_name=r['normalized_name'],
+                    claim_hash=r['claim_hash'],
+                    tx_num=r['tx_num'],
+                    position=r['tx_nout'],
+                    tx_hash=r['tx_hash'],
+                    height=r['height'],
+                    amount=r['amount'],
+                    short_url=r['short_url'],
+                    is_controlling=r['is_controlling'],
+                    canonical_url=r['canonical_url'],
+                    creation_height=r['creation_height'],
+                    activation_height=r['activation_height'],
+                    expiration_height=r['expiration_height'],
+                    effective_amount=r['effective_amount'],
+                    support_amount=r['support_amount'],
+                    last_takeover_height=r['last_take_over_height'],
+                    claims_in_channel=r['claims_in_channel'],
+                    channel_hash=r['channel_hash'],
+                    reposted_claim_hash=r['reposted_claim_hash'],
+                    reposted=r['reposted'],
+                    signature_valid=r['signature_valid']
+                ) for r in response
+            ]
+            extra = [
+                ResolveResult(
+                    name=r['claim_name'],
+                    normalized_name=r['normalized_name'],
+                    claim_hash=r['claim_hash'],
+                    tx_num=r['tx_num'],
+                    position=r['tx_nout'],
+                    tx_hash=r['tx_hash'],
+                    height=r['height'],
+                    amount=r['amount'],
+                    short_url=r['short_url'],
+                    is_controlling=r['is_controlling'],
+                    canonical_url=r['canonical_url'],
+                    creation_height=r['creation_height'],
+                    activation_height=r['activation_height'],
+                    expiration_height=r['expiration_height'],
+                    effective_amount=r['effective_amount'],
+                    support_amount=r['support_amount'],
+                    last_takeover_height=r['last_take_over_height'],
+                    claims_in_channel=r['claims_in_channel'],
+                    channel_hash=r['channel_hash'],
+                    reposted_claim_hash=r['reposted_claim_hash'],
+                    reposted=r['reposted'],
+                    signature_valid=r['signature_valid']
+                ) for r in await self._get_referenced_rows(total_referenced)
+            ]
+            result = Outputs.to_base64(
+                response, extra, offset, total, censor
+            )
+            cache_item.result = result
+            return result
+
+    async def get_many(self, *claim_ids):
+        await self.populate_claim_cache(*claim_ids)
+        return filter(None, map(self.claim_cache.get, claim_ids))
+
+    async def populate_claim_cache(self, *claim_ids):
+        missing = [claim_id for claim_id in claim_ids if self.claim_cache.get(claim_id) is None]
+        if missing:
+            results = await self.search_client.mget(
+                index=self.index, body={"ids": missing}
+            )
+            for result in expand_result(filter(lambda doc: doc['found'], results["docs"])):
+                self.claim_cache.set(result['claim_id'], result)
+
+
+    async def search(self, **kwargs):
+        try:
+            return await self.search_ahead(**kwargs)
+        except NotFoundError:
+            return [], 0, 0
+        # return expand_result(result['hits']), 0, result.get('total', {}).get('value', 0)
+
+    async def search_ahead(self, **kwargs):
+        # 'limit_claims_per_channel' case. Fetch 1000 results, reorder, slice, inflate and return
+        per_channel_per_page = kwargs.pop('limit_claims_per_channel', 0) or 0
+        remove_duplicates = kwargs.pop('remove_duplicates', False)
+        page_size = kwargs.pop('limit', 10)
+        offset = kwargs.pop('offset', 0)
+        kwargs['limit'] = 1000
+        cache_item = ResultCacheItem.from_cache(f"ahead{per_channel_per_page}{kwargs}", self.search_cache)
+        if cache_item.result is not None:
+            reordered_hits = cache_item.result
+        else:
+            async with cache_item.lock:
+                if cache_item.result:
+                    reordered_hits = cache_item.result
+                else:
+                    query = expand_query(**kwargs)
+                    search_hits = deque((await self.search_client.search(
+                        query, index=self.index, track_total_hits=False,
+                        _source_includes=['_id', 'channel_id', 'reposted_claim_id', 'creation_height']
+                    ))['hits']['hits'])
+                    if remove_duplicates:
+                        search_hits = self.__remove_duplicates(search_hits)
+                    if per_channel_per_page > 0:
+                        reordered_hits = self.__search_ahead(search_hits, page_size, per_channel_per_page)
+                    else:
+                        reordered_hits = [(hit['_id'], hit['_source']['channel_id']) for hit in search_hits]
+                    cache_item.result = reordered_hits
+        result = list(await self.get_many(*(claim_id for claim_id, _ in reordered_hits[offset:(offset + page_size)])))
+        return result, 0, len(reordered_hits)
+
+    def __remove_duplicates(self, search_hits: deque) -> deque:
+        known_ids = {}  # claim_id -> (creation_height, hit_id), where hit_id is either reposted claim id or original
+        dropped = set()
+        for hit in search_hits:
+            hit_height, hit_id = hit['_source']['creation_height'], hit['_source']['reposted_claim_id'] or hit['_id']
+            if hit_id not in known_ids:
+                known_ids[hit_id] = (hit_height, hit['_id'])
+            else:
+                previous_height, previous_id = known_ids[hit_id]
+                if hit_height < previous_height:
+                    known_ids[hit_id] = (hit_height, hit['_id'])
+                    dropped.add(previous_id)
+                else:
+                    dropped.add(hit['_id'])
+        return deque(hit for hit in search_hits if hit['_id'] not in dropped)
+
+    def __search_ahead(self, search_hits: list, page_size: int, per_channel_per_page: int):
+        reordered_hits = []
+        channel_counters = Counter()
+        next_page_hits_maybe_check_later = deque()
+        while search_hits or next_page_hits_maybe_check_later:
+            if reordered_hits and len(reordered_hits) % page_size == 0:
+                channel_counters.clear()
+            elif not reordered_hits:
+                pass
+            else:
+                break  # means last page was incomplete and we are left with bad replacements
+            for _ in range(len(next_page_hits_maybe_check_later)):
+                claim_id, channel_id = next_page_hits_maybe_check_later.popleft()
+                if per_channel_per_page > 0 and channel_counters[channel_id] < per_channel_per_page:
+                    reordered_hits.append((claim_id, channel_id))
+                    channel_counters[channel_id] += 1
+                else:
+                    next_page_hits_maybe_check_later.append((claim_id, channel_id))
+            while search_hits:
+                hit = search_hits.popleft()
+                hit_id, hit_channel_id = hit['_id'], hit['_source']['channel_id']
+                if hit_channel_id is None or per_channel_per_page <= 0:
+                    reordered_hits.append((hit_id, hit_channel_id))
+                elif channel_counters[hit_channel_id] < per_channel_per_page:
+                    reordered_hits.append((hit_id, hit_channel_id))
+                    channel_counters[hit_channel_id] += 1
+                    if len(reordered_hits) % page_size == 0:
+                        break
+                else:
+                    next_page_hits_maybe_check_later.append((hit_id, hit_channel_id))
+        return reordered_hits
+
+    async def _get_referenced_rows(self, txo_rows: List[dict]):
+        txo_rows = [row for row in txo_rows if isinstance(row, dict)]
+        referenced_ids = set(filter(None, map(itemgetter('reposted_claim_id'), txo_rows)))
+        referenced_ids |= set(filter(None, (row['channel_id'] for row in txo_rows)))
+        referenced_ids |= set(filter(None, (row['censoring_channel_id'] for row in txo_rows)))
+
+        referenced_txos = []
+        if referenced_ids:
+            referenced_txos.extend(await self.get_many(*referenced_ids))
+            referenced_ids = set(filter(None, (row['channel_id'] for row in referenced_txos)))
+
+        if referenced_ids:
+            referenced_txos.extend(await self.get_many(*referenced_ids))
+
+        return referenced_txos
+
+
+def expand_query(**kwargs):
+    if "amount_order" in kwargs:
+        kwargs["limit"] = 1
+        kwargs["order_by"] = "effective_amount"
+        kwargs["offset"] = int(kwargs["amount_order"]) - 1
+    if 'name' in kwargs:
+        kwargs['name'] = normalize_name(kwargs.pop('name'))
+    if kwargs.get('is_controlling') is False:
+        kwargs.pop('is_controlling')
+    query = {'must': [], 'must_not': []}
+    collapse = None
+    if 'fee_currency' in kwargs and kwargs['fee_currency'] is not None:
+        kwargs['fee_currency'] = kwargs['fee_currency'].upper()
+    for key, value in kwargs.items():
+        key = key.replace('claim.', '')
+        many = key.endswith('__in') or isinstance(value, list)
+        if many and len(value) > 2048:
+            raise TooManyClaimSearchParametersError(key, 2048)
+        if many:
+            key = key.replace('__in', '')
+            value = list(filter(None, value))
+        if value is None or isinstance(value, list) and len(value) == 0:
+            continue
+        key = REPLACEMENTS.get(key, key)
+        if key in FIELDS:
+            partial_id = False
+            if key == 'claim_type':
+                if isinstance(value, str):
+                    value = CLAIM_TYPES[value]
+                else:
+                    value = [CLAIM_TYPES[claim_type] for claim_type in value]
+            elif key == 'stream_type':
+                value = [STREAM_TYPES[value]] if isinstance(value, str) else list(map(STREAM_TYPES.get, value))
+            if key == '_id':
+                if isinstance(value, Iterable):
+                    value = [item[::-1].hex() for item in value]
+                else:
+                    value = value[::-1].hex()
+            if not many and key in ('_id', 'claim_id', 'sd_hash') and len(value) < 20:
+                partial_id = True
+            if key in ('signature_valid', 'has_source'):
+                continue  # handled later
+            if key in TEXT_FIELDS:
+                key += '.keyword'
+            ops = {'<=': 'lte', '>=': 'gte', '<': 'lt', '>': 'gt'}
+            if partial_id:
+                query['must'].append({"prefix": {key: value}})
+            elif key in RANGE_FIELDS and isinstance(value, str) and value[0] in ops:
+                operator_length = 2 if value[:2] in ops else 1
+                operator, value = value[:operator_length], value[operator_length:]
+                if key == 'fee_amount':
+                    value = str(Decimal(value)*1000)
+                query['must'].append({"range": {key: {ops[operator]: value}}})
+            elif key in RANGE_FIELDS and isinstance(value, list) and all(v[0] in ops for v in value):
+                range_constraints = []
+                for v in value:
+                    operator_length = 2 if v[:2] in ops else 1
+                    operator, stripped_op_v = v[:operator_length], v[operator_length:]
+                    if key == 'fee_amount':
+                        stripped_op_v = str(Decimal(stripped_op_v)*1000)
+                    range_constraints.append((operator, stripped_op_v))
+                query['must'].append({"range": {key: {ops[operator]: v for operator, v in range_constraints}}})
+            elif many:
+                query['must'].append({"terms": {key: value}})
+            else:
+                if key == 'fee_amount':
+                    value = str(Decimal(value)*1000)
+                query['must'].append({"term": {key: {"value": value}}})
+        elif key == 'not_channel_ids':
+            for channel_id in value:
+                query['must_not'].append({"term": {'channel_id.keyword': channel_id}})
+                query['must_not'].append({"term": {'_id': channel_id}})
+        elif key == 'channel_ids':
+            query['must'].append({"terms": {'channel_id.keyword': value}})
+        elif key == 'claim_ids':
+            query['must'].append({"terms": {'claim_id.keyword': value}})
+        elif key == 'media_types':
+            query['must'].append({"terms": {'media_type.keyword': value}})
+        elif key == 'any_languages':
+            query['must'].append({"terms": {'languages': clean_tags(value)}})
+        elif key == 'any_languages':
+            query['must'].append({"terms": {'languages': value}})
+        elif key == 'all_languages':
+            query['must'].extend([{"term": {'languages': tag}} for tag in value])
+        elif key == 'any_tags':
+            query['must'].append({"terms": {'tags.keyword': clean_tags(value)}})
+        elif key == 'all_tags':
+            query['must'].extend([{"term": {'tags.keyword': tag}} for tag in clean_tags(value)])
+        elif key == 'not_tags':
+            query['must_not'].extend([{"term": {'tags.keyword': tag}} for tag in clean_tags(value)])
+        elif key == 'not_claim_id':
+            query['must_not'].extend([{"term": {'claim_id.keyword': cid}} for cid in value])
+        elif key == 'limit_claims_per_channel':
+            collapse = ('channel_id.keyword', value)
+    if kwargs.get('has_channel_signature'):
+        query['must'].append({"exists": {"field": "signature"}})
+        if 'signature_valid' in kwargs:
+            query['must'].append({"term": {"is_signature_valid": bool(kwargs["signature_valid"])}})
+    elif 'signature_valid' in kwargs:
+        query.setdefault('should', [])
+        query["minimum_should_match"] = 1
+        query['should'].append({"bool": {"must_not": {"exists": {"field": "signature"}}}})
+        query['should'].append({"term": {"is_signature_valid": bool(kwargs["signature_valid"])}})
+    if 'has_source' in kwargs:
+        query.setdefault('should', [])
+        query["minimum_should_match"] = 1
+        is_stream_or_repost = {"terms": {"claim_type": [CLAIM_TYPES['stream'], CLAIM_TYPES['repost']]}}
+        query['should'].append(
+            {"bool": {"must": [{"match": {"has_source": kwargs['has_source']}}, is_stream_or_repost]}})
+        query['should'].append({"bool": {"must_not": [is_stream_or_repost]}})
+        query['should'].append({"bool": {"must": [{"term": {"reposted_claim_type": CLAIM_TYPES['channel']}}]}})
+    if kwargs.get('text'):
+        query['must'].append(
+                    {"simple_query_string":
+                         {"query": kwargs["text"], "fields": [
+                             "claim_name^4", "channel_name^8", "title^1", "description^.5", "author^1", "tags^.5"
+                         ]}})
+    query = {
+        "_source": {"excludes": ["description", "title"]},
+        'query': {'bool': query},
+        "sort": [],
+    }
+    if "limit" in kwargs:
+        query["size"] = kwargs["limit"]
+    if 'offset' in kwargs:
+        query["from"] = kwargs["offset"]
+    if 'order_by' in kwargs:
+        if isinstance(kwargs["order_by"], str):
+            kwargs["order_by"] = [kwargs["order_by"]]
+        for value in kwargs['order_by']:
+            if 'trending_group' in value:
+                # fixme: trending_mixed is 0 for all records on variable decay, making sort slow.
+                continue
+            is_asc = value.startswith('^')
+            value = value[1:] if is_asc else value
+            value = REPLACEMENTS.get(value, value)
+            if value in TEXT_FIELDS:
+                value += '.keyword'
+            query['sort'].append({value: "asc" if is_asc else "desc"})
+    if collapse:
+        query["collapse"] = {
+            "field": collapse[0],
+            "inner_hits": {
+                "name": collapse[0],
+                "size": collapse[1],
+                "sort": query["sort"]
+            }
+        }
+    return query
+
+
+def expand_result(results):
+    inner_hits = []
+    expanded = []
+    for result in results:
+        if result.get("inner_hits"):
+            for _, inner_hit in result["inner_hits"].items():
+                inner_hits.extend(inner_hit["hits"]["hits"])
+            continue
+        result = result['_source']
+        result['claim_hash'] = unhexlify(result['claim_id'])[::-1]
+        if result['reposted_claim_id']:
+            result['reposted_claim_hash'] = unhexlify(result['reposted_claim_id'])[::-1]
+        else:
+            result['reposted_claim_hash'] = None
+        result['channel_hash'] = unhexlify(result['channel_id'])[::-1] if result['channel_id'] else None
+        result['txo_hash'] = unhexlify(result['tx_id'])[::-1] + struct.pack('<I', result['tx_nout'])
+        result['tx_hash'] = unhexlify(result['tx_id'])[::-1]
+        result['reposted'] = result.pop('repost_count')
+        result['signature_valid'] = result.pop('is_signature_valid')
+        # result['normalized'] = result.pop('normalized_name')
+        # if result['censoring_channel_hash']:
+        #     result['censoring_channel_hash'] = unhexlify(result['censoring_channel_hash'])[::-1]
+        expanded.append(result)
+    if inner_hits:
+        return expand_result(inner_hits)
+    return expanded
+
+
+class ResultCacheItem:
+    __slots__ = '_result', 'lock', 'has_result'
+
+    def __init__(self):
+        self.has_result = asyncio.Event()
+        self.lock = asyncio.Lock()
+        self._result = None
+
+    @property
+    def result(self) -> str:
+        return self._result
+
+    @result.setter
+    def result(self, result: str):
+        self._result = result
+        if result is not None:
+            self.has_result.set()
+
+    @classmethod
+    def from_cache(cls, cache_key, cache):
+        cache_item = cache.get(cache_key)
+        if cache_item is None:
+            cache_item = cache[cache_key] = ResultCacheItem()
+        return cache_item

scribe/env.py

@@ -0,0 +1,393 @@
+import os
+import re
+import resource
+import logging
+from collections import namedtuple
+from ipaddress import ip_address
+from scribe.blockchain.network import LBCMainNet, LBCTestNet, LBCRegTest
+
+
+NetIdentity = namedtuple('NetIdentity', 'host tcp_port ssl_port nick_suffix')
+
+
+SEGMENT_REGEX = re.compile("(?!-)[A-Z_\\d-]{1,63}(?<!-)$", re.IGNORECASE)
+
+
+def is_valid_hostname(hostname):
+    if len(hostname) > 255:
+        return False
+    # strip exactly one dot from the right, if present
+    if hostname and hostname[-1] == ".":
+        hostname = hostname[:-1]
+    return all(SEGMENT_REGEX.match(x) for x in hostname.split("."))
+
+
+class Env:
+
+    # Peer discovery
+    PD_OFF, PD_SELF, PD_ON = range(3)
+
+    class Error(Exception):
+        pass
+
+    def __init__(self, db_dir=None, daemon_url=None, host=None, rpc_host=None, elastic_host=None,
+                 elastic_port=None, loop_policy=None, max_query_workers=None, websocket_host=None, websocket_port=None,
+                 chain=None, es_index_prefix=None, cache_MB=None, reorg_limit=None, tcp_port=None,
+                 udp_port=None, ssl_port=None, ssl_certfile=None, ssl_keyfile=None, rpc_port=None,
+                 prometheus_port=None, max_subscriptions=None, banner_file=None, anon_logs=None, log_sessions=None,
+                 allow_lan_udp=None, cache_all_tx_hashes=None, cache_all_claim_txos=None, country=None,
+                 payment_address=None, donation_address=None, max_send=None, max_receive=None, max_sessions=None,
+                 session_timeout=None, drop_client=None, description=None, daily_fee=None,
+                 database_query_timeout=None, db_max_open_files=64, elastic_notifier_port=None,
+                 blocking_channel_ids=None, filtering_channel_ids=None, peer_hubs=None, peer_announce=None):
+        self.logger = logging.getLogger(__name__)
+        self.db_dir = db_dir if db_dir is not None else self.required('DB_DIRECTORY')
+        self.daemon_url = daemon_url if daemon_url is not None else self.required('DAEMON_URL')
+        self.db_max_open_files = db_max_open_files
+
+        self.host = host if host is not None else self.default('HOST', 'localhost')
+        self.rpc_host = rpc_host if rpc_host is not None else self.default('RPC_HOST', 'localhost')
+        self.elastic_host = elastic_host if elastic_host is not None else self.default('ELASTIC_HOST', 'localhost')
+        self.elastic_port = elastic_port if elastic_port is not None else self.integer('ELASTIC_PORT', 9200)
+        self.elastic_notifier_port = elastic_notifier_port if elastic_notifier_port is not None else self.integer('ELASTIC_NOTIFIER_PORT', 19080)
+
+        self.loop_policy = self.set_event_loop_policy(
+            loop_policy if loop_policy is not None else self.default('EVENT_LOOP_POLICY', None)
+        )
+        self.obsolete(['UTXO_MB', 'HIST_MB', 'NETWORK'])
+        self.max_query_workers = max_query_workers if max_query_workers is not None else self.integer('MAX_QUERY_WORKERS', 4)
+        self.websocket_host = websocket_host if websocket_host is not None else self.default('WEBSOCKET_HOST', self.host)
+        self.websocket_port = websocket_port if websocket_port is not None else self.integer('WEBSOCKET_PORT', None)
+        if chain == 'mainnet':
+            self.coin = LBCMainNet
+        elif chain == 'testnet':
+            self.coin = LBCTestNet
+        else:
+            self.coin = LBCRegTest
+        self.es_index_prefix = es_index_prefix if es_index_prefix is not None else self.default('ES_INDEX_PREFIX', '')
+        self.cache_MB = cache_MB if cache_MB is not None else self.integer('CACHE_MB', 1024)
+        self.reorg_limit = reorg_limit if reorg_limit is not None else self.integer('REORG_LIMIT', self.coin.REORG_LIMIT)
+        # Server stuff
+        self.tcp_port = tcp_port if tcp_port is not None else self.integer('TCP_PORT', None)
+        self.udp_port = udp_port if udp_port is not None else self.integer('UDP_PORT', self.tcp_port)
+        self.ssl_port = ssl_port if ssl_port is not None else self.integer('SSL_PORT', None)
+        if self.ssl_port:
+            self.ssl_certfile = ssl_certfile if ssl_certfile is not None else self.required('SSL_CERTFILE')
+            self.ssl_keyfile = ssl_keyfile if ssl_keyfile is not None else self.required('SSL_KEYFILE')
+        self.rpc_port = rpc_port if rpc_port is not None else self.integer('RPC_PORT', 8000)
+        self.prometheus_port = prometheus_port if prometheus_port is not None else self.integer('PROMETHEUS_PORT', 0)
+        self.max_subscriptions = max_subscriptions if max_subscriptions is not None else self.integer('MAX_SUBSCRIPTIONS', 10000)
+        self.banner_file = banner_file if banner_file is not None else self.default('BANNER_FILE', None)
+        # self.tor_banner_file = self.default('TOR_BANNER_FILE', self.banner_file)
+        self.anon_logs = anon_logs if anon_logs is not None else self.boolean('ANON_LOGS', False)
+        self.log_sessions = log_sessions if log_sessions is not None else self.integer('LOG_SESSIONS', 3600)
+        self.allow_lan_udp = allow_lan_udp if allow_lan_udp is not None else self.boolean('ALLOW_LAN_UDP', False)
+        self.cache_all_tx_hashes = cache_all_tx_hashes if cache_all_tx_hashes is not None else self.boolean('CACHE_ALL_TX_HASHES', False)
+        self.cache_all_claim_txos = cache_all_claim_txos if cache_all_claim_txos is not None else self.boolean('CACHE_ALL_CLAIM_TXOS', False)
+        self.country = country if country is not None else self.default('COUNTRY', 'US')
+        # Peer discovery
+        self.peer_discovery = self.peer_discovery_enum()
+        self.peer_announce = peer_announce if peer_announce is not None else self.boolean('PEER_ANNOUNCE', True)
+        if peer_hubs is not None:
+            self.peer_hubs = [p.strip("") for p in peer_hubs.split(",")]
+        else:
+            self.peer_hubs = self.extract_peer_hubs()
+        # self.tor_proxy_host = self.default('TOR_PROXY_HOST', 'localhost')
+        # self.tor_proxy_port = self.integer('TOR_PROXY_PORT', None)
+        # The electrum client takes the empty string as unspecified
+        self.payment_address = payment_address if payment_address is not None else self.default('PAYMENT_ADDRESS', '')
+        self.donation_address = donation_address if donation_address is not None else self.default('DONATION_ADDRESS', '')
+        # Server limits to help prevent DoS
+        self.max_send = max_send if max_send is not None else self.integer('MAX_SEND', 1000000)
+        self.max_receive = max_receive if max_receive is not None else self.integer('MAX_RECEIVE', 1000000)
+        # self.max_subs = self.integer('MAX_SUBS', 250000)
+        self.max_sessions = max_sessions if max_sessions is not None else self.sane_max_sessions()
+        # self.max_session_subs = self.integer('MAX_SESSION_SUBS', 50000)
+        self.session_timeout = session_timeout if session_timeout is not None else self.integer('SESSION_TIMEOUT', 600)
+        self.drop_client = drop_client if drop_client is not None else self.custom("DROP_CLIENT", None, re.compile)
+        self.description = description if description is not None else self.default('DESCRIPTION', '')
+        self.daily_fee = daily_fee if daily_fee is not None else self.string_amount('DAILY_FEE', '0')
+
+        # Identities
+        clearnet_identity = self.clearnet_identity()
+        tor_identity = self.tor_identity(clearnet_identity)
+        self.identities = [identity
+                           for identity in (clearnet_identity, tor_identity)
+                           if identity is not None]
+        self.database_query_timeout = database_query_timeout if database_query_timeout is not None else \
+            (float(self.integer('QUERY_TIMEOUT_MS', 10000)) / 1000.0)
+
+        # Filtering / Blocking
+        self.blocking_channel_ids = blocking_channel_ids if blocking_channel_ids is not None else self.default('BLOCKING_CHANNEL_IDS', '').split(' ')
+        self.filtering_channel_ids = filtering_channel_ids if filtering_channel_ids is not None else self.default('FILTERING_CHANNEL_IDS', '').split(' ')
+
+    @classmethod
+    def default(cls, envvar, default):
+        return os.environ.get(envvar, default)
+
+    @classmethod
+    def boolean(cls, envvar, default):
+        default = 'Yes' if default else ''
+        return bool(cls.default(envvar, default).strip())
+
+    @classmethod
+    def required(cls, envvar):
+        value = os.environ.get(envvar)
+        if value is None:
+            raise cls.Error(f'required envvar {envvar} not set')
+        return value
+
+    @classmethod
+    def string_amount(cls, envvar, default):
+        value = os.environ.get(envvar, default)
+        amount_pattern = re.compile("[0-9]{0,10}(\.[0-9]{1,8})?")
+        if len(value) > 0 and not amount_pattern.fullmatch(value):
+            raise cls.Error(f'{value} is not a valid amount for {envvar}')
+        return value
+
+    @classmethod
+    def integer(cls, envvar, default):
+        value = os.environ.get(envvar)
+        if value is None:
+            return default
+        try:
+            return int(value)
+        except Exception:
+            raise cls.Error(f'cannot convert envvar {envvar} value {value} to an integer')
+
+    @classmethod
+    def custom(cls, envvar, default, parse):
+        value = os.environ.get(envvar)
+        if value is None:
+            return default
+        try:
+            return parse(value)
+        except Exception as e:
+            raise cls.Error(f'cannot parse envvar {envvar} value {value}') from e
+
+    @classmethod
+    def obsolete(cls, envvars):
+        bad = [envvar for envvar in envvars if os.environ.get(envvar)]
+        if bad:
+            raise cls.Error(f'remove obsolete os.environment variables {bad}')
+
+    @classmethod
+    def set_event_loop_policy(cls, policy_name: str = None):
+        if not policy_name or policy_name == 'default':
+            import asyncio
+            return asyncio.get_event_loop_policy()
+        elif policy_name == 'uvloop':
+            import uvloop
+            import asyncio
+            loop_policy = uvloop.EventLoopPolicy()
+            asyncio.set_event_loop_policy(loop_policy)
+            return loop_policy
+        raise cls.Error(f'unknown event loop policy "{policy_name}"')
+
+    def cs_host(self, *, for_rpc):
+        """Returns the 'host' argument to pass to asyncio's create_server
+        call.  The result can be a single host name string, a list of
+        host name strings, or an empty string to bind to all interfaces.
+
+        If rpc is True the host to use for the RPC server is returned.
+        Otherwise the host to use for SSL/TCP servers is returned.
+        """
+        host = self.rpc_host if for_rpc else self.host
+        result = [part.strip() for part in host.split(',')]
+        if len(result) == 1:
+            result = result[0]
+        # An empty result indicates all interfaces, which we do not
+        # permitted for an RPC server.
+        if for_rpc and not result:
+            result = 'localhost'
+        if result == 'localhost':
+            # 'localhost' resolves to ::1 (ipv6) on many systems, which fails on default setup of
+            # docker, using 127.0.0.1 instead forces ipv4
+            result = '127.0.0.1'
+        return result
+
+    def sane_max_sessions(self):
+        """Return the maximum number of sessions to permit.  Normally this
+        is MAX_SESSIONS.  However, to prevent open file exhaustion, ajdust
+        downwards if running with a small open file rlimit."""
+        env_value = self.integer('MAX_SESSIONS', 1000)
+        nofile_limit = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
+        # We give the DB 250 files; allow ElectrumX 100 for itself
+        value = max(0, min(env_value, nofile_limit - 350))
+        if value < env_value:
+            self.logger.warning(f'lowered maximum sessions from {env_value:,d} to {value:,d} '
+                                f'because your open file limit is {nofile_limit:,d}')
+        return value
+
+    def clearnet_identity(self):
+        host = self.default('REPORT_HOST', None)
+        if host is None:
+            return None
+        try:
+            ip = ip_address(host)
+        except ValueError:
+            bad = (not is_valid_hostname(host)
+                   or host.lower() == 'localhost')
+        else:
+            bad = (ip.is_multicast or ip.is_unspecified
+                   or (ip.is_private and self.peer_announce))
+        if bad:
+            raise self.Error(f'"{host}" is not a valid REPORT_HOST')
+        tcp_port = self.integer('REPORT_TCP_PORT', self.tcp_port) or None
+        ssl_port = self.integer('REPORT_SSL_PORT', self.ssl_port) or None
+        if tcp_port == ssl_port:
+            raise self.Error('REPORT_TCP_PORT and REPORT_SSL_PORT '
+                             f'both resolve to {tcp_port}')
+        return NetIdentity(
+            host,
+            tcp_port,
+            ssl_port,
+            ''
+        )
+
+    def tor_identity(self, clearnet):
+        host = self.default('REPORT_HOST_TOR', None)
+        if host is None:
+            return None
+        if not host.endswith('.onion'):
+            raise self.Error(f'tor host "{host}" must end with ".onion"')
+
+        def port(port_kind):
+            """Returns the clearnet identity port, if any and not zero,
+            otherwise the listening port."""
+            result = 0
+            if clearnet:
+                result = getattr(clearnet, port_kind)
+            return result or getattr(self, port_kind)
+
+        tcp_port = self.integer('REPORT_TCP_PORT_TOR',
+                                port('tcp_port')) or None
+        ssl_port = self.integer('REPORT_SSL_PORT_TOR',
+                                port('ssl_port')) or None
+        if tcp_port == ssl_port:
+            raise self.Error('REPORT_TCP_PORT_TOR and REPORT_SSL_PORT_TOR '
+                             f'both resolve to {tcp_port}')
+
+        return NetIdentity(
+            host,
+            tcp_port,
+            ssl_port,
+            '_tor',
+        )
+
+    def hosts_dict(self):
+        return {identity.host: {'tcp_port': identity.tcp_port,
+                                'ssl_port': identity.ssl_port}
+                for identity in self.identities}
+
+    def peer_discovery_enum(self):
+        pd = self.default('PEER_DISCOVERY', 'on').strip().lower()
+        if pd in ('off', ''):
+            return self.PD_OFF
+        elif pd == 'self':
+            return self.PD_SELF
+        else:
+            return self.PD_ON
+
+    def extract_peer_hubs(self):
+        return [hub.strip() for hub in self.default('PEER_HUBS', '').split(',')]
+
+    @classmethod
+    def contribute_to_arg_parser(cls, parser):
+        parser.add_argument('--db_dir', type=str, help='path of the directory containing lbry-leveldb',
+                            default=cls.default('DB_DIRECTORY', None))
+        parser.add_argument('--daemon_url',
+                            help='URL for rpc from lbrycrd, <rpcuser>:<rpcpassword>@<lbrycrd rpc ip><lbrycrd rpc port>',
+                            default=cls.default('DAEMON_URL', None))
+        parser.add_argument('--db_max_open_files', type=int, default=64,
+                            help='number of files rocksdb can have open at a time')
+        parser.add_argument('--host', type=str, default=cls.default('HOST', 'localhost'),
+                            help='Interface for hub server to listen on')
+        parser.add_argument('--tcp_port', type=int, default=cls.integer('TCP_PORT', 50001),
+                            help='TCP port to listen on for hub server')
+        parser.add_argument('--udp_port', type=int, default=cls.integer('UDP_PORT', 50001),
+                            help='UDP port to listen on for hub server')
+        parser.add_argument('--rpc_host', default=cls.default('RPC_HOST', 'localhost'), type=str,
+                            help='Listening interface for admin rpc')
+        parser.add_argument('--rpc_port', default=cls.integer('RPC_PORT', 8000), type=int,
+                            help='Listening port for admin rpc')
+        parser.add_argument('--websocket_host', default=cls.default('WEBSOCKET_HOST', 'localhost'), type=str,
+                            help='Listening interface for websocket')
+        parser.add_argument('--websocket_port', default=cls.integer('WEBSOCKET_PORT', None), type=int,
+                            help='Listening port for websocket')
+
+        parser.add_argument('--ssl_port', default=cls.integer('SSL_PORT', None), type=int,
+                            help='SSL port to listen on for hub server')
+        parser.add_argument('--ssl_certfile', default=cls.default('SSL_CERTFILE', None), type=str,
+                            help='Path to SSL cert file')
+        parser.add_argument('--ssl_keyfile', default=cls.default('SSL_KEYFILE', None), type=str,
+                            help='Path to SSL key file')
+        parser.add_argument('--reorg_limit', default=cls.integer('REORG_LIMIT', 200), type=int, help='Max reorg depth')
+        parser.add_argument('--elastic_host', default=cls.default('ELASTIC_HOST', 'localhost'), type=str,
+                            help='elasticsearch host')
+        parser.add_argument('--elastic_port', default=cls.integer('ELASTIC_PORT', 9200), type=int,
+                            help='elasticsearch port')
+        parser.add_argument('--es_index_prefix', default=cls.default('ES_INDEX_PREFIX', ''), type=str)
+        parser.add_argument('--loop_policy', default=cls.default('EVENT_LOOP_POLICY', 'default'), type=str,
+                            choices=['default', 'uvloop'])
+        parser.add_argument('--max_query_workers', type=int, default=cls.integer('MAX_QUERY_WORKERS', 4),
+                            help='number of threads used by the request handler to read the database')
+        parser.add_argument('--cache_MB', type=int, default=cls.integer('CACHE_MB', 1024),
+                            help='size of the leveldb lru cache, in megabytes')
+        parser.add_argument('--cache_all_tx_hashes', type=bool,
+                            help='Load all tx hashes into memory. This will make address subscriptions and sync, '
+                                 'resolve, transaction fetching, and block sync all faster at the expense of higher '
+                                 'memory usage')
+        parser.add_argument('--cache_all_claim_txos', type=bool,
+                            help='Load all claim txos into memory. This will make address subscriptions and sync, '
+                                 'resolve, transaction fetching, and block sync all faster at the expense of higher '
+                                 'memory usage')
+        parser.add_argument('--prometheus_port', type=int, default=cls.integer('PROMETHEUS_PORT', 0),
+                            help='port for hub prometheus metrics to listen on, disabled by default')
+        parser.add_argument('--max_subscriptions', type=int, default=cls.integer('MAX_SUBSCRIPTIONS', 10000),
+                            help='max subscriptions per connection')
+        parser.add_argument('--banner_file', type=str, default=cls.default('BANNER_FILE', None),
+                            help='path to file containing banner text')
+        parser.add_argument('--anon_logs', type=bool, default=cls.boolean('ANON_LOGS', False),
+                            help="don't log ip addresses")
+        parser.add_argument('--allow_lan_udp', type=bool, default=cls.boolean('ALLOW_LAN_UDP', False),
+                            help='reply to hub UDP ping messages from LAN ip addresses')
+        parser.add_argument('--country', type=str, default=cls.default('COUNTRY', 'US'), help='')
+        parser.add_argument('--max_send', type=int, default=cls.default('MAX_SEND', 1000000), help='')
+        parser.add_argument('--max_receive', type=int, default=cls.default('MAX_RECEIVE', 1000000), help='')
+        parser.add_argument('--max_sessions', type=int, default=cls.default('MAX_SESSIONS', 1000), help='')
+        parser.add_argument('--session_timeout', type=int, default=cls.default('SESSION_TIMEOUT', 600), help='')
+        parser.add_argument('--drop_client', type=str, default=cls.default('DROP_CLIENT', None), help='')
+        parser.add_argument('--description', type=str, default=cls.default('DESCRIPTION', ''), help='')
+        parser.add_argument('--daily_fee', type=float, default=cls.default('DAILY_FEE', 0.0), help='')
+        parser.add_argument('--payment_address', type=str, default=cls.default('PAYMENT_ADDRESS', ''), help='')
+        parser.add_argument('--donation_address', type=str, default=cls.default('DONATION_ADDRESS', ''), help='')
+        parser.add_argument('--chain', type=str, default=cls.default('NET', 'mainnet'),
+                            help="Which chain to use, default is mainnet", choices=['mainnet', 'regtest', 'testnet'])
+        parser.add_argument('--query_timeout_ms', type=int, default=cls.integer('QUERY_TIMEOUT_MS', 10000),
+                            help="elasticsearch query timeout")
+
+        parser.add_argument('--blocking_channel_ids', nargs='*', help='',
+                            default=cls.default('BLOCKING_CHANNEL_IDS', '').split(' '))
+        parser.add_argument('--filtering_channel_ids', nargs='*', help='',
+                            default=cls.default('FILTERING_CHANNEL_IDS', '').split(' '))
+
+    @classmethod
+    def from_arg_parser(cls, args):
+        return cls(
+            db_dir=args.db_dir, daemon_url=args.daemon_url, db_max_open_files=args.db_max_open_files,
+            host=args.host, rpc_host=args.rpc_host, elastic_host=args.elastic_host, elastic_port=args.elastic_port,
+            loop_policy=args.loop_policy, max_query_workers=args.max_query_workers, websocket_host=args.websocket_host,
+            websocket_port=args.websocket_port, chain=args.chain, es_index_prefix=args.es_index_prefix,
+            cache_MB=args.cache_MB, reorg_limit=args.reorg_limit, tcp_port=args.tcp_port,
+            udp_port=args.udp_port, ssl_port=args.ssl_port, ssl_certfile=args.ssl_certfile,
+            ssl_keyfile=args.ssl_keyfile, rpc_port=args.rpc_port, prometheus_port=args.prometheus_port,
+            max_subscriptions=args.max_subscriptions, banner_file=args.banner_file, anon_logs=args.anon_logs,
+            log_sessions=None, allow_lan_udp=args.allow_lan_udp,
+            cache_all_tx_hashes=args.cache_all_tx_hashes, cache_all_claim_txos=args.cache_all_claim_txos,
+            country=args.country, payment_address=args.payment_address, donation_address=args.donation_address,
+            max_send=args.max_send, max_receive=args.max_receive, max_sessions=args.max_sessions,
+            session_timeout=args.session_timeout, drop_client=args.drop_client, description=args.description,
+            daily_fee=args.daily_fee, database_query_timeout=(args.query_timeout_ms / 1000),
+            blocking_channel_ids=args.blocking_channel_ids, filtering_channel_ids=args.filtering_channel_ids
+        )

scribe/error/Makefile

@@ -0,0 +1,5 @@
+generate:
+	python generate.py generate > __init__.py
+
+analyze:
+	python generate.py analyze

scribe/error/README.md

@@ -0,0 +1,95 @@
+# Exceptions
+
+Exceptions in LBRY are defined and generated from the Markdown table at the end of this README.
+
+## Guidelines
+
+When possible, use [built-in Python exceptions](https://docs.python.org/3/library/exceptions.html) or `aiohttp` [general client](https://docs.aiohttp.org/en/latest/client_reference.html#client-exceptions) / [HTTP](https://docs.aiohttp.org/en/latest/web_exceptions.html) exceptions, unless:
+1. You want to provide a better error message (extend the closest built-in/`aiohttp` exception in this case).
+2. You need to represent a new situation.
+
+When defining your own exceptions, consider:
+1. Extending a built-in Python or `aiohttp` exception.
+2. Using contextual variables in the error message.
+
+## Table Column Definitions
+
+Column | Meaning
+---|---
+Code | Codes are used only to define the hierarchy of exceptions and do not end up in the generated output, it is okay to re-number things as necessary at anytime to achieve the desired hierarchy.
+Name | Becomes the class name of the exception with "Error" appended to the end. Changing names of existing exceptions makes the API backwards incompatible. When extending other exceptions you must specify the full class name, manually adding "Error" as necessary (if extending another SDK exception).
+Message | User friendly error message explaining the exceptional event. Supports Python formatted strings: any variables used in the string will be generated as arguments in the `__init__` method. Use `--` to provide a doc string after the error message to be added to the class definition.
+
+## Exceptions Table
+
+Code | Name | Message
+---:|---|---
+**1xx** | UserInput | User input errors.
+**10x** | Command | Errors preparing to execute commands.
+101 | CommandDoesNotExist | Command '{command}' does not exist.
+102 | CommandDeprecated | Command '{command}' is deprecated.
+103 | CommandInvalidArgument | Invalid argument '{argument}' to command '{command}'.
+104 | CommandTemporarilyUnavailable | Command '{command}' is temporarily unavailable. -- Such as waiting for required components to start.
+105 | CommandPermanentlyUnavailable | Command '{command}' is permanently unavailable. -- such as when required component was intentionally configured not to start.
+**11x** | InputValue(ValueError) | Invalid argument value provided to command.
+111 | GenericInputValue | The value '{value}' for argument '{argument}' is not valid.
+112 | InputValueIsNone | None or null is not valid value for argument '{argument}'.
+113 | ConflictingInputValue | Only '{first_argument}' or '{second_argument}' is allowed, not both.
+114 | InputStringIsBlank | {argument} cannot be blank.
+115 | EmptyPublishedFile | Cannot publish empty file: {file_path}
+116 | MissingPublishedFile | File does not exist: {file_path}
+117 | InvalidStreamURL | Invalid LBRY stream URL: '{url}' -- When an URL cannot be downloaded, such as '@Channel/' or a collection
+**2xx** | Configuration | Configuration errors.
+201 | ConfigWrite | Cannot write configuration file '{path}'. -- When writing the default config fails on startup, such as due to permission issues.
+202 | ConfigRead | Cannot find provided configuration file '{path}'. -- Can't open the config file user provided via command line args.
+203 | ConfigParse | Failed to parse the configuration file '{path}'. -- Includes the syntax error / line number to help user fix it.
+204 | ConfigMissing | Configuration file '{path}' is missing setting that has no default / fallback.
+205 | ConfigInvalid | Configuration file '{path}' has setting with invalid value.
+**3xx** | Network | **Networking**
+301 | NoInternet | No internet connection.
+302 | NoUPnPSupport | Router does not support UPnP.
+**4xx** | Wallet | **Wallet Errors**
+401 | TransactionRejected | Transaction rejected, unknown reason.
+402 | TransactionFeeTooLow | Fee too low.
+403 | TransactionInvalidSignature | Invalid signature.
+404 | InsufficientFunds |  Not enough funds to cover this transaction. -- determined by wallet prior to attempting to broadcast a tx; this is different for example from a TX being created and sent but then rejected by lbrycrd for unspendable utxos.
+405 | ChannelKeyNotFound | Channel signing key not found.
+406 | ChannelKeyInvalid | Channel signing key is out of date. -- For example, channel was updated but you don't have the updated key.
+407 | DataDownload | Failed to download blob. *generic*
+408 | PrivateKeyNotFound | Couldn't find private key for {key} '{value}'.
+410 | Resolve | Failed to resolve '{url}'.
+411 | ResolveTimeout | Failed to resolve '{url}' within the timeout.
+411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{censor_id}'.
+420 | KeyFeeAboveMaxAllowed | {message}
+421 | InvalidPassword | Password is invalid.
+422 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version.
+423 | TooManyClaimSearchParameters | {key} cant have more than {limit} items.
+424 | AlreadyPurchased | You already have a purchase for claim_id '{claim_id_hex}'. Use --allow-duplicate-purchase flag to override.
+431 | ServerPaymentInvalidAddress | Invalid address from wallet server: '{address}' - skipping payment round.
+432 | ServerPaymentWalletLocked | Cannot spend funds with locked wallet, skipping payment round.
+433 | ServerPaymentFeeAboveMaxAllowed | Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.
+434 | WalletNotLoaded | Wallet {wallet_id} is not loaded.
+435 | WalletAlreadyLoaded | Wallet {wallet_path} is already loaded.
+436 | WalletNotFound | Wallet not found at {wallet_path}.
+437 | WalletAlreadyExists | Wallet {wallet_path} already exists, use `wallet_add` to load it.
+**5xx** | Blob | **Blobs**
+500 | BlobNotFound | Blob not found.
+501 | BlobPermissionDenied | Permission denied to read blob.
+502 | BlobTooBig | Blob is too big.
+503 | BlobEmpty | Blob is empty.
+510 | BlobFailedDecryption | Failed to decrypt blob.
+511 | CorruptBlob | Blobs is corrupted.
+520 | BlobFailedEncryption | Failed to encrypt blob.
+531 | DownloadCancelled | Download was canceled.
+532 | DownloadSDTimeout | Failed to download sd blob {download} within timeout.
+533 | DownloadDataTimeout | Failed to download data blobs for sd hash {download} within timeout.
+534 | InvalidStreamDescriptor | {message}
+535 | InvalidData | {message}
+536 | InvalidBlobHash | {message}
+**6xx** | Component | **Components**
+601 | ComponentStartConditionNotMet | Unresolved dependencies for: {components}
+602 | ComponentsNotStarted | {message}
+**7xx** | CurrencyExchange | **Currency Exchange**
+701 | InvalidExchangeRateResponse | Failed to get exchange rate from {source}: {reason}
+702 | CurrencyConversion | {message}
+703 | InvalidCurrency | Invalid currency: {currency} is not a supported currency.

scribe/error/__init__.py

@@ -0,0 +1,494 @@
+from .base import BaseError, claim_id
+
+
+class UserInputError(BaseError):
+    """
+    User input errors.
+    """
+
+
+class CommandError(UserInputError):
+    """
+    Errors preparing to execute commands.
+    """
+
+
+class CommandDoesNotExistError(CommandError):
+
+    def __init__(self, command):
+        self.command = command
+        super().__init__(f"Command '{command}' does not exist.")
+
+
+class CommandDeprecatedError(CommandError):
+
+    def __init__(self, command):
+        self.command = command
+        super().__init__(f"Command '{command}' is deprecated.")
+
+
+class CommandInvalidArgumentError(CommandError):
+
+    def __init__(self, argument, command):
+        self.argument = argument
+        self.command = command
+        super().__init__(f"Invalid argument '{argument}' to command '{command}'.")
+
+
+class CommandTemporarilyUnavailableError(CommandError):
+    """
+    Such as waiting for required components to start.
+    """
+
+    def __init__(self, command):
+        self.command = command
+        super().__init__(f"Command '{command}' is temporarily unavailable.")
+
+
+class CommandPermanentlyUnavailableError(CommandError):
+    """
+    such as when required component was intentionally configured not to start.
+    """
+
+    def __init__(self, command):
+        self.command = command
+        super().__init__(f"Command '{command}' is permanently unavailable.")
+
+
+class InputValueError(UserInputError, ValueError):
+    """
+    Invalid argument value provided to command.
+    """
+
+
+class GenericInputValueError(InputValueError):
+
+    def __init__(self, value, argument):
+        self.value = value
+        self.argument = argument
+        super().__init__(f"The value '{value}' for argument '{argument}' is not valid.")
+
+
+class InputValueIsNoneError(InputValueError):
+
+    def __init__(self, argument):
+        self.argument = argument
+        super().__init__(f"None or null is not valid value for argument '{argument}'.")
+
+
+class ConflictingInputValueError(InputValueError):
+
+    def __init__(self, first_argument, second_argument):
+        self.first_argument = first_argument
+        self.second_argument = second_argument
+        super().__init__(f"Only '{first_argument}' or '{second_argument}' is allowed, not both.")
+
+
+class InputStringIsBlankError(InputValueError):
+
+    def __init__(self, argument):
+        self.argument = argument
+        super().__init__(f"{argument} cannot be blank.")
+
+
+class EmptyPublishedFileError(InputValueError):
+
+    def __init__(self, file_path):
+        self.file_path = file_path
+        super().__init__(f"Cannot publish empty file: {file_path}")
+
+
+class MissingPublishedFileError(InputValueError):
+
+    def __init__(self, file_path):
+        self.file_path = file_path
+        super().__init__(f"File does not exist: {file_path}")
+
+
+class InvalidStreamURLError(InputValueError):
+    """
+    When an URL cannot be downloaded, such as '@Channel/' or a collection
+    """
+
+    def __init__(self, url):
+        self.url = url
+        super().__init__(f"Invalid LBRY stream URL: '{url}'")
+
+
+class ConfigurationError(BaseError):
+    """
+    Configuration errors.
+    """
+
+
+class ConfigWriteError(ConfigurationError):
+    """
+    When writing the default config fails on startup, such as due to permission issues.
+    """
+
+    def __init__(self, path):
+        self.path = path
+        super().__init__(f"Cannot write configuration file '{path}'.")
+
+
+class ConfigReadError(ConfigurationError):
+    """
+    Can't open the config file user provided via command line args.
+    """
+
+    def __init__(self, path):
+        self.path = path
+        super().__init__(f"Cannot find provided configuration file '{path}'.")
+
+
+class ConfigParseError(ConfigurationError):
+    """
+    Includes the syntax error / line number to help user fix it.
+    """
+
+    def __init__(self, path):
+        self.path = path
+        super().__init__(f"Failed to parse the configuration file '{path}'.")
+
+
+class ConfigMissingError(ConfigurationError):
+
+    def __init__(self, path):
+        self.path = path
+        super().__init__(f"Configuration file '{path}' is missing setting that has no default / fallback.")
+
+
+class ConfigInvalidError(ConfigurationError):
+
+    def __init__(self, path):
+        self.path = path
+        super().__init__(f"Configuration file '{path}' has setting with invalid value.")
+
+
+class NetworkError(BaseError):
+    """
+    **Networking**
+    """
+
+
+class NoInternetError(NetworkError):
+
+    def __init__(self):
+        super().__init__("No internet connection.")
+
+
+class NoUPnPSupportError(NetworkError):
+
+    def __init__(self):
+        super().__init__("Router does not support UPnP.")
+
+
+class WalletError(BaseError):
+    """
+    **Wallet Errors**
+    """
+
+
+class TransactionRejectedError(WalletError):
+
+    def __init__(self):
+        super().__init__("Transaction rejected, unknown reason.")
+
+
+class TransactionFeeTooLowError(WalletError):
+
+    def __init__(self):
+        super().__init__("Fee too low.")
+
+
+class TransactionInvalidSignatureError(WalletError):
+
+    def __init__(self):
+        super().__init__("Invalid signature.")
+
+
+class InsufficientFundsError(WalletError):
+    """
+    determined by wallet prior to attempting to broadcast a tx; this is different for example from a TX
+    being created and sent but then rejected by lbrycrd for unspendable utxos.
+    """
+
+    def __init__(self):
+        super().__init__("Not enough funds to cover this transaction.")
+
+
+class ChannelKeyNotFoundError(WalletError):
+
+    def __init__(self):
+        super().__init__("Channel signing key not found.")
+
+
+class ChannelKeyInvalidError(WalletError):
+    """
+    For example, channel was updated but you don't have the updated key.
+    """
+
+    def __init__(self):
+        super().__init__("Channel signing key is out of date.")
+
+
+class DataDownloadError(WalletError):
+
+    def __init__(self):
+        super().__init__("Failed to download blob. *generic*")
+
+
+class PrivateKeyNotFoundError(WalletError):
+
+    def __init__(self, key, value):
+        self.key = key
+        self.value = value
+        super().__init__(f"Couldn't find private key for {key} '{value}'.")
+
+
+class ResolveError(WalletError):
+
+    def __init__(self, url):
+        self.url = url
+        super().__init__(f"Failed to resolve '{url}'.")
+
+
+class ResolveTimeoutError(WalletError):
+
+    def __init__(self, url):
+        self.url = url
+        super().__init__(f"Failed to resolve '{url}' within the timeout.")
+
+
+class ResolveCensoredError(WalletError):
+
+    def __init__(self, url, censor_id, censor_row):
+        self.url = url
+        self.censor_id = censor_id
+        self.censor_row = censor_row
+        super().__init__(f"Resolve of '{url}' was censored by channel with claim id '{censor_id}'.")
+
+
+class KeyFeeAboveMaxAllowedError(WalletError):
+
+    def __init__(self, message):
+        self.message = message
+        super().__init__(f"{message}")
+
+
+class InvalidPasswordError(WalletError):
+
+    def __init__(self):
+        super().__init__("Password is invalid.")
+
+
+class IncompatibleWalletServerError(WalletError):
+
+    def __init__(self, server, port):
+        self.server = server
+        self.port = port
+        super().__init__(f"'{server}:{port}' has an incompatibly old version.")
+
+
+class TooManyClaimSearchParametersError(WalletError):
+
+    def __init__(self, key, limit):
+        self.key = key
+        self.limit = limit
+        super().__init__(f"{key} cant have more than {limit} items.")
+
+
+class AlreadyPurchasedError(WalletError):
+    """
+    allow-duplicate-purchase flag to override.
+    """
+
+    def __init__(self, claim_id_hex):
+        self.claim_id_hex = claim_id_hex
+        super().__init__(f"You already have a purchase for claim_id '{claim_id_hex}'. Use")
+
+
+class ServerPaymentInvalidAddressError(WalletError):
+
+    def __init__(self, address):
+        self.address = address
+        super().__init__(f"Invalid address from wallet server: '{address}' - skipping payment round.")
+
+
+class ServerPaymentWalletLockedError(WalletError):
+
+    def __init__(self):
+        super().__init__("Cannot spend funds with locked wallet, skipping payment round.")
+
+
+class ServerPaymentFeeAboveMaxAllowedError(WalletError):
+
+    def __init__(self, daily_fee, max_fee):
+        self.daily_fee = daily_fee
+        self.max_fee = max_fee
+        super().__init__(f"Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.")
+
+
+class WalletNotLoadedError(WalletError):
+
+    def __init__(self, wallet_id):
+        self.wallet_id = wallet_id
+        super().__init__(f"Wallet {wallet_id} is not loaded.")
+
+
+class WalletAlreadyLoadedError(WalletError):
+
+    def __init__(self, wallet_path):
+        self.wallet_path = wallet_path
+        super().__init__(f"Wallet {wallet_path} is already loaded.")
+
+
+class WalletNotFoundError(WalletError):
+
+    def __init__(self, wallet_path):
+        self.wallet_path = wallet_path
+        super().__init__(f"Wallet not found at {wallet_path}.")
+
+
+class WalletAlreadyExistsError(WalletError):
+
+    def __init__(self, wallet_path):
+        self.wallet_path = wallet_path
+        super().__init__(f"Wallet {wallet_path} already exists, use `wallet_add` to load it.")
+
+
+class BlobError(BaseError):
+    """
+    **Blobs**
+    """
+
+
+class BlobNotFoundError(BlobError):
+
+    def __init__(self):
+        super().__init__("Blob not found.")
+
+
+class BlobPermissionDeniedError(BlobError):
+
+    def __init__(self):
+        super().__init__("Permission denied to read blob.")
+
+
+class BlobTooBigError(BlobError):
+
+    def __init__(self):
+        super().__init__("Blob is too big.")
+
+
+class BlobEmptyError(BlobError):
+
+    def __init__(self):
+        super().__init__("Blob is empty.")
+
+
+class BlobFailedDecryptionError(BlobError):
+
+    def __init__(self):
+        super().__init__("Failed to decrypt blob.")
+
+
+class CorruptBlobError(BlobError):
+
+    def __init__(self):
+        super().__init__("Blobs is corrupted.")
+
+
+class BlobFailedEncryptionError(BlobError):
+
+    def __init__(self):
+        super().__init__("Failed to encrypt blob.")
+
+
+class DownloadCancelledError(BlobError):
+
+    def __init__(self):
+        super().__init__("Download was canceled.")
+
+
+class DownloadSDTimeoutError(BlobError):
+
+    def __init__(self, download):
+        self.download = download
+        super().__init__(f"Failed to download sd blob {download} within timeout.")
+
+
+class DownloadDataTimeoutError(BlobError):
+
+    def __init__(self, download):
+        self.download = download
+        super().__init__(f"Failed to download data blobs for sd hash {download} within timeout.")
+
+
+class InvalidStreamDescriptorError(BlobError):
+
+    def __init__(self, message):
+        self.message = message
+        super().__init__(f"{message}")
+
+
+class InvalidDataError(BlobError):
+
+    def __init__(self, message):
+        self.message = message
+        super().__init__(f"{message}")
+
+
+class InvalidBlobHashError(BlobError):
+
+    def __init__(self, message):
+        self.message = message
+        super().__init__(f"{message}")
+
+
+class ComponentError(BaseError):
+    """
+    **Components**
+    """
+
+
+class ComponentStartConditionNotMetError(ComponentError):
+
+    def __init__(self, components):
+        self.components = components
+        super().__init__(f"Unresolved dependencies for: {components}")
+
+
+class ComponentsNotStartedError(ComponentError):
+
+    def __init__(self, message):
+        self.message = message
+        super().__init__(f"{message}")
+
+
+class CurrencyExchangeError(BaseError):
+    """
+    **Currency Exchange**
+    """
+
+
+class InvalidExchangeRateResponseError(CurrencyExchangeError):
+
+    def __init__(self, source, reason):
+        self.source = source
+        self.reason = reason
+        super().__init__(f"Failed to get exchange rate from {source}: {reason}")
+
+
+class CurrencyConversionError(CurrencyExchangeError):
+
+    def __init__(self, message):
+        self.message = message
+        super().__init__(f"{message}")
+
+
+class InvalidCurrencyError(CurrencyExchangeError):
+
+    def __init__(self, currency):
+        self.currency = currency
+        super().__init__(f"Invalid currency: {currency} is not a supported currency.")

scribe/error/base.py

@@ -0,0 +1,9 @@
+from binascii import hexlify
+
+
+def claim_id(claim_hash):
+    return hexlify(claim_hash[::-1]).decode()
+
+
+class BaseError(Exception):
+    pass

scribe/error/generate.py

@@ -0,0 +1,167 @@
+import re
+import sys
+import argparse
+from pathlib import Path
+from textwrap import fill, indent
+
+
+INDENT = ' ' * 4
+
+CLASS = """
+
+class {name}({parents}):{doc}
+"""
+
+INIT = """
+    def __init__({args}):{fields}
+        super().__init__({format}"{message}")
+"""
+
+FUNCTIONS = ['claim_id']
+
+
+class ErrorClass:
+
+    def __init__(self, hierarchy, name, message):
+        self.hierarchy = hierarchy.replace('**', '')
+        self.other_parents = []
+        if '(' in name:
+            assert ')' in name, f"Missing closing parenthesis in '{name}'."
+            self.other_parents = name[name.find('(')+1:name.find(')')].split(',')
+            name = name[:name.find('(')]
+        self.name = name
+        self.class_name = name+'Error'
+        self.message = message
+        self.comment = ""
+        if '--' in message:
+            self.message, self.comment = message.split('--')
+        self.message = self.message.strip()
+        self.comment = self.comment.strip()
+
+    @property
+    def is_leaf(self):
+        return 'x' not in self.hierarchy
+
+    @property
+    def code(self):
+        return self.hierarchy.replace('x', '')
+
+    @property
+    def parent_codes(self):
+        return self.hierarchy[0:2], self.hierarchy[0]
+
+    def get_arguments(self):
+        args = ['self']
+        for arg in re.findall('{([a-z0-1_()]+)}', self.message):
+            for func in FUNCTIONS:
+                if arg.startswith(f'{func}('):
+                    arg = arg[len(f'{func}('):-1]
+                    break
+            args.append(arg)
+        return args
+
+    @staticmethod
+    def get_fields(args):
+        if len(args) > 1:
+            return ''.join(f'\n{INDENT*2}self.{field} = {field}' for field in args[1:])
+        return ''
+
+    @staticmethod
+    def get_doc_string(doc):
+        if doc:
+            return f'\n{INDENT}"""\n{indent(fill(doc, 100), INDENT)}\n{INDENT}"""'
+        return ""
+
+    def render(self, out, parent):
+        if not parent:
+            parents = ['BaseError']
+        else:
+            parents = [parent.class_name]
+        parents += self.other_parents
+        args = self.get_arguments()
+        if self.is_leaf:
+            out.write((CLASS + INIT).format(
+                name=self.class_name, parents=', '.join(parents),
+                args=', '.join(args), fields=self.get_fields(args),
+                message=self.message, doc=self.get_doc_string(self.comment), format='f' if len(args) > 1 else ''
+            ))
+        else:
+            out.write(CLASS.format(
+                name=self.class_name, parents=', '.join(parents),
+                doc=self.get_doc_string(self.comment or self.message)
+            ))
+
+
+def get_errors():
+    with open('README.md', 'r') as readme:
+        lines = iter(readme.readlines())
+        for line in lines:
+            if line.startswith('## Exceptions Table'):
+                break
+        for line in lines:
+            if line.startswith('---:|'):
+                break
+        for line in lines:
+            if not line:
+                break
+            yield ErrorClass(*[c.strip() for c in line.split('|')])
+
+
+def find_parent(stack, child):
+    for parent_code in child.parent_codes:
+        parent = stack.get(parent_code)
+        if parent:
+            return parent
+
+
+def generate(out):
+    out.write(f"from .base import BaseError, {', '.join(FUNCTIONS)}\n")
+    stack = {}
+    for error in get_errors():
+        error.render(out, find_parent(stack, error))
+        if not error.is_leaf:
+            assert error.code not in stack, f"Duplicate code: {error.code}"
+            stack[error.code] = error
+
+
+def analyze():
+    errors = {e.class_name: [] for e in get_errors() if e.is_leaf}
+    here = Path(__file__).absolute().parents[0]
+    module = here.parent
+    for file_path in module.glob('**/*.py'):
+        if here in file_path.parents:
+            continue
+        with open(file_path) as src_file:
+            src = src_file.read()
+            for error in errors.keys():
+                found = src.count(error)
+                if found > 0:
+                    errors[error].append((file_path, found))
+
+    print('Unused Errors:\n')
+    for error, used in errors.items():
+        if used:
+            print(f' - {error}')
+            for use in used:
+                print(f'   {use[0].relative_to(module.parent)} {use[1]}')
+            print('')
+
+    print('')
+    print('Unused Errors:')
+    for error, used in errors.items():
+        if not used:
+            print(f' - {error}')
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("action", choices=['generate', 'analyze'])
+    args = parser.parse_args()
+    if args.action == "analyze":
+        analyze()
+    elif args.action == "generate":
+        generate(sys.stdout)
+
+
+if __name__ == "__main__":
+    main()

scribe/hub/common.py

@@ -0,0 +1,209 @@
+import logging
+import itertools
+import time
+import json
+import typing
+import asyncio
+import inspect
+from asyncio import Event
+from collections import namedtuple
+from functools import partial, lru_cache
+from numbers import Number
+from asyncio import Queue
+from scribe.common import RPCError, CodeMessageError
+from scribe import PROMETHEUS_NAMESPACE
+
+
+HISTOGRAM_BUCKETS = (
+    .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
+)
+NAMESPACE = "scribe"
+
+
+SignatureInfo = namedtuple('SignatureInfo', 'min_args max_args '
+                           'required_names other_names')
+
+PARSE_ERROR = -32700
+INVALID_REQUEST = -32600
+METHOD_NOT_FOUND = -32601
+INVALID_ARGS = -32602
+INTERNAL_ERROR = -32603
+QUERY_TIMEOUT = -32000
+
+
+@lru_cache(256)
+def signature_info(func):
+    params = inspect.signature(func).parameters
+    min_args = max_args = 0
+    required_names = []
+    other_names = []
+    no_names = False
+    for p in params.values():
+        if p.kind == p.POSITIONAL_OR_KEYWORD:
+            max_args += 1
+            if p.default is p.empty:
+                min_args += 1
+                required_names.append(p.name)
+            else:
+                other_names.append(p.name)
+        elif p.kind == p.KEYWORD_ONLY:
+            other_names.append(p.name)
+        elif p.kind == p.VAR_POSITIONAL:
+            max_args = None
+        elif p.kind == p.VAR_KEYWORD:
+            other_names = any
+        elif p.kind == p.POSITIONAL_ONLY:
+            max_args += 1
+            if p.default is p.empty:
+                min_args += 1
+            no_names = True
+
+    if no_names:
+        other_names = None
+
+    return SignatureInfo(min_args, max_args, required_names, other_names)
+
+
+class BatchError(Exception):
+
+    def __init__(self, request):
+        self.request = request   # BatchRequest object
+
+
+class BatchRequest:
+    """Used to build a batch request to send to the server.  Stores
+    the
+
+    Attributes batch and results are initially None.
+
+    Adding an invalid request or notification immediately raises a
+    ProtocolError.
+
+    On exiting the with clause, it will:
+
+    1) create a Batch object for the requests in the order they were
+       added.  If the batch is empty this raises a ProtocolError.
+
+    2) set the "batch" attribute to be that batch
+
+    3) send the batch request and wait for a response
+
+    4) raise a ProtocolError if the protocol was violated by the
+       server.  Currently this only happens if it gave more than one
+       response to any request
+
+    5) otherwise there is precisely one response to each Request.  Set
+       the "results" attribute to the tuple of results; the responses
+       are ordered to match the Requests in the batch.  Notifications
+       do not get a response.
+
+    6) if raise_errors is True and any individual response was a JSON
+       RPC error response, or violated the protocol in some way, a
+       BatchError exception is raised.  Otherwise the caller can be
+       certain each request returned a standard result.
+    """
+
+    def __init__(self, session, raise_errors):
+        self._session = session
+        self._raise_errors = raise_errors
+        self._requests = []
+        self.batch = None
+        self.results = None
+
+    def add_request(self, method, args=()):
+        self._requests.append(Request(method, args))
+
+    def add_notification(self, method, args=()):
+        self._requests.append(Notification(method, args))
+
+    def __len__(self):
+        return len(self._requests)
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc_value, traceback):
+        if exc_type is None:
+            self.batch = Batch(self._requests)
+            message, event = self._session.connection.send_batch(self.batch)
+            await self._session._send_message(message)
+            await event.wait()
+            self.results = event.result
+            if self._raise_errors:
+                if any(isinstance(item, Exception) for item in event.result):
+                    raise BatchError(self)
+
+
+class SingleRequest:
+    __slots__ = ('method', 'args')
+
+    def __init__(self, method, args):
+        if not isinstance(method, str):
+            raise ProtocolError(METHOD_NOT_FOUND,
+                                'method must be a string')
+        if not isinstance(args, (list, tuple, dict)):
+            raise ProtocolError.invalid_args('request arguments must be a '
+                                             'list or a dictionary')
+        self.args = args
+        self.method = method
+
+    def __repr__(self):
+        return f'{self.__class__.__name__}({self.method!r}, {self.args!r})'
+
+    def __eq__(self, other):
+        return (isinstance(other, self.__class__) and
+                self.method == other.method and self.args == other.args)
+
+
+class Request(SingleRequest):
+    def send_result(self, response):
+        return None
+
+
+class Notification(SingleRequest):
+    pass
+
+
+class Batch:
+    __slots__ = ('items', )
+
+    def __init__(self, items):
+        if not isinstance(items, (list, tuple)):
+            raise ProtocolError.invalid_request('items must be a list')
+        if not items:
+            raise ProtocolError.empty_batch()
+        if not (all(isinstance(item, SingleRequest) for item in items) or
+                all(isinstance(item, Response) for item in items)):
+            raise ProtocolError.invalid_request('batch must be homogeneous')
+        self.items = items
+
+    def __len__(self):
+        return len(self.items)
+
+    def __getitem__(self, item):
+        return self.items[item]
+
+    def __iter__(self):
+        return iter(self.items)
+
+    def __repr__(self):
+        return f'Batch({len(self.items)} items)'
+
+
+class Response:
+    __slots__ = ('result', )
+
+    def __init__(self, result):
+        # Type checking happens when converting to a message
+        self.result = result
+
+
+class ProtocolError(CodeMessageError):
+    def __init__(self, code, message):
+        super().__init__(code, message)
+        # If not None send this unframed message over the network
+        self.error_message = None
+        # If the error was in a JSON response message; its message ID.
+        # Since None can be a response message ID, "id" means the
+        # error was not sent in a JSON response
+        self.response_msg_id = id

scribe/hub/framer.py

@@ -0,0 +1,51 @@
+from asyncio import Queue
+
+
+class NewlineFramer:
+    """A framer for a protocol where messages are separated by newlines."""
+
+    # The default max_size value is motivated by JSONRPC, where a
+    # normal request will be 250 bytes or less, and a reasonable
+    # batch may contain 4000 requests.
+    def __init__(self, max_size=250 * 4000):
+        """max_size - an anti-DoS measure.  If, after processing an incoming
+        message, buffered data would exceed max_size bytes, that
+        buffered data is dropped entirely and the framer waits for a
+        newline character to re-synchronize the stream.
+        """
+        self.max_size = max_size
+        self.queue = Queue()
+        self.received_bytes = self.queue.put_nowait
+        self.synchronizing = False
+        self.residual = b''
+
+    def frame(self, message):
+        return message + b'\n'
+
+    async def receive_message(self):
+        parts = []
+        buffer_size = 0
+        while True:
+            part = self.residual
+            self.residual = b''
+            if not part:
+                part = await self.queue.get()
+
+            npos = part.find(b'\n')
+            if npos == -1:
+                parts.append(part)
+                buffer_size += len(part)
+                # Ignore over-sized messages; re-synchronize
+                if buffer_size <= self.max_size:
+                    continue
+                self.synchronizing = True
+                raise MemoryError(f'dropping message over {self.max_size:,d} '
+                                  f'bytes and re-synchronizing')
+
+            tail, self.residual = part[:npos], part[npos + 1:]
+            if self.synchronizing:
+                self.synchronizing = False
+                return await self.receive_message()
+            else:
+                parts.append(tail)
+                return b''.join(parts)

scribe/hub/jsonrpc.py

@@ -0,0 +1,616 @@
+import itertools
+import json
+import typing
+import asyncio
+from asyncio import Event
+from functools import partial
+from numbers import Number
+from scribe.common import RPCError, CodeMessageError
+from scribe.hub.common import Notification, Request, Response, Batch, ProtocolError
+
+
+HISTOGRAM_BUCKETS = (
+    .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
+)
+NAMESPACE = "scribe"
+
+
+class JSONRPC:
+    """Abstract base class that interprets and constructs JSON RPC messages."""
+
+    # Error codes.  See http://www.jsonrpc.org/specification
+    PARSE_ERROR = -32700
+    INVALID_REQUEST = -32600
+    METHOD_NOT_FOUND = -32601
+    INVALID_ARGS = -32602
+    INTERNAL_ERROR = -32603
+    QUERY_TIMEOUT = -32000
+
+    # Codes specific to this library
+    ERROR_CODE_UNAVAILABLE = -100
+
+    # Can be overridden by derived classes
+    allow_batches = True
+
+    @classmethod
+    def _message_id(cls, message, require_id):
+        """Validate the message is a dictionary and return its ID.
+
+        Raise an error if the message is invalid or the ID is of an
+        invalid type.  If it has no ID, raise an error if require_id
+        is True, otherwise return None.
+        """
+        raise NotImplementedError
+
+    @classmethod
+    def _validate_message(cls, message):
+        """Validate other parts of the message other than those
+        done in _message_id."""
+        pass
+
+    @classmethod
+    def _request_args(cls, request):
+        """Validate the existence and type of the arguments passed
+        in the request dictionary."""
+        raise NotImplementedError
+
+    @classmethod
+    def _process_request(cls, payload):
+        request_id = None
+        try:
+            request_id = cls._message_id(payload, False)
+            cls._validate_message(payload)
+            method = payload.get('method')
+            if request_id is None:
+                item = Notification(method, cls._request_args(payload))
+            else:
+                item = Request(method, cls._request_args(payload))
+            return item, request_id
+        except ProtocolError as error:
+            code, message = error.code, error.message
+        raise cls._error(code, message, True, request_id)
+
+    @classmethod
+    def _process_response(cls, payload):
+        request_id = None
+        try:
+            request_id = cls._message_id(payload, True)
+            cls._validate_message(payload)
+            return Response(cls.response_value(payload)), request_id
+        except ProtocolError as error:
+            code, message = error.code, error.message
+        raise cls._error(code, message, False, request_id)
+
+    @classmethod
+    def _message_to_payload(cls, message):
+        """Returns a Python object or a ProtocolError."""
+        try:
+            return json.loads(message.decode())
+        except UnicodeDecodeError:
+            message = 'messages must be encoded in UTF-8'
+        except json.JSONDecodeError:
+            message = 'invalid JSON'
+        raise cls._error(cls.PARSE_ERROR, message, True, None)
+
+    @classmethod
+    def _error(cls, code, message, send, msg_id):
+        error = ProtocolError(code, message)
+        if send:
+            error.error_message = cls.response_message(error, msg_id)
+        else:
+            error.response_msg_id = msg_id
+        return error
+
+    #
+    # External API
+    #
+
+    @classmethod
+    def message_to_item(cls, message):
+        """Translate an unframed received message and return an
+        (item, request_id) pair.
+
+        The item can be a Request, Notification, Response or a list.
+
+        A JSON RPC error response is returned as an RPCError inside a
+        Response object.
+
+        If a Batch is returned, request_id is an iterable of request
+        ids, one per batch member.
+
+        If the message violates the protocol in some way a
+        ProtocolError is returned, except if the message was
+        determined to be a response, in which case the ProtocolError
+        is placed inside a Response object.  This is so that client
+        code can mark a request as having been responded to even if
+        the response was bad.
+
+        raises: ProtocolError
+        """
+        payload = cls._message_to_payload(message)
+        if isinstance(payload, dict):
+            if 'method' in payload:
+                return cls._process_request(payload)
+            else:
+                return cls._process_response(payload)
+        elif isinstance(payload, list) and cls.allow_batches:
+            if not payload:
+                raise cls._error(JSONRPC.INVALID_REQUEST, 'batch is empty',
+                                 True, None)
+            return payload, None
+        raise cls._error(cls.INVALID_REQUEST,
+                         'request object must be a dictionary', True, None)
+
+    # Message formation
+    @classmethod
+    def request_message(cls, item, request_id):
+        """Convert an RPCRequest item to a message."""
+        assert isinstance(item, Request)
+        return cls.encode_payload(cls.request_payload(item, request_id))
+
+    @classmethod
+    def notification_message(cls, item):
+        """Convert an RPCRequest item to a message."""
+        assert isinstance(item, Notification)
+        return cls.encode_payload(cls.request_payload(item, None))
+
+    @classmethod
+    def response_message(cls, result, request_id):
+        """Convert a response result (or RPCError) to a message."""
+        if isinstance(result, CodeMessageError):
+            payload = cls.error_payload(result, request_id)
+        else:
+            payload = cls.response_payload(result, request_id)
+        return cls.encode_payload(payload)
+
+    @classmethod
+    def batch_message(cls, batch, request_ids):
+        """Convert a request Batch to a message."""
+        assert isinstance(batch, Batch)
+        if not cls.allow_batches:
+            raise ProtocolError.invalid_request(
+                'protocol does not permit batches')
+        id_iter = iter(request_ids)
+        rm = cls.request_message
+        nm = cls.notification_message
+        parts = (rm(request, next(id_iter)) if isinstance(request, Request)
+                 else nm(request) for request in batch)
+        return cls.batch_message_from_parts(parts)
+
+    @classmethod
+    def batch_message_from_parts(cls, messages):
+        """Convert messages, one per batch item, into a batch message.  At
+        least one message must be passed.
+        """
+        # Comma-separate the messages and wrap the lot in square brackets
+        middle = b', '.join(messages)
+        if not middle:
+            raise ProtocolError.empty_batch()
+        return b''.join([b'[', middle, b']'])
+
+    @classmethod
+    def encode_payload(cls, payload):
+        """Encode a Python object as JSON and convert it to bytes."""
+        try:
+            return json.dumps(payload).encode()
+        except TypeError:
+            msg = f'JSON payload encoding error: {payload}'
+            raise ProtocolError(cls.INTERNAL_ERROR, msg) from None
+
+
+class JSONRPCv1(JSONRPC):
+    """JSON RPC version 1.0."""
+
+    allow_batches = False
+
+    @classmethod
+    def _message_id(cls, message, require_id):
+        # JSONv1 requires an ID always, but without constraint on its type
+        # No need to test for a dictionary here as we don't handle batches.
+        if 'id' not in message:
+            raise ProtocolError.invalid_request('request has no "id"')
+        return message['id']
+
+    @classmethod
+    def _request_args(cls, request):
+        args = request.get('params')
+        if not isinstance(args, list):
+            raise ProtocolError.invalid_args(
+                f'invalid request arguments: {args}')
+        return args
+
+    @classmethod
+    def _best_effort_error(cls, error):
+        # Do our best to interpret the error
+        code = cls.ERROR_CODE_UNAVAILABLE
+        message = 'no error message provided'
+        if isinstance(error, str):
+            message = error
+        elif isinstance(error, int):
+            code = error
+        elif isinstance(error, dict):
+            if isinstance(error.get('message'), str):
+                message = error['message']
+            if isinstance(error.get('code'), int):
+                code = error['code']
+
+        return RPCError(code, message)
+
+    @classmethod
+    def response_value(cls, payload):
+        if 'result' not in payload or 'error' not in payload:
+            raise ProtocolError.invalid_request(
+                'response must contain both "result" and "error"')
+
+        result = payload['result']
+        error = payload['error']
+        if error is None:
+            return result   # It seems None can be a valid result
+        if result is not None:
+            raise ProtocolError.invalid_request(
+                'response has a "result" and an "error"')
+
+        return cls._best_effort_error(error)
+
+    @classmethod
+    def request_payload(cls, request, request_id):
+        """JSON v1 request (or notification) payload."""
+        if isinstance(request.args, dict):
+            raise ProtocolError.invalid_args(
+                'JSONRPCv1 does not support named arguments')
+        return {
+            'method': request.method,
+            'params': request.args,
+            'id': request_id
+        }
+
+    @classmethod
+    def response_payload(cls, result, request_id):
+        """JSON v1 response payload."""
+        return {
+            'result': result,
+            'error': None,
+            'id': request_id
+        }
+
+    @classmethod
+    def error_payload(cls, error, request_id):
+        return {
+            'result': None,
+            'error': {'code': error.code, 'message': error.message},
+            'id': request_id
+        }
+
+
+class JSONRPCv2(JSONRPC):
+    """JSON RPC version 2.0."""
+
+    @classmethod
+    def _message_id(cls, message, require_id):
+        if not isinstance(message, dict):
+            raise ProtocolError.invalid_request(
+                'request object must be a dictionary')
+        if 'id' in message:
+            request_id = message['id']
+            if not isinstance(request_id, (Number, str, type(None))):
+                raise ProtocolError.invalid_request(
+                    f'invalid "id": {request_id}')
+            return request_id
+        else:
+            if require_id:
+                raise ProtocolError.invalid_request('request has no "id"')
+            return None
+
+    @classmethod
+    def _validate_message(cls, message):
+        if message.get('jsonrpc') != '2.0':
+            raise ProtocolError.invalid_request('"jsonrpc" is not "2.0"')
+
+    @classmethod
+    def _request_args(cls, request):
+        args = request.get('params', [])
+        if not isinstance(args, (dict, list)):
+            raise ProtocolError.invalid_args(
+                f'invalid request arguments: {args}')
+        return args
+
+    @classmethod
+    def response_value(cls, payload):
+        if 'result' in payload:
+            if 'error' in payload:
+                raise ProtocolError.invalid_request(
+                    'response contains both "result" and "error"')
+            return payload['result']
+
+        if 'error' not in payload:
+            raise ProtocolError.invalid_request(
+                'response contains neither "result" nor "error"')
+
+        # Return an RPCError object
+        error = payload['error']
+        if isinstance(error, dict):
+            code = error.get('code')
+            message = error.get('message')
+            if isinstance(code, int) and isinstance(message, str):
+                return RPCError(code, message)
+
+        raise ProtocolError.invalid_request(
+            f'ill-formed response error object: {error}')
+
+    @classmethod
+    def request_payload(cls, request, request_id):
+        """JSON v2 request (or notification) payload."""
+        payload = {
+            'jsonrpc': '2.0',
+            'method': request.method,
+        }
+        # A notification?
+        if request_id is not None:
+            payload['id'] = request_id
+        # Preserve empty dicts as missing params is read as an array
+        if request.args or request.args == {}:
+            payload['params'] = request.args
+        return payload
+
+    @classmethod
+    def response_payload(cls, result, request_id):
+        """JSON v2 response payload."""
+        return {
+            'jsonrpc': '2.0',
+            'result': result,
+            'id': request_id
+        }
+
+    @classmethod
+    def error_payload(cls, error, request_id):
+        return {
+            'jsonrpc': '2.0',
+            'error': {'code': error.code, 'message': error.message},
+            'id': request_id
+        }
+
+
+class JSONRPCLoose(JSONRPC):
+    """A relaxed version of JSON RPC."""
+
+    # Don't be so loose we accept any old message ID
+    _message_id = JSONRPCv2._message_id
+    _validate_message = JSONRPC._validate_message
+    _request_args = JSONRPCv2._request_args
+    # Outoing messages are JSONRPCv2 so we give the other side the
+    # best chance to assume / detect JSONRPCv2 as default protocol.
+    error_payload = JSONRPCv2.error_payload
+    request_payload = JSONRPCv2.request_payload
+    response_payload = JSONRPCv2.response_payload
+
+    @classmethod
+    def response_value(cls, payload):
+        # Return result, unless it is None and there is an error
+        if payload.get('error') is not None:
+            if payload.get('result') is not None:
+                raise ProtocolError.invalid_request(
+                    'response contains both "result" and "error"')
+            return JSONRPCv1._best_effort_error(payload['error'])
+
+        if 'result' not in payload:
+            raise ProtocolError.invalid_request(
+                'response contains neither "result" nor "error"')
+
+        # Can be None
+        return payload['result']
+
+
+class JSONRPCAutoDetect(JSONRPCv2):
+
+    @classmethod
+    def message_to_item(cls, message):
+        return cls.detect_protocol(message), None
+
+    @classmethod
+    def detect_protocol(cls, message):
+        """Attempt to detect the protocol from the message."""
+        main = cls._message_to_payload(message)
+
+        def protocol_for_payload(payload):
+            if not isinstance(payload, dict):
+                return JSONRPCLoose   # Will error
+            # Obey an explicit "jsonrpc"
+            version = payload.get('jsonrpc')
+            if version == '2.0':
+                return JSONRPCv2
+            if version == '1.0':
+                return JSONRPCv1
+
+            # Now to decide between JSONRPCLoose and JSONRPCv1 if possible
+            if 'result' in payload and 'error' in payload:
+                return JSONRPCv1
+            return JSONRPCLoose
+
+        if isinstance(main, list):
+            parts = {protocol_for_payload(payload) for payload in main}
+            # If all same protocol, return it
+            if len(parts) == 1:
+                return parts.pop()
+            # If strict protocol detected, return it, preferring JSONRPCv2.
+            # This means a batch of JSONRPCv1 will fail
+            for protocol in (JSONRPCv2, JSONRPCv1):
+                if protocol in parts:
+                    return protocol
+            # Will error if no parts
+            return JSONRPCLoose
+
+        return protocol_for_payload(main)
+
+
+class JSONRPCConnection:
+    """Maintains state of a JSON RPC connection, in particular
+    encapsulating the handling of request IDs.
+
+    protocol - the JSON RPC protocol to follow
+    max_response_size - responses over this size send an error response
+        instead.
+    """
+
+    _id_counter = itertools.count()
+
+    def __init__(self, protocol):
+        self._protocol = protocol
+        # Sent Requests and Batches that have not received a response.
+        # The key is its request ID; for a batch it is sorted tuple
+        # of request IDs
+        self._requests: typing.Dict[str, typing.Tuple[Request, Event]] = {}
+        # A public attribute intended to be settable dynamically
+        self.max_response_size = 0
+
+    def _oversized_response_message(self, request_id):
+        text = f'response too large (over {self.max_response_size:,d} bytes'
+        error = RPCError.invalid_request(text)
+        return self._protocol.response_message(error, request_id)
+
+    def _receive_response(self, result, request_id):
+        if request_id not in self._requests:
+            if request_id is None and isinstance(result, RPCError):
+                message = f'diagnostic error received: {result}'
+            else:
+                message = f'response to unsent request (ID: {request_id})'
+            raise ProtocolError.invalid_request(message) from None
+        request, event = self._requests.pop(request_id)
+        event.result = result
+        event.set()
+        return []
+
+    def _receive_request_batch(self, payloads):
+        def item_send_result(request_id, result):
+            nonlocal size
+            part = protocol.response_message(result, request_id)
+            size += len(part) + 2
+            if size > self.max_response_size > 0:
+                part = self._oversized_response_message(request_id)
+            parts.append(part)
+            if len(parts) == count:
+                return protocol.batch_message_from_parts(parts)
+            return None
+
+        parts = []
+        items = []
+        size = 0
+        count = 0
+        protocol = self._protocol
+        for payload in payloads:
+            try:
+                item, request_id = protocol._process_request(payload)
+                items.append(item)
+                if isinstance(item, Request):
+                    count += 1
+                    item.send_result = partial(item_send_result, request_id)
+            except ProtocolError as error:
+                count += 1
+                parts.append(error.error_message)
+
+        if not items and parts:
+            protocol_error = ProtocolError(0, "")
+            protocol_error.error_message = protocol.batch_message_from_parts(parts)
+            raise protocol_error
+        return items
+
+    def _receive_response_batch(self, payloads):
+        request_ids = []
+        results = []
+        for payload in payloads:
+            # Let ProtocolError exceptions through
+            item, request_id = self._protocol._process_response(payload)
+            request_ids.append(request_id)
+            results.append(item.result)
+
+        ordered = sorted(zip(request_ids, results), key=lambda t: t[0])
+        ordered_ids, ordered_results = zip(*ordered)
+        if ordered_ids not in self._requests:
+            raise ProtocolError.invalid_request('response to unsent batch')
+        request_batch, event = self._requests.pop(ordered_ids)
+        event.result = ordered_results
+        event.set()
+        return []
+
+    def _send_result(self, request_id, result):
+        message = self._protocol.response_message(result, request_id)
+        if len(message) > self.max_response_size > 0:
+            message = self._oversized_response_message(request_id)
+        return message
+
+    def _event(self, request, request_id):
+        event = Event()
+        self._requests[request_id] = (request, event)
+        return event
+
+    #
+    # External API
+    #
+    def send_request(self, request: Request) -> typing.Tuple[bytes, Event]:
+        """Send a Request.  Return a (message, event) pair.
+
+        The message is an unframed message to send over the network.
+        Wait on the event for the response; which will be in the
+        "result" attribute.
+
+        Raises: ProtocolError if the request violates the protocol
+        in some way..
+        """
+        request_id = next(self._id_counter)
+        message = self._protocol.request_message(request, request_id)
+        return message, self._event(request, request_id)
+
+    def send_notification(self, notification):
+        return self._protocol.notification_message(notification)
+
+    def send_batch(self, batch):
+        ids = tuple(next(self._id_counter)
+                    for request in batch if isinstance(request, Request))
+        message = self._protocol.batch_message(batch, ids)
+        event = self._event(batch, ids) if ids else None
+        return message, event
+
+    def receive_message(self, message):
+        """Call with an unframed message received from the network.
+
+        Raises: ProtocolError if the message violates the protocol in
+        some way.  However, if it happened in a response that can be
+        paired with a request, the ProtocolError is instead set in the
+        result attribute of the send_request() that caused the error.
+        """
+        try:
+            item, request_id = self._protocol.message_to_item(message)
+        except ProtocolError as e:
+            if e.response_msg_id is not id:
+                return self._receive_response(e, e.response_msg_id)
+            raise
+
+        if isinstance(item, Request):
+            item.send_result = partial(self._send_result, request_id)
+            return [item]
+        if isinstance(item, Notification):
+            return [item]
+        if isinstance(item, Response):
+            return self._receive_response(item.result, request_id)
+        if isinstance(item, list):
+            if all(isinstance(payload, dict)
+                   and ('result' in payload or 'error' in payload)
+                   for payload in item):
+                return self._receive_response_batch(item)
+            else:
+                return self._receive_request_batch(item)
+        else:
+            # Protocol auto-detection hack
+            assert issubclass(item, JSONRPC)
+            self._protocol = item
+            return self.receive_message(message)
+
+    def raise_pending_requests(self, exception):
+        exception = exception or asyncio.TimeoutError()
+        for request, event in self._requests.values():
+            event.result = exception
+            event.set()
+        self._requests.clear()
+
+    def pending_requests(self):
+        """All sent requests that have not received a response."""
+        return [request for request, event in self._requests.values()]

scribe/hub/mempool.py

@@ -0,0 +1,200 @@
+import asyncio
+import itertools
+import attr
+import typing
+import logging
+from collections import defaultdict
+from prometheus_client import Histogram
+from scribe import PROMETHEUS_NAMESPACE
+from scribe.blockchain.transaction.deserializer import Deserializer
+
+if typing.TYPE_CHECKING:
+    from scribe.hub.session import SessionManager
+    from scribe.db import HubDB
+
+
+@attr.s(slots=True)
+class MemPoolTx:
+    prevouts = attr.ib()
+    # A pair is a (hashX, value) tuple
+    in_pairs = attr.ib()
+    out_pairs = attr.ib()
+    fee = attr.ib()
+    size = attr.ib()
+    raw_tx = attr.ib()
+
+
+@attr.s(slots=True)
+class MemPoolTxSummary:
+    hash = attr.ib()
+    fee = attr.ib()
+    has_unconfirmed_inputs = attr.ib()
+
+
+NAMESPACE = f"{PROMETHEUS_NAMESPACE}_mempool"
+HISTOGRAM_BUCKETS = (
+    .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
+)
+mempool_process_time_metric = Histogram(
+    "processed_mempool", "Time to process mempool and notify touched addresses",
+    namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS
+)
+
+
+class MemPool:
+    def __init__(self, coin, db: 'HubDB', refresh_secs=1.0):
+        self.coin = coin
+        self._db = db
+        self.logger = logging.getLogger(__name__)
+        self.txs = {}
+        self.raw_mempool = {}
+        self.touched_hashXs: typing.DefaultDict[bytes, typing.Set[bytes]] = defaultdict(set)  # None can be a key
+        self.refresh_secs = refresh_secs
+        self.mempool_process_time_metric = mempool_process_time_metric
+        self.session_manager: typing.Optional['SessionManager'] = None
+
+    def refresh(self) -> typing.Set[bytes]:  # returns list of new touched hashXs
+        prefix_db = self._db.prefix_db
+        new_mempool = {k.tx_hash: v.raw_tx for k, v in prefix_db.mempool_tx.iterate()}  # TODO: make this more efficient
+        self.raw_mempool.clear()
+        self.raw_mempool.update(new_mempool)
+
+        # hashXs = self.hashXs  # hashX: [tx_hash, ...]
+        touched_hashXs = set()
+
+        # Remove txs that aren't in mempool anymore
+        for tx_hash in set(self.txs).difference(self.raw_mempool.keys()):
+            tx = self.txs.pop(tx_hash)
+            tx_hashXs = {hashX for hashX, value in tx.in_pairs}.union({hashX for hashX, value in tx.out_pairs})
+            for hashX in tx_hashXs:
+                if hashX in self.touched_hashXs and tx_hash in self.touched_hashXs[hashX]:
+                    self.touched_hashXs[hashX].remove(tx_hash)
+                    if not self.touched_hashXs[hashX]:
+                        self.touched_hashXs.pop(hashX)
+            touched_hashXs.update(tx_hashXs)
+
+        # Re-sync with the new set of hashes
+        tx_map = {}
+        for tx_hash, raw_tx in self.raw_mempool.items():
+            if tx_hash in self.txs:
+                continue
+            tx, tx_size = Deserializer(raw_tx).read_tx_and_vsize()
+            # Convert the inputs and outputs into (hashX, value) pairs
+            # Drop generation-like inputs from MemPoolTx.prevouts
+            txin_pairs = tuple((txin.prev_hash, txin.prev_idx)
+                               for txin in tx.inputs
+                               if not txin.is_generation())
+            txout_pairs = tuple((self.coin.hashX_from_txo(txout), txout.value)
+                                for txout in tx.outputs if txout.pk_script)
+
+            tx_map[tx_hash] = MemPoolTx(None, txin_pairs, txout_pairs, 0, tx_size, raw_tx)
+
+        for tx_hash, tx in tx_map.items():
+            prevouts = []
+            # Look up the prevouts
+            for prev_hash, prev_index in tx.in_pairs:
+                if prev_hash in self.txs:  # accepted mempool
+                    utxo = self.txs[prev_hash].out_pairs[prev_index]
+                elif prev_hash in tx_map:  # this set of changes
+                    utxo = tx_map[prev_hash].out_pairs[prev_index]
+                else:  # get it from the db
+                    prev_tx_num = prefix_db.tx_num.get(prev_hash)
+                    if not prev_tx_num:
+                        continue
+                    prev_tx_num = prev_tx_num.tx_num
+                    hashX_val = prefix_db.hashX_utxo.get(prev_hash[:4], prev_tx_num, prev_index)
+                    if not hashX_val:
+                        continue
+                    hashX = hashX_val.hashX
+                    utxo_value = prefix_db.utxo.get(hashX, prev_tx_num, prev_index)
+                    utxo = (hashX, utxo_value.amount)
+                prevouts.append(utxo)
+
+            # Save the prevouts, compute the fee and accept the TX
+            tx.prevouts = tuple(prevouts)
+            # Avoid negative fees if dealing with generation-like transactions
+            # because some in_parts would be missing
+            tx.fee = max(0, (sum(v for _, v in tx.prevouts) -
+                             sum(v for _, v in tx.out_pairs)))
+            self.txs[tx_hash] = tx
+            # print(f"added {tx_hash[::-1].hex()} reader to mempool")
+
+            for hashX, value in itertools.chain(tx.prevouts, tx.out_pairs):
+                self.touched_hashXs[hashX].add(tx_hash)
+                touched_hashXs.add(hashX)
+        return touched_hashXs
+
+    def transaction_summaries(self, hashX):
+        """Return a list of MemPoolTxSummary objects for the hashX."""
+        result = []
+        for tx_hash in self.touched_hashXs.get(hashX, ()):
+            if tx_hash not in self.txs:
+                continue  # the tx hash for the touched address is an input that isn't in mempool anymore
+            tx = self.txs[tx_hash]
+            has_ui = any(hash in self.txs for hash, idx in tx.in_pairs)
+            result.append(MemPoolTxSummary(tx_hash, tx.fee, has_ui))
+        return result
+
+    def get_mempool_height(self, tx_hash: bytes) -> int:
+        # Height Progression
+        #   -2: not broadcast
+        #   -1: in mempool but has unconfirmed inputs
+        #    0: in mempool and all inputs confirmed
+        # +num: confirmed in a specific block (height)
+        if tx_hash not in self.txs:
+            return -2
+        tx = self.txs[tx_hash]
+        unspent_inputs = any(hash in self.raw_mempool for hash, idx in tx.in_pairs)
+        if unspent_inputs:
+            return -1
+        return 0
+
+    async def start(self, height, session_manager: 'SessionManager'):
+        self.session_manager = session_manager
+        await self._notify_sessions(height, set(), set())
+
+    async def on_mempool(self, touched, new_touched, height):
+        await self._notify_sessions(height, touched, new_touched)
+
+    async def on_block(self, touched, height):
+        await self._notify_sessions(height, touched, set())
+
+    async def _notify_sessions(self, height, touched, new_touched):
+        """Notify sessions about height changes and touched addresses."""
+        height_changed = height != self.session_manager.notified_height
+        if height_changed:
+            await self.session_manager._refresh_hsub_results(height)
+
+        if not self.session_manager.sessions:
+            return
+
+        if height_changed:
+            header_tasks = [
+                session.send_notification('blockchain.headers.subscribe', (self.session_manager.hsub_results[session.subscribe_headers_raw], ))
+                for session in self.session_manager.sessions.values() if session.subscribe_headers
+            ]
+            if header_tasks:
+                self.logger.info(f'notify {len(header_tasks)} sessions of new header')
+                asyncio.create_task(asyncio.wait(header_tasks))
+            for hashX in touched.intersection(self.session_manager.mempool_statuses.keys()):
+                self.session_manager.mempool_statuses.pop(hashX, None)
+        # self.bp._chain_executor
+        await asyncio.get_event_loop().run_in_executor(
+            self._db._executor, touched.intersection_update, self.session_manager.hashx_subscriptions_by_session.keys()
+        )
+
+        if touched or new_touched or (height_changed and self.session_manager.mempool_statuses):
+            notified_hashxs = 0
+            session_hashxes_to_notify = defaultdict(list)
+            to_notify = touched if height_changed else new_touched
+
+            for hashX in to_notify:
+                if hashX not in self.session_manager.hashx_subscriptions_by_session:
+                    continue
+                for session_id in self.session_manager.hashx_subscriptions_by_session[hashX]:
+                    session_hashxes_to_notify[session_id].append(hashX)
+                    notified_hashxs += 1
+            for session_id, hashXes in session_hashxes_to_notify.items():
+                asyncio.create_task(self.session_manager.sessions[session_id].send_history_notifications(*hashXes))
+            if session_hashxes_to_notify:
+                self.logger.info(f'notified {len(session_hashxes_to_notify)} sessions/{notified_hashxs:,d} touched addresses')

scribe/hub/prometheus.py

@@ -0,0 +1,68 @@
+import time
+import logging
+import asyncio
+import asyncio.tasks
+from aiohttp import web
+from prometheus_client import generate_latest as prom_generate_latest
+from prometheus_client import Counter, Histogram, Gauge
+
+
+PROBES_IN_FLIGHT = Counter("probes_in_flight", "Number of loop probes in flight", namespace='asyncio')
+PROBES_FINISHED = Counter("probes_finished", "Number of finished loop probes", namespace='asyncio')
+PROBE_TIMES = Histogram("probe_times", "Loop probe times", namespace='asyncio')
+TASK_COUNT = Gauge("running_tasks", "Number of running tasks", namespace='asyncio')
+
+
+def get_loop_metrics(delay=1):
+    loop = asyncio.get_event_loop()
+
+    def callback(started):
+        PROBE_TIMES.observe(time.perf_counter() - started - delay)
+        PROBES_FINISHED.inc()
+
+    async def monitor_loop_responsiveness():
+        while True:
+            now = time.perf_counter()
+            loop.call_later(delay, callback, now)
+            PROBES_IN_FLIGHT.inc()
+            TASK_COUNT.set(len(asyncio.tasks._all_tasks))
+            await asyncio.sleep(delay)
+
+    return loop.create_task(monitor_loop_responsiveness())
+
+
+class PrometheusServer:
+    def __init__(self, logger=None):
+        self.runner = None
+        self.logger = logger or logging.getLogger(__name__)
+        self._monitor_loop_task = None
+
+    async def start(self, interface: str, port: int):
+        self.logger.info("start prometheus metrics")
+        prom_app = web.Application()
+        prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
+        self.runner = web.AppRunner(prom_app)
+        await self.runner.setup()
+
+        metrics_site = web.TCPSite(self.runner, interface, port, shutdown_timeout=.5)
+        await metrics_site.start()
+        self.logger.info(
+            'prometheus metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2]
+        )
+        self._monitor_loop_task = get_loop_metrics()
+
+    async def handle_metrics_get_request(self, request: web.Request):
+        try:
+            return web.Response(
+                text=prom_generate_latest().decode(),
+                content_type='text/plain; version=0.0.4'
+            )
+        except Exception:
+            self.logger.exception('could not generate prometheus data')
+            raise
+
+    async def stop(self):
+        if self._monitor_loop_task and not self._monitor_loop_task.done():
+            self._monitor_loop_task.cancel()
+        self._monitor_loop_task = None
+        await self.runner.cleanup()

scribe/hub/udp.py

@@ -0,0 +1,240 @@
+import asyncio
+import struct
+from time import perf_counter
+import logging
+from typing import Optional, Tuple, NamedTuple
+from scribe.schema.attrs import country_str_to_int, country_int_to_str
+from scribe.common import LRUCache, is_valid_public_ipv4
+# from prometheus_client import Counter
+
+
+log = logging.getLogger(__name__)
+_MAGIC = 1446058291  # genesis blocktime (which is actually wrong)
+# ping_count_metric = Counter("ping_count", "Number of pings received", namespace='wallet_server_status')
+_PAD_BYTES = b'\x00' * 64
+
+
+PROTOCOL_VERSION = 1
+
+
+class SPVPing(NamedTuple):
+    magic: int
+    protocol_version: int
+    pad_bytes: bytes
+
+    def encode(self):
+        return struct.pack(b'!lB64s', *self)
+
+    @staticmethod
+    def make() -> bytes:
+        return SPVPing(_MAGIC, PROTOCOL_VERSION, _PAD_BYTES).encode()
+
+    @classmethod
+    def decode(cls, packet: bytes):
+        decoded = cls(*struct.unpack(b'!lB64s', packet[:69]))
+        if decoded.magic != _MAGIC:
+            raise ValueError("invalid magic bytes")
+        return decoded
+
+
+PONG_ENCODING = b'!BBL32s4sH'
+
+
+class SPVPong(NamedTuple):
+    protocol_version: int
+    flags: int
+    height: int
+    tip: bytes
+    source_address_raw: bytes
+    country: int
+
+    def encode(self):
+        return struct.pack(PONG_ENCODING, *self)
+
+    @staticmethod
+    def encode_address(address: str):
+        return bytes(int(b) for b in address.split("."))
+
+    @classmethod
+    def make(cls, flags: int, height: int, tip: bytes, source_address: str, country: str) -> bytes:
+        return SPVPong(
+            PROTOCOL_VERSION, flags, height, tip,
+            cls.encode_address(source_address),
+            country_str_to_int(country)
+        ).encode()
+
+    @classmethod
+    def make_sans_source_address(cls, flags: int, height: int, tip: bytes, country: str) -> Tuple[bytes, bytes]:
+        pong = cls.make(flags, height, tip, '0.0.0.0', country)
+        return pong[:38], pong[42:]
+
+    @classmethod
+    def decode(cls, packet: bytes):
+        return cls(*struct.unpack(PONG_ENCODING, packet[:44]))
+
+    @property
+    def available(self) -> bool:
+        return (self.flags & 0b00000001) > 0
+
+    @property
+    def ip_address(self) -> str:
+        return ".".join(map(str, self.source_address_raw))
+
+    @property
+    def country_name(self):
+        return country_int_to_str(self.country)
+
+    def __repr__(self) -> str:
+        return f"SPVPong(external_ip={self.ip_address}, version={self.protocol_version}, " \
+               f"available={'True' if self.flags & 1 > 0 else 'False'}," \
+               f" height={self.height}, tip={self.tip[::-1].hex()}, country={self.country_name})"
+
+
+class SPVServerStatusProtocol(asyncio.DatagramProtocol):
+
+    def __init__(
+        self, height: int, tip: bytes, country: str,
+        throttle_cache_size: int = 1024, throttle_reqs_per_sec: int = 10,
+        allow_localhost: bool = False, allow_lan: bool = False
+    ):
+        super().__init__()
+        self.transport: Optional[asyncio.transports.DatagramTransport] = None
+        self._height = height
+        self._tip = tip
+        self._flags = 0
+        self._country = country
+        self._left_cache = self._right_cache = None
+        self.update_cached_response()
+        self._throttle = LRUCache(throttle_cache_size)
+        self._should_log = LRUCache(throttle_cache_size)
+        self._min_delay = 1 / throttle_reqs_per_sec
+        self._allow_localhost = allow_localhost
+        self._allow_lan = allow_lan
+        self.closed = asyncio.Event()
+
+    def update_cached_response(self):
+        self._left_cache, self._right_cache = SPVPong.make_sans_source_address(
+            self._flags, max(0, self._height), self._tip, self._country
+        )
+
+    def set_unavailable(self):
+        self._flags &= 0b11111110
+        self.update_cached_response()
+
+    def set_available(self):
+        self._flags |= 0b00000001
+        self.update_cached_response()
+
+    def set_height(self, height: int, tip: bytes):
+        self._height, self._tip = height, tip
+        self.update_cached_response()
+
+    def should_throttle(self, host: str):
+        now = perf_counter()
+        last_requested = self._throttle.get(host, default=0)
+        self._throttle[host] = now
+        if now - last_requested < self._min_delay:
+            log_cnt = self._should_log.get(host, default=0) + 1
+            if log_cnt % 100 == 0:
+                log.warning("throttle spv status to %s", host)
+            self._should_log[host] = log_cnt
+            return True
+        return False
+
+    def make_pong(self, host):
+        return self._left_cache + SPVPong.encode_address(host) + self._right_cache
+
+    def datagram_received(self, data: bytes, addr: Tuple[str, int]):
+        if self.should_throttle(addr[0]):
+            return
+        try:
+            SPVPing.decode(data)
+        except (ValueError, struct.error, AttributeError, TypeError):
+            # log.exception("derp")
+            return
+        if addr[1] >= 1024 and is_valid_public_ipv4(
+                addr[0], allow_localhost=self._allow_localhost, allow_lan=self._allow_lan):
+            self.transport.sendto(self.make_pong(addr[0]), addr)
+        else:
+            log.warning("odd packet from %s:%i", addr[0], addr[1])
+        # ping_count_metric.inc()
+
+    def connection_made(self, transport) -> None:
+        self.transport = transport
+        self.closed.clear()
+
+    def connection_lost(self, exc: Optional[Exception]) -> None:
+        self.transport = None
+        self.closed.set()
+
+    async def close(self):
+        if self.transport:
+            self.transport.close()
+        await self.closed.wait()
+
+
+class StatusServer:
+    def __init__(self):
+        self._protocol: Optional[SPVServerStatusProtocol] = None
+
+    async def start(self, height: int, tip: bytes, country: str, interface: str, port: int, allow_lan: bool = False):
+        if self.is_running:
+            return
+        loop = asyncio.get_event_loop()
+        interface = interface if interface.lower() != 'localhost' else '127.0.0.1'
+        self._protocol = SPVServerStatusProtocol(
+            height, tip, country, allow_localhost=interface == '127.0.0.1', allow_lan=allow_lan
+        )
+        await loop.create_datagram_endpoint(lambda: self._protocol, (interface, port))
+        log.info("started udp status server on %s:%i", interface, port)
+
+    async def stop(self):
+        if self.is_running:
+            await self._protocol.close()
+            self._protocol = None
+
+    @property
+    def is_running(self):
+        return self._protocol is not None
+
+    def set_unavailable(self):
+        if self.is_running:
+            self._protocol.set_unavailable()
+
+    def set_available(self):
+        if self.is_running:
+            self._protocol.set_available()
+
+    def set_height(self, height: int, tip: bytes):
+        if self.is_running:
+            self._protocol.set_height(height, tip)
+
+
+class SPVStatusClientProtocol(asyncio.DatagramProtocol):
+
+    def __init__(self, responses: asyncio.Queue):
+        super().__init__()
+        self.transport: Optional[asyncio.transports.DatagramTransport] = None
+        self.responses = responses
+        self._ping_packet = SPVPing.make()
+
+    def datagram_received(self, data: bytes, addr: Tuple[str, int]):
+        try:
+            self.responses.put_nowait(((addr, perf_counter()), SPVPong.decode(data)))
+        except (ValueError, struct.error, AttributeError, TypeError, RuntimeError):
+            return
+
+    def connection_made(self, transport) -> None:
+        self.transport = transport
+
+    def connection_lost(self, exc: Optional[Exception]) -> None:
+        self.transport = None
+        log.info("closed udp spv server selection client")
+
+    def ping(self, server: Tuple[str, int]):
+        self.transport.sendto(self._ping_packet, server)
+
+    def close(self):
+        # log.info("close udp client")
+        if self.transport:
+            self.transport.close()

scribe/readers/__init__.py

@@ -0,0 +1,3 @@
+from scribe.readers.interface import BaseBlockchainReader
+from scribe.readers.hub_server import BlockchainReaderServer
+from scribe.readers.elastic_sync import ElasticWriter

scribe/readers/elastic_sync.py

@@ -0,0 +1,421 @@
+import os
+import signal
+import json
+import typing
+import struct
+from collections import defaultdict
+import asyncio
+import logging
+from decimal import Decimal
+from elasticsearch import AsyncElasticsearch, NotFoundError
+from elasticsearch.helpers import async_streaming_bulk
+from prometheus_client import Gauge, Histogram
+
+from scribe.schema.result import Censor
+from scribe import PROMETHEUS_NAMESPACE
+from scribe.elasticsearch.notifier_protocol import ElasticNotifierProtocol
+from scribe.elasticsearch.search import IndexVersionMismatch, expand_query
+from scribe.elasticsearch.constants import ALL_FIELDS, INDEX_DEFAULT_SETTINGS
+from scribe.elasticsearch.fast_ar_trending import FAST_AR_TRENDING_SCRIPT
+from scribe.readers import BaseBlockchainReader
+from scribe.db.revertable import RevertableOp
+from scribe.db.common import TrendingNotification, DB_PREFIXES
+
+
+log = logging.getLogger(__name__)
+
+NAMESPACE = f"{PROMETHEUS_NAMESPACE}_elastic_sync"
+HISTOGRAM_BUCKETS = (
+    .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
+)
+
+
+class ElasticWriter(BaseBlockchainReader):
+    VERSION = 1
+    prometheus_namespace = ""
+    block_count_metric = Gauge(
+        "block_count", "Number of processed blocks", namespace=NAMESPACE
+    )
+    block_update_time_metric = Histogram(
+        "block_time", "Block update times", namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS
+    )
+    reorg_count_metric = Gauge(
+        "reorg_count", "Number of reorgs", namespace=NAMESPACE
+    )
+
+    def __init__(self, env):
+        super().__init__(env, 'lbry-elastic-writer', thread_workers=1, thread_prefix='lbry-elastic-writer')
+        # self._refresh_interval = 0.1
+        self._task = None
+        self.index = self.env.es_index_prefix + 'claims'
+        self._elastic_host = env.elastic_host
+        self._elastic_port = env.elastic_port
+        self.sync_timeout = 1800
+        self.sync_client = None
+        self._es_info_path = os.path.join(env.db_dir, 'es_info')
+        self._last_wrote_height = 0
+        self._last_wrote_block_hash = None
+
+        self._touched_claims = set()
+        self._deleted_claims = set()
+
+        self._removed_during_undo = set()
+
+        self._trending = defaultdict(list)
+        self._advanced = True
+        self.synchronized = asyncio.Event()
+        self._listeners: typing.List[ElasticNotifierProtocol] = []
+
+    async def run_es_notifier(self, synchronized: asyncio.Event):
+        server = await asyncio.get_event_loop().create_server(
+            lambda: ElasticNotifierProtocol(self._listeners), '127.0.0.1', self.env.elastic_notifier_port
+        )
+        self.log.info("ES notifier server listening on TCP localhost:%i", self.env.elastic_notifier_port)
+        synchronized.set()
+        async with server:
+            await server.serve_forever()
+
+    def notify_es_notification_listeners(self, height: int, block_hash: bytes):
+        for p in self._listeners:
+            p.send_height(height, block_hash)
+            self.log.info("notify listener %i", height)
+
+    def _read_es_height(self):
+        info = {}
+        if os.path.exists(self._es_info_path):
+            with open(self._es_info_path, 'r') as f:
+                info.update(json.loads(f.read()))
+        self._last_wrote_height = int(info.get('height', 0))
+        self._last_wrote_block_hash = info.get('block_hash', None)
+
+    async def read_es_height(self):
+        await asyncio.get_event_loop().run_in_executor(self._executor, self._read_es_height)
+
+    def write_es_height(self, height: int, block_hash: str):
+        with open(self._es_info_path, 'w') as f:
+            f.write(json.dumps({'height': height, 'block_hash': block_hash}, indent=2))
+        self._last_wrote_height = height
+        self._last_wrote_block_hash = block_hash
+
+    async def get_index_version(self) -> int:
+        try:
+            template = await self.sync_client.indices.get_template(self.index)
+            return template[self.index]['version']
+        except NotFoundError:
+            return 0
+
+    async def set_index_version(self, version):
+        await self.sync_client.indices.put_template(
+            self.index, body={'version': version, 'index_patterns': ['ignored']}, ignore=400
+        )
+
+    async def start_index(self) -> bool:
+        if self.sync_client:
+            return False
+        hosts = [{'host': self._elastic_host, 'port': self._elastic_port}]
+        self.sync_client = AsyncElasticsearch(hosts, timeout=self.sync_timeout)
+        while True:
+            try:
+                await self.sync_client.cluster.health(wait_for_status='yellow')
+                self.log.info("ES is ready to connect to")
+                break
+            except ConnectionError:
+                self.log.warning("Failed to connect to Elasticsearch. Waiting for it!")
+                await asyncio.sleep(1)
+
+        index_version = await self.get_index_version()
+
+        res = await self.sync_client.indices.create(self.index, INDEX_DEFAULT_SETTINGS, ignore=400)
+        acked = res.get('acknowledged', False)
+
+        if acked:
+            await self.set_index_version(self.VERSION)
+            return True
+        elif index_version != self.VERSION:
+            self.log.error("es search index has an incompatible version: %s vs %s", index_version, self.VERSION)
+            raise IndexVersionMismatch(index_version, self.VERSION)
+        else:
+            await self.sync_client.indices.refresh(self.index)
+            return False
+
+    async def stop_index(self):
+        if self.sync_client:
+            await self.sync_client.close()
+        self.sync_client = None
+
+    async def delete_index(self):
+        if self.sync_client:
+            return await self.sync_client.indices.delete(self.index, ignore_unavailable=True)
+
+    def update_filter_query(self, censor_type, blockdict, channels=False):
+        blockdict = {blocked.hex(): blocker.hex() for blocked, blocker in blockdict.items()}
+        if channels:
+            update = expand_query(channel_id__in=list(blockdict.keys()), censor_type=f"<{censor_type}")
+        else:
+            update = expand_query(claim_id__in=list(blockdict.keys()), censor_type=f"<{censor_type}")
+        key = 'channel_id' if channels else 'claim_id'
+        update['script'] = {
+            "source": f"ctx._source.censor_type={censor_type}; "
+                      f"ctx._source.censoring_channel_id=params[ctx._source.{key}];",
+            "lang": "painless",
+            "params": blockdict
+        }
+        return update
+
+    async def apply_filters(self, blocked_streams, blocked_channels, filtered_streams, filtered_channels):
+        if filtered_streams:
+            await self.sync_client.update_by_query(
+                self.index, body=self.update_filter_query(Censor.SEARCH, filtered_streams), slices=4)
+            await self.sync_client.indices.refresh(self.index)
+        if filtered_channels:
+            await self.sync_client.update_by_query(
+                self.index, body=self.update_filter_query(Censor.SEARCH, filtered_channels), slices=4)
+            await self.sync_client.indices.refresh(self.index)
+            await self.sync_client.update_by_query(
+                self.index, body=self.update_filter_query(Censor.SEARCH, filtered_channels, True), slices=4)
+            await self.sync_client.indices.refresh(self.index)
+        if blocked_streams:
+            await self.sync_client.update_by_query(
+                self.index, body=self.update_filter_query(Censor.RESOLVE, blocked_streams), slices=4)
+            await self.sync_client.indices.refresh(self.index)
+        if blocked_channels:
+            await self.sync_client.update_by_query(
+                self.index, body=self.update_filter_query(Censor.RESOLVE, blocked_channels), slices=4)
+            await self.sync_client.indices.refresh(self.index)
+            await self.sync_client.update_by_query(
+                self.index, body=self.update_filter_query(Censor.RESOLVE, blocked_channels, True), slices=4)
+            await self.sync_client.indices.refresh(self.index)
+
+    @staticmethod
+    def _upsert_claim_query(index, claim):
+        return {
+            'doc': {key: value for key, value in claim.items() if key in ALL_FIELDS},
+            '_id': claim['claim_id'],
+            '_index': index,
+            '_op_type': 'update',
+            'doc_as_upsert': True
+        }
+
+    @staticmethod
+    def _delete_claim_query(index, claim_hash: bytes):
+        return {
+            '_index': index,
+            '_op_type': 'delete',
+            '_id': claim_hash.hex()
+        }
+
+    @staticmethod
+    def _update_trending_query(index, claim_hash, notifications):
+        return {
+            '_id': claim_hash.hex(),
+            '_index': index,
+            '_op_type': 'update',
+            'script': {
+                'lang': 'painless',
+                'source': FAST_AR_TRENDING_SCRIPT,
+                'params': {'src': {
+                    'changes': [
+                        {
+                            'height': notification.height,
+                            'prev_amount': notification.prev_amount / 1E8,
+                            'new_amount': notification.new_amount / 1E8,
+                        } for notification in notifications
+                    ]
+                }}
+            },
+        }
+
+    async def _claim_producer(self):
+        for deleted in self._deleted_claims:
+            yield self._delete_claim_query(self.index, deleted)
+        for touched in self._touched_claims:
+            claim = self.db.claim_producer(touched)
+            if claim:
+                yield self._upsert_claim_query(self.index, claim)
+        for claim_hash, notifications in self._trending.items():
+            yield self._update_trending_query(self.index, claim_hash, notifications)
+
+    def advance(self, height: int):
+        super().advance(height)
+
+        touched_or_deleted = self.db.prefix_db.touched_or_deleted.get(height)
+        for k, v in self.db.prefix_db.trending_notification.iterate((height,)):
+            self._trending[k.claim_hash].append(TrendingNotification(k.height, v.previous_amount, v.new_amount))
+        if touched_or_deleted:
+            readded_after_reorg = self._removed_during_undo.intersection(touched_or_deleted.touched_claims)
+            self._deleted_claims.difference_update(readded_after_reorg)
+            self._touched_claims.update(touched_or_deleted.touched_claims)
+            self._deleted_claims.update(touched_or_deleted.deleted_claims)
+            self._touched_claims.difference_update(self._deleted_claims)
+            for to_del in touched_or_deleted.deleted_claims:
+                if to_del in self._trending:
+                    self._trending.pop(to_del)
+        self._advanced = True
+
+    def unwind(self):
+        self.db.tx_counts.pop()
+        reverted_block_hash = self.db.coin.header_hash(self.db.headers.pop())
+        packed = self.db.prefix_db.undo.get(len(self.db.tx_counts), reverted_block_hash)
+        touched_or_deleted = None
+        claims_to_delete = []
+        # find and apply the touched_or_deleted items in the undos for the reverted blocks
+        assert packed, f'missing undo information for block {len(self.db.tx_counts)}'
+        while packed:
+            op, packed = RevertableOp.unpack(packed)
+            if op.is_delete and op.key.startswith(DB_PREFIXES.touched_or_deleted.value):
+                assert touched_or_deleted is None, 'only should have one match'
+                touched_or_deleted = self.db.prefix_db.touched_or_deleted.unpack_value(op.value)
+            elif op.is_delete and op.key.startswith(DB_PREFIXES.claim_to_txo.value):
+                v = self.db.prefix_db.claim_to_txo.unpack_value(op.value)
+                if v.root_tx_num == v.tx_num and v.root_tx_num > self.db.tx_counts[-1]:
+                    claims_to_delete.append(self.db.prefix_db.claim_to_txo.unpack_key(op.key).claim_hash)
+        if touched_or_deleted:
+            self._touched_claims.update(set(touched_or_deleted.deleted_claims).union(
+                touched_or_deleted.touched_claims.difference(set(claims_to_delete))))
+            self._deleted_claims.update(claims_to_delete)
+            self._removed_during_undo.update(claims_to_delete)
+        self._advanced = True
+        self.log.warning("delete %i claim and upsert %i from reorg", len(self._deleted_claims), len(self._touched_claims))
+
+    async def poll_for_changes(self):
+        await super().poll_for_changes()
+        cnt = 0
+        success = 0
+        if self._advanced:
+            if self._touched_claims or self._deleted_claims or self._trending:
+                async for ok, item in async_streaming_bulk(
+                        self.sync_client, self._claim_producer(),
+                        raise_on_error=False):
+                    cnt += 1
+                    if not ok:
+                        self.log.warning("indexing failed for an item: %s", item)
+                    else:
+                        success += 1
+                await self.sync_client.indices.refresh(self.index)
+                await self.db.reload_blocking_filtering_streams()
+                await self.apply_filters(
+                    self.db.blocked_streams, self.db.blocked_channels, self.db.filtered_streams,
+                    self.db.filtered_channels
+                )
+            self.write_es_height(self.db.db_height, self.db.db_tip[::-1].hex())
+            self.log.info("Indexing block %i done. %i/%i successful", self._last_wrote_height, success, cnt)
+            self._touched_claims.clear()
+            self._deleted_claims.clear()
+            self._removed_during_undo.clear()
+            self._trending.clear()
+            self._advanced = False
+            self.synchronized.set()
+            self.notify_es_notification_listeners(self._last_wrote_height, self.db.db_tip)
+
+    @property
+    def last_synced_height(self) -> int:
+        return self._last_wrote_height
+
+    async def start(self, reindex=False):
+        await super().start()
+
+        def _start_cancellable(run, *args):
+            _flag = asyncio.Event()
+            self.cancellable_tasks.append(asyncio.ensure_future(run(*args, _flag)))
+            return _flag.wait()
+
+        self.db.open_db()
+        await self.db.initialize_caches()
+        await self.read_es_height()
+        await self.start_index()
+        self.last_state = self.db.read_db_state()
+
+        await _start_cancellable(self.run_es_notifier)
+
+        if reindex or self._last_wrote_height == 0 and self.db.db_height > 0:
+            if self._last_wrote_height == 0:
+                self.log.info("running initial ES indexing of rocksdb at block height %i", self.db.db_height)
+            else:
+                self.log.info("reindex (last wrote: %i, db height: %i)", self._last_wrote_height, self.db.db_height)
+            await self.reindex()
+        await _start_cancellable(self.refresh_blocks_forever)
+
+    async def stop(self, delete_index=False):
+        async with self._lock:
+            while self.cancellable_tasks:
+                t = self.cancellable_tasks.pop()
+                if not t.done():
+                    t.cancel()
+        if delete_index:
+            await self.delete_index()
+        await self.stop_index()
+        self._executor.shutdown(wait=True)
+        self._executor = None
+        self.shutdown_event.set()
+
+    def run(self, reindex=False):
+        loop = asyncio.get_event_loop()
+        loop.set_default_executor(self._executor)
+
+        def __exit():
+            raise SystemExit()
+        try:
+            loop.add_signal_handler(signal.SIGINT, __exit)
+            loop.add_signal_handler(signal.SIGTERM, __exit)
+            loop.run_until_complete(self.start(reindex=reindex))
+            loop.run_until_complete(self.shutdown_event.wait())
+        except (SystemExit, KeyboardInterrupt):
+            pass
+        finally:
+            loop.run_until_complete(self.stop())
+
+    async def reindex(self):
+        async with self._lock:
+            self.log.info("reindexing %i claims (estimate)", self.db.prefix_db.claim_to_txo.estimate_num_keys())
+            await self.delete_index()
+            res = await self.sync_client.indices.create(self.index, INDEX_DEFAULT_SETTINGS, ignore=400)
+            acked = res.get('acknowledged', False)
+            if acked:
+                await self.set_index_version(self.VERSION)
+            await self.sync_client.indices.refresh(self.index)
+            self.write_es_height(0, self.env.coin.GENESIS_HASH)
+            await self._sync_all_claims()
+            await self.sync_client.indices.refresh(self.index)
+            self.write_es_height(self.db.db_height, self.db.db_tip[::-1].hex())
+            self.notify_es_notification_listeners(self.db.db_height, self.db.db_tip)
+            self.log.info("finished reindexing")
+
+    async def _sync_all_claims(self, batch_size=100000):
+        def load_historic_trending():
+            notifications = self._trending
+            for k, v in self.db.prefix_db.trending_notification.iterate():
+                notifications[k.claim_hash].append(TrendingNotification(k.height, v.previous_amount, v.new_amount))
+
+        async def all_claims_producer():
+            async for claim in self.db.all_claims_producer(batch_size=batch_size):
+                yield self._upsert_claim_query(self.index, claim)
+                claim_hash = bytes.fromhex(claim['claim_id'])
+                if claim_hash in self._trending:
+                    yield self._update_trending_query(self.index, claim_hash, self._trending.pop(claim_hash))
+            self._trending.clear()
+
+        self.log.info("loading about %i historic trending updates", self.db.prefix_db.trending_notification.estimate_num_keys())
+        await asyncio.get_event_loop().run_in_executor(self._executor, load_historic_trending)
+        self.log.info("loaded historic trending updates for %i claims", len(self._trending))
+
+        cnt = 0
+        success = 0
+        producer = all_claims_producer()
+
+        finished = False
+        try:
+            async for ok, item in async_streaming_bulk(self.sync_client, producer, raise_on_error=False):
+                cnt += 1
+                if not ok:
+                    self.log.warning("indexing failed for an item: %s", item)
+                else:
+                    success += 1
+                if cnt % batch_size == 0:
+                    self.log.info(f"indexed {success} claims")
+            finished = True
+            await self.sync_client.indices.refresh(self.index)
+            self.log.info("indexed %i/%i claims", success, cnt)
+        finally:
+            if not finished:
+                await producer.aclose()
+            self.shutdown_event.set()

scribe/readers/hub_server.py

@@ -0,0 +1,162 @@
+import signal
+import asyncio
+import typing
+from scribe import __version__
+from scribe.blockchain.daemon import LBCDaemon
+from scribe.readers import BaseBlockchainReader
+from scribe.elasticsearch import ElasticNotifierClientProtocol
+from scribe.hub.session import SessionManager
+from scribe.hub.mempool import MemPool
+from scribe.hub.udp import StatusServer
+from scribe.hub.prometheus import PrometheusServer
+
+
+class BlockchainReaderServer(BaseBlockchainReader):
+    def __init__(self, env):
+        super().__init__(env, 'lbry-reader', thread_workers=max(1, env.max_query_workers), thread_prefix='hub-worker')
+        self.history_cache = {}
+        self.resolve_outputs_cache = {}
+        self.resolve_cache = {}
+        self.notifications_to_send = []
+        self.mempool_notifications = set()
+        self.status_server = StatusServer()
+        self.daemon = LBCDaemon(env.coin, env.daemon_url)  # only needed for broadcasting txs
+        self.prometheus_server: typing.Optional[PrometheusServer] = None
+        self.mempool = MemPool(self.env.coin, self.db)
+        self.session_manager = SessionManager(
+            env, self.db, self.mempool, self.history_cache, self.resolve_cache,
+            self.resolve_outputs_cache, self.daemon,
+            self.shutdown_event,
+            on_available_callback=self.status_server.set_available,
+            on_unavailable_callback=self.status_server.set_unavailable
+        )
+        self.mempool.session_manager = self.session_manager
+        self.es_notifications = asyncio.Queue()
+        self.es_notification_client = ElasticNotifierClientProtocol(self.es_notifications)
+        self.synchronized = asyncio.Event()
+        self._es_height = None
+        self._es_block_hash = None
+
+    def clear_caches(self):
+        self.history_cache.clear()
+        self.resolve_outputs_cache.clear()
+        self.resolve_cache.clear()
+        # self.clear_search_cache()
+        # self.mempool.notified_mempool_txs.clear()
+
+    def clear_search_cache(self):
+        self.session_manager.search_index.clear_caches()
+
+    def advance(self, height: int):
+        super().advance(height)
+        touched_hashXs = self.db.prefix_db.touched_hashX.get(height).touched_hashXs
+        self.notifications_to_send.append((set(touched_hashXs), height))
+
+    def _detect_changes(self):
+        super()._detect_changes()
+        self.mempool_notifications.update(self.mempool.refresh())
+
+    async def poll_for_changes(self):
+        await super().poll_for_changes()
+        if self.db.fs_height <= 0:
+            return
+        self.status_server.set_height(self.db.fs_height, self.db.db_tip)
+        if self.notifications_to_send:
+            for (touched, height) in self.notifications_to_send:
+                await self.mempool.on_block(touched, height)
+                self.log.info("reader advanced to %i", height)
+                if self._es_height == self.db.db_height:
+                    self.synchronized.set()
+        if self.mempool_notifications:
+            await self.mempool.on_mempool(
+                set(self.mempool.touched_hashXs), self.mempool_notifications, self.db.db_height
+            )
+        self.mempool_notifications.clear()
+        self.notifications_to_send.clear()
+
+    async def receive_es_notifications(self, synchronized: asyncio.Event):
+        await asyncio.get_event_loop().create_connection(
+            lambda: self.es_notification_client, '127.0.0.1', self.env.elastic_notifier_port
+        )
+        synchronized.set()
+        try:
+            while True:
+                self._es_height, self._es_block_hash = await self.es_notifications.get()
+                self.clear_search_cache()
+                if self.last_state and self._es_block_hash == self.last_state.tip:
+                    self.synchronized.set()
+                    self.log.info("es and reader are in sync at block %i", self.last_state.height)
+                else:
+                    self.log.info("es and reader are not yet in sync (block %s vs %s)", self._es_height,
+                                  self.db.db_height)
+        finally:
+            self.es_notification_client.close()
+
+    async def start(self):
+        await super().start()
+        env = self.env
+        # min_str, max_str = env.coin.SESSIONCLS.protocol_min_max_strings()
+        self.log.info(f'software version: {__version__}')
+        # self.log.info(f'supported protocol versions: {min_str}-{max_str}')
+        self.log.info(f'event loop policy: {env.loop_policy}')
+        self.log.info(f'reorg limit is {env.reorg_limit:,d} blocks')
+        await self.daemon.height()
+
+        def _start_cancellable(run, *args):
+            _flag = asyncio.Event()
+            self.cancellable_tasks.append(asyncio.ensure_future(run(*args, _flag)))
+            return _flag.wait()
+
+        self.db.open_db()
+        await self.db.initialize_caches()
+
+        self.last_state = self.db.read_db_state()
+
+        await self.start_prometheus()
+        if self.env.udp_port and int(self.env.udp_port):
+            await self.status_server.start(
+                0, bytes.fromhex(self.env.coin.GENESIS_HASH)[::-1], self.env.country,
+                self.env.host, self.env.udp_port, self.env.allow_lan_udp
+            )
+        await _start_cancellable(self.receive_es_notifications)
+        await _start_cancellable(self.refresh_blocks_forever)
+        await self.session_manager.search_index.start()
+        await _start_cancellable(self.session_manager.serve, self.mempool)
+
+    async def stop(self):
+        await self.status_server.stop()
+        async with self._lock:
+            while self.cancellable_tasks:
+                t = self.cancellable_tasks.pop()
+                if not t.done():
+                    t.cancel()
+        await self.session_manager.search_index.stop()
+        self.db.close()
+        if self.prometheus_server:
+            await self.prometheus_server.stop()
+            self.prometheus_server = None
+        await self.daemon.close()
+        self._executor.shutdown(wait=True)
+        self._executor = None
+        self.shutdown_event.set()
+
+    def run(self):
+        loop = asyncio.get_event_loop()
+        loop.set_default_executor(self._executor)
+
+        def __exit():
+            raise SystemExit()
+        try:
+            loop.add_signal_handler(signal.SIGINT, __exit)
+            loop.add_signal_handler(signal.SIGTERM, __exit)
+            loop.run_until_complete(self.start())
+            loop.run_until_complete(self.shutdown_event.wait())
+        except (SystemExit, KeyboardInterrupt):
+            pass
+        finally:
+            loop.run_until_complete(self.stop())
+
+    async def start_prometheus(self):
+        if not self.prometheus_server and self.env.prometheus_port:
+            self.prometheus_server = PrometheusServer()
+            await self.prometheus_server.start("0.0.0.0", self.env.prometheus_port)

scribe/readers/interface.py

@@ -0,0 +1,119 @@
+import logging
+import asyncio
+import typing
+from concurrent.futures.thread import ThreadPoolExecutor
+from prometheus_client import Gauge, Histogram
+from scribe import PROMETHEUS_NAMESPACE
+from scribe.db.prefixes import DBState
+from scribe.db import HubDB
+
+HISTOGRAM_BUCKETS = (
+    .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
+)
+
+NAMESPACE = f"{PROMETHEUS_NAMESPACE}_reader"
+
+
+class BaseBlockchainReader:
+    block_count_metric = Gauge(
+        "block_count", "Number of processed blocks", namespace=NAMESPACE
+    )
+    block_update_time_metric = Histogram(
+        "block_time", "Block update times", namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS
+    )
+    reorg_count_metric = Gauge(
+        "reorg_count", "Number of reorgs", namespace=NAMESPACE
+    )
+
+    def __init__(self, env, secondary_name: str, thread_workers: int = 1, thread_prefix: str = 'blockchain-reader'):
+        self.env = env
+        self.log = logging.getLogger(__name__).getChild(self.__class__.__name__)
+        self.shutdown_event = asyncio.Event()
+        self.cancellable_tasks = []
+        self._thread_workers = thread_workers
+        self._thread_prefix = thread_prefix
+        self._executor = ThreadPoolExecutor(thread_workers, thread_name_prefix=thread_prefix)
+        self.db = HubDB(
+            env.coin, env.db_dir, env.cache_MB, env.reorg_limit, env.cache_all_claim_txos, env.cache_all_tx_hashes,
+            secondary_name=secondary_name, max_open_files=-1, blocking_channel_ids=env.blocking_channel_ids,
+            filtering_channel_ids=env.filtering_channel_ids, executor=self._executor
+        )
+        self.last_state: typing.Optional[DBState] = None
+        self._refresh_interval = 0.1
+        self._lock = asyncio.Lock()
+
+    def _detect_changes(self):
+        try:
+            self.db.prefix_db.try_catch_up_with_primary()
+        except:
+            self.log.exception('failed to update secondary db')
+            raise
+        state = self.db.prefix_db.db_state.get()
+        if not state or state.height <= 0:
+            return
+        if self.last_state and self.last_state.height > state.height:
+            self.log.warning("reorg detected, waiting until the writer has flushed the new blocks to advance")
+            return
+        last_height = 0 if not self.last_state else self.last_state.height
+        rewound = False
+        if self.last_state:
+            while True:
+                if self.db.headers[-1] == self.db.prefix_db.header.get(last_height, deserialize_value=False):
+                    self.log.debug("connects to block %i", last_height)
+                    break
+                else:
+                    self.log.warning("disconnect block %i", last_height)
+                    self.unwind()
+                    rewound = True
+                    last_height -= 1
+        if rewound:
+            self.reorg_count_metric.inc()
+        self.db.read_db_state()
+        if not self.last_state or last_height < state.height:
+            for height in range(last_height + 1, state.height + 1):
+                self.log.info("advancing to %i", height)
+                self.advance(height)
+            self.clear_caches()
+            self.last_state = state
+            self.block_count_metric.set(self.last_state.height)
+            self.db.blocked_streams, self.db.blocked_channels = self.db.get_streams_and_channels_reposted_by_channel_hashes(
+                self.db.blocking_channel_hashes
+            )
+            self.db.filtered_streams, self.db.filtered_channels = self.db.get_streams_and_channels_reposted_by_channel_hashes(
+                self.db.filtering_channel_hashes
+            )
+
+    async def poll_for_changes(self):
+        await asyncio.get_event_loop().run_in_executor(self._executor, self._detect_changes)
+
+    async def refresh_blocks_forever(self, synchronized: asyncio.Event):
+        while True:
+            try:
+                async with self._lock:
+                    await self.poll_for_changes()
+            except asyncio.CancelledError:
+                raise
+            except:
+                self.log.exception("blockchain reader main loop encountered an unexpected error")
+                raise
+            await asyncio.sleep(self._refresh_interval)
+            synchronized.set()
+
+    def clear_caches(self):
+        pass
+
+    def advance(self, height: int):
+        tx_count = self.db.prefix_db.tx_count.get(height).tx_count
+        assert tx_count not in self.db.tx_counts, f'boom {tx_count} in {len(self.db.tx_counts)} tx counts'
+        assert len(self.db.tx_counts) == height, f"{len(self.db.tx_counts)} != {height}"
+        self.db.tx_counts.append(tx_count)
+        self.db.headers.append(self.db.prefix_db.header.get(height, deserialize_value=False))
+
+    def unwind(self):
+        self.db.tx_counts.pop()
+        self.db.headers.pop()
+
+    async def start(self):
+        if not self._executor:
+            self._executor = ThreadPoolExecutor(self._thread_workers, thread_name_prefix=self._thread_prefix)
+            self.db._executor = self._executor

scribe/schema/Makefile

@@ -0,0 +1,5 @@
+build:
+	rm types/v2/* -rf
+	touch types/v2/__init__.py
+	cd types/v2/ && protoc --python_out=. -I ../../../../../types/v2/proto/ ../../../../../types/v2/proto/*.proto
+	sed -e 's/^import\ \(.*\)_pb2\ /from . import\ \1_pb2\ /g' -i types/v2/*.py

scribe/schema/README.md

@@ -0,0 +1,24 @@
+Schema
+=====
+
+Those files are generated from the [types repo](https://github.com/lbryio/types). If you are modifying/adding a new type, make sure it is cloned in the same root folder as the SDK repo, like:
+
+```
+repos/
+    - lbry-sdk/
+    - types/
+```
+
+Then, [download protoc 3.2.0](https://github.com/protocolbuffers/protobuf/releases/tag/v3.2.0), add it to your PATH. On linux it is:
+
+```bash
+cd ~/.local/bin
+wget https://github.com/protocolbuffers/protobuf/releases/download/v3.2.0/protoc-3.2.0-linux-x86_64.zip
+unzip protoc-3.2.0-linux-x86_64.zip bin/protoc -d..
+```
+
+Finally, `make` should update everything in place.
+
+
+### Why protoc 3.2.0?
+Different/newer versions will generate larger diffs and we need to make sure they are good. In theory, we can just update to latest and it will all work, but it is a good practice to check blockchain data and retro compatibility before bumping versions (if you do, please update this section!).

scribe/schema/__init__.py

@@ -0,0 +1 @@
+from .claim import Claim

scribe/schema/attrs.py

@@ -0,0 +1,573 @@
+import json
+import logging
+import os.path
+import hashlib
+from typing import Tuple, List
+from string import ascii_letters
+from decimal import Decimal, ROUND_UP
+from google.protobuf.json_format import MessageToDict
+
+from scribe.base58 import Base58, b58_encode
+from scribe.error import MissingPublishedFileError, EmptyPublishedFileError
+
+from scribe.schema.mime_types import guess_media_type
+from scribe.schema.base import Metadata, BaseMessageList
+from scribe.schema.tags import clean_tags, normalize_tag
+from scribe.schema.types.v2.claim_pb2 import (
+    Fee as FeeMessage,
+    Location as LocationMessage,
+    Language as LanguageMessage
+)
+
+
+log = logging.getLogger(__name__)
+
+
+CENT = 1000000
+COIN = 100*CENT
+
+
+def calculate_sha384_file_hash(file_path):
+    sha384 = hashlib.sha384()
+    with open(file_path, 'rb') as f:
+        for chunk in iter(lambda: f.read(128 * sha384.block_size), b''):
+            sha384.update(chunk)
+    return sha384.digest()
+
+
+def country_int_to_str(country: int) -> str:
+    r = LocationMessage.Country.Name(country)
+    return r[1:] if r.startswith('R') else r
+
+
+def country_str_to_int(country: str) -> int:
+    if len(country) == 3:
+        country = 'R' + country
+    return LocationMessage.Country.Value(country)
+
+
+class Dimmensional(Metadata):
+
+    __slots__ = ()
+
+    @property
+    def width(self) -> int:
+        return self.message.width
+
+    @width.setter
+    def width(self, width: int):
+        self.message.width = width
+
+    @property
+    def height(self) -> int:
+        return self.message.height
+
+    @height.setter
+    def height(self, height: int):
+        self.message.height = height
+
+    @property
+    def dimensions(self) -> Tuple[int, int]:
+        return self.width, self.height
+
+    @dimensions.setter
+    def dimensions(self, dimensions: Tuple[int, int]):
+        self.message.width, self.message.height = dimensions
+
+    def _extract(self, file_metadata, field):
+        try:
+            setattr(self, field, file_metadata.getValues(field)[0])
+        except:
+            log.exception(f'Could not extract {field} from file metadata.')
+
+    def update(self, file_metadata=None, height=None, width=None):
+        if height is not None:
+            self.height = height
+        elif file_metadata:
+            self._extract(file_metadata, 'height')
+
+        if width is not None:
+            self.width = width
+        elif file_metadata:
+            self._extract(file_metadata, 'width')
+
+
+class Playable(Metadata):
+
+    __slots__ = ()
+
+    @property
+    def duration(self) -> int:
+        return self.message.duration
+
+    @duration.setter
+    def duration(self, duration: int):
+        self.message.duration = duration
+
+    def update(self, file_metadata=None, duration=None):
+        if duration is not None:
+            self.duration = duration
+        elif file_metadata:
+            try:
+                self.duration = file_metadata.getValues('duration')[0].seconds
+            except:
+                log.exception('Could not extract duration from file metadata.')
+
+
+class Image(Dimmensional):
+
+    __slots__ = ()
+
+
+class Audio(Playable):
+
+    __slots__ = ()
+
+
+class Video(Dimmensional, Playable):
+
+    __slots__ = ()
+
+    def update(self, file_metadata=None, height=None, width=None, duration=None):
+        Dimmensional.update(self, file_metadata, height, width)
+        Playable.update(self, file_metadata, duration)
+
+
+class Source(Metadata):
+
+    __slots__ = ()
+
+    def update(self, file_path=None):
+        if file_path is not None:
+            self.name = os.path.basename(file_path)
+            self.media_type, stream_type = guess_media_type(file_path)
+            if not os.path.isfile(file_path):
+                raise MissingPublishedFileError(file_path)
+            self.size = os.path.getsize(file_path)
+            if self.size == 0:
+                raise EmptyPublishedFileError(file_path)
+            self.file_hash_bytes = calculate_sha384_file_hash(file_path)
+            return stream_type
+
+    @property
+    def name(self) -> str:
+        return self.message.name
+
+    @name.setter
+    def name(self, name: str):
+        self.message.name = name
+
+    @property
+    def size(self) -> int:
+        return self.message.size
+
+    @size.setter
+    def size(self, size: int):
+        self.message.size = size
+
+    @property
+    def media_type(self) -> str:
+        return self.message.media_type
+
+    @media_type.setter
+    def media_type(self, media_type: str):
+        self.message.media_type = media_type
+
+    @property
+    def file_hash(self) -> str:
+        return self.message.hash.hex()
+
+    @file_hash.setter
+    def file_hash(self, file_hash: str):
+        self.message.hash = bytes.fromhex(file_hash)
+
+    @property
+    def file_hash_bytes(self) -> bytes:
+        return self.message.hash
+
+    @file_hash_bytes.setter
+    def file_hash_bytes(self, file_hash_bytes: bytes):
+        self.message.hash = file_hash_bytes
+
+    @property
+    def sd_hash(self) -> str:
+        return self.message.sd_hash.hex()
+
+    @sd_hash.setter
+    def sd_hash(self, sd_hash: str):
+        self.message.sd_hash = bytes.fromhex(sd_hash)
+
+    @property
+    def sd_hash_bytes(self) -> bytes:
+        return self.message.sd_hash
+
+    @sd_hash_bytes.setter
+    def sd_hash_bytes(self, sd_hash: bytes):
+        self.message.sd_hash = sd_hash
+
+    @property
+    def bt_infohash(self) -> str:
+        return self.message.bt_infohash.hex()
+
+    @bt_infohash.setter
+    def bt_infohash(self, bt_infohash: str):
+        self.message.bt_infohash = bytes.fromhex(bt_infohash)
+
+    @property
+    def bt_infohash_bytes(self) -> bytes:
+        return self.message.bt_infohash.decode()
+
+    @bt_infohash_bytes.setter
+    def bt_infohash_bytes(self, bt_infohash: bytes):
+        self.message.bt_infohash = bt_infohash
+
+    @property
+    def url(self) -> str:
+        return self.message.url
+
+    @url.setter
+    def url(self, url: str):
+        self.message.url = url
+
+
+class Fee(Metadata):
+
+    __slots__ = ()
+
+    def update(self, address: str = None, currency: str = None, amount=None):
+        if amount:
+            currency = (currency or self.currency or '').lower()
+            if not currency:
+                raise Exception('In order to set a fee amount, please specify a fee currency.')
+            if currency not in ('lbc', 'btc', 'usd'):
+                raise Exception(f'Missing or unknown currency provided: {currency}')
+            setattr(self, currency, Decimal(amount))
+        elif currency:
+            raise Exception('In order to set a fee currency, please specify a fee amount.')
+        if address:
+            if not self.currency:
+                raise Exception('In order to set a fee address, please specify a fee amount and currency.')
+            self.address = address
+
+    @property
+    def currency(self) -> str:
+        if self.message.currency:
+            return FeeMessage.Currency.Name(self.message.currency)
+
+    @property
+    def address(self) -> str:
+        if self.address_bytes:
+            return b58_encode(self.address_bytes)
+
+    @address.setter
+    def address(self, address: str):
+        self.address_bytes = Base58.decode(address)
+
+    @property
+    def address_bytes(self) -> bytes:
+        return self.message.address
+
+    @address_bytes.setter
+    def address_bytes(self, address: bytes):
+        self.message.address = address
+
+    @property
+    def amount(self) -> Decimal:
+        if self.currency == 'LBC':
+            return self.lbc
+        if self.currency == 'BTC':
+            return self.btc
+        if self.currency == 'USD':
+            return self.usd
+
+    DEWIES = Decimal(COIN)
+
+    @property
+    def lbc(self) -> Decimal:
+        if self.message.currency != FeeMessage.LBC:
+            raise ValueError('LBC can only be returned for LBC fees.')
+        return Decimal(self.message.amount / self.DEWIES)
+
+    @lbc.setter
+    def lbc(self, amount: Decimal):
+        self.dewies = int(amount * self.DEWIES)
+
+    @property
+    def dewies(self) -> int:
+        if self.message.currency != FeeMessage.LBC:
+            raise ValueError('Dewies can only be returned for LBC fees.')
+        return self.message.amount
+
+    @dewies.setter
+    def dewies(self, amount: int):
+        self.message.amount = amount
+        self.message.currency = FeeMessage.LBC
+
+    SATOSHIES = Decimal(COIN)
+
+    @property
+    def btc(self) -> Decimal:
+        if self.message.currency != FeeMessage.BTC:
+            raise ValueError('BTC can only be returned for BTC fees.')
+        return Decimal(self.message.amount / self.SATOSHIES)
+
+    @btc.setter
+    def btc(self, amount: Decimal):
+        self.satoshis = int(amount * self.SATOSHIES)
+
+    @property
+    def satoshis(self) -> int:
+        if self.message.currency != FeeMessage.BTC:
+            raise ValueError('Satoshies can only be returned for BTC fees.')
+        return self.message.amount
+
+    @satoshis.setter
+    def satoshis(self, amount: int):
+        self.message.amount = amount
+        self.message.currency = FeeMessage.BTC
+
+    PENNIES = Decimal('100.0')
+    PENNY = Decimal('0.01')
+
+    @property
+    def usd(self) -> Decimal:
+        if self.message.currency != FeeMessage.USD:
+            raise ValueError('USD can only be returned for USD fees.')
+        return Decimal(self.message.amount / self.PENNIES)
+
+    @usd.setter
+    def usd(self, amount: Decimal):
+        self.pennies = int(amount.quantize(self.PENNY, ROUND_UP) * self.PENNIES)
+
+    @property
+    def pennies(self) -> int:
+        if self.message.currency != FeeMessage.USD:
+            raise ValueError('Pennies can only be returned for USD fees.')
+        return self.message.amount
+
+    @pennies.setter
+    def pennies(self, amount: int):
+        self.message.amount = amount
+        self.message.currency = FeeMessage.USD
+
+
+class ClaimReference(Metadata):
+
+    __slots__ = ()
+
+    @property
+    def claim_id(self) -> str:
+        return self.claim_hash[::-1].hex()
+
+    @claim_id.setter
+    def claim_id(self, claim_id: str):
+        self.claim_hash = bytes.fromhex(claim_id)[::-1]
+
+    @property
+    def claim_hash(self) -> bytes:
+        return self.message.claim_hash
+
+    @claim_hash.setter
+    def claim_hash(self, claim_hash: bytes):
+        self.message.claim_hash = claim_hash
+
+
+class ClaimList(BaseMessageList[ClaimReference]):
+
+    __slots__ = ()
+    item_class = ClaimReference
+
+    @property
+    def _message(self):
+        return self.message.claim_references
+
+    def append(self, value):
+        self.add().claim_id = value
+
+    @property
+    def ids(self) -> List[str]:
+        return [c.claim_id for c in self]
+
+
+class Language(Metadata):
+
+    __slots__ = ()
+
+    @property
+    def langtag(self) -> str:
+        langtag = []
+        if self.language:
+            langtag.append(self.language)
+        if self.script:
+            langtag.append(self.script)
+        if self.region:
+            langtag.append(self.region)
+        return '-'.join(langtag)
+
+    @langtag.setter
+    def langtag(self, langtag: str):
+        parts = langtag.split('-')
+        self.language = parts.pop(0)
+        if parts and len(parts[0]) == 4:
+            self.script = parts.pop(0)
+        if parts and len(parts[0]) == 2 and parts[0].isalpha():
+            self.region = parts.pop(0)
+        if parts and len(parts[0]) == 3 and parts[0].isdigit():
+            self.region = parts.pop(0)
+        assert not parts, f"Failed to parse language tag: {langtag}"
+
+    @property
+    def language(self) -> str:
+        if self.message.language:
+            return LanguageMessage.Language.Name(self.message.language)
+
+    @language.setter
+    def language(self, language: str):
+        self.message.language = LanguageMessage.Language.Value(language)
+
+    @property
+    def script(self) -> str:
+        if self.message.script:
+            return LanguageMessage.Script.Name(self.message.script)
+
+    @script.setter
+    def script(self, script: str):
+        self.message.script = LanguageMessage.Script.Value(script)
+
+    @property
+    def region(self) -> str:
+        if self.message.region:
+            return country_int_to_str(self.message.region)
+
+    @region.setter
+    def region(self, region: str):
+        self.message.region = country_str_to_int(region)
+
+
+class LanguageList(BaseMessageList[Language]):
+    __slots__ = ()
+    item_class = Language
+
+    def append(self, value: str):
+        self.add().langtag = value
+
+
+class Location(Metadata):
+
+    __slots__ = ()
+
+    def from_value(self, value):
+        if isinstance(value, str) and value.startswith('{'):
+            value = json.loads(value)
+
+        if isinstance(value, dict):
+            for key, val in value.items():
+                setattr(self, key, val)
+
+        elif isinstance(value, str):
+            parts = value.split(':')
+            if len(parts) > 2 or (parts[0] and parts[0][0] in ascii_letters):
+                country = parts and parts.pop(0)
+                if country:
+                    self.country = country
+                state = parts and parts.pop(0)
+                if state:
+                    self.state = state
+                city = parts and parts.pop(0)
+                if city:
+                    self.city = city
+                code = parts and parts.pop(0)
+                if code:
+                    self.code = code
+            latitude = parts and parts.pop(0)
+            if latitude:
+                self.latitude = latitude
+            longitude = parts and parts.pop(0)
+            if longitude:
+                self.longitude = longitude
+
+        else:
+            raise ValueError(f'Could not parse country value: {value}')
+
+    def to_dict(self):
+        d = MessageToDict(self.message)
+        if self.message.longitude:
+            d['longitude'] = self.longitude
+        if self.message.latitude:
+            d['latitude'] = self.latitude
+        return d
+
+    @property
+    def country(self) -> str:
+        if self.message.country:
+            return LocationMessage.Country.Name(self.message.country)
+
+    @country.setter
+    def country(self, country: str):
+        self.message.country = LocationMessage.Country.Value(country)
+
+    @property
+    def state(self) -> str:
+        return self.message.state
+
+    @state.setter
+    def state(self, state: str):
+        self.message.state = state
+
+    @property
+    def city(self) -> str:
+        return self.message.city
+
+    @city.setter
+    def city(self, city: str):
+        self.message.city = city
+
+    @property
+    def code(self) -> str:
+        return self.message.code
+
+    @code.setter
+    def code(self, code: str):
+        self.message.code = code
+
+    GPS_PRECISION = Decimal('10000000')
+
+    @property
+    def latitude(self) -> str:
+        if self.message.latitude:
+            return str(Decimal(self.message.latitude) / self.GPS_PRECISION)
+
+    @latitude.setter
+    def latitude(self, latitude: str):
+        latitude = Decimal(latitude)
+        assert -90 <= latitude <= 90, "Latitude must be between -90 and 90 degrees."
+        self.message.latitude = int(latitude * self.GPS_PRECISION)
+
+    @property
+    def longitude(self) -> str:
+        if self.message.longitude:
+            return str(Decimal(self.message.longitude) / self.GPS_PRECISION)
+
+    @longitude.setter
+    def longitude(self, longitude: str):
+        longitude = Decimal(longitude)
+        assert -180 <= longitude <= 180, "Longitude must be between -180 and 180 degrees."
+        self.message.longitude = int(longitude * self.GPS_PRECISION)
+
+
+class LocationList(BaseMessageList[Location]):
+    __slots__ = ()
+    item_class = Location
+
+    def append(self, value):
+        self.add().from_value(value)
+
+
+class TagList(BaseMessageList[str]):
+    __slots__ = ()
+    item_class = str
+
+    def append(self, tag: str):
+        tag = normalize_tag(tag)
+        if tag and tag not in self.message:
+            self.message.append(tag)

scribe/schema/base.py

@@ -0,0 +1,124 @@
+from binascii import hexlify, unhexlify
+from typing import List, Iterator, TypeVar, Generic
+
+from google.protobuf.message import DecodeError
+from google.protobuf.json_format import MessageToDict
+
+
+class Signable:
+
+    __slots__ = (
+        'message', 'version', 'signature',
+        'signature_type', 'unsigned_payload', 'signing_channel_hash'
+    )
+
+    message_class = None
+
+    def __init__(self, message=None):
+        self.message = message or self.message_class()
+        self.version = 2
+        self.signature = None
+        self.signature_type = 'SECP256k1'
+        self.unsigned_payload = None
+        self.signing_channel_hash = None
+
+    def clear_signature(self):
+        self.signature = None
+        self.unsigned_payload = None
+        self.signing_channel_hash = None
+
+    @property
+    def signing_channel_id(self):
+        return hexlify(self.signing_channel_hash[::-1]).decode() if self.signing_channel_hash else None
+
+    @signing_channel_id.setter
+    def signing_channel_id(self, channel_id: str):
+        self.signing_channel_hash = unhexlify(channel_id)[::-1]
+
+    @property
+    def is_signed(self):
+        return self.signature is not None
+
+    def to_dict(self):
+        return MessageToDict(self.message)
+
+    def to_message_bytes(self) -> bytes:
+        return self.message.SerializeToString()
+
+    def to_bytes(self) -> bytes:
+        pieces = bytearray()
+        if self.is_signed:
+            pieces.append(1)
+            pieces.extend(self.signing_channel_hash)
+            pieces.extend(self.signature)
+        else:
+            pieces.append(0)
+        pieces.extend(self.to_message_bytes())
+        return bytes(pieces)
+
+    @classmethod
+    def from_bytes(cls, data: bytes):
+        signable = cls()
+        if data[0] == 0:
+            signable.message.ParseFromString(data[1:])
+        elif data[0] == 1:
+            signable.signing_channel_hash = data[1:21]
+            signable.signature = data[21:85]
+            signable.message.ParseFromString(data[85:])
+        else:
+            raise DecodeError('Could not determine message format version.')
+        return signable
+
+    def __len__(self):
+        return len(self.to_bytes())
+
+    def __bytes__(self):
+        return self.to_bytes()
+
+
+class Metadata:
+
+    __slots__ = 'message',
+
+    def __init__(self, message):
+        self.message = message
+
+
+I = TypeVar('I')
+
+
+class BaseMessageList(Metadata, Generic[I]):
+
+    __slots__ = ()
+
+    item_class = None
+
+    @property
+    def _message(self):
+        return self.message
+
+    def add(self) -> I:
+        return self.item_class(self._message.add())
+
+    def extend(self, values: List[str]):
+        for value in values:
+            self.append(value)
+
+    def append(self, value: str):
+        raise NotImplemented
+
+    def __len__(self):
+        return len(self._message)
+
+    def __iter__(self) -> Iterator[I]:
+        for item in self._message:
+            yield self.item_class(item)
+
+    def __getitem__(self, item) -> I:
+        return self.item_class(self._message[item])
+
+    def __delitem__(self, key):
+        del self._message[key]
+
+    def __eq__(self, other) -> bool:
+        return self._message == other

scribe/schema/claim.py

@@ -0,0 +1,422 @@
+import logging
+from typing import List
+from binascii import hexlify, unhexlify
+
+from asn1crypto.keys import PublicKeyInfo
+from coincurve import PublicKey as cPublicKey
+
+from google.protobuf.json_format import MessageToDict
+from google.protobuf.message import DecodeError
+from hachoir.core.log import log as hachoir_log
+from hachoir.parser import createParser as binary_file_parser
+from hachoir.metadata import extractMetadata as binary_file_metadata
+
+from scribe.schema import compat
+from scribe.schema.base import Signable
+from scribe.schema.mime_types import guess_media_type, guess_stream_type
+from scribe.schema.attrs import (
+    Source, Playable, Dimmensional, Fee, Image, Video, Audio,
+    LanguageList, LocationList, ClaimList, ClaimReference, TagList
+)
+from scribe.schema.types.v2.claim_pb2 import Claim as ClaimMessage
+from scribe.error import InputValueIsNoneError
+
+
+hachoir_log.use_print = False
+log = logging.getLogger(__name__)
+
+
+class Claim(Signable):
+
+    STREAM = 'stream'
+    CHANNEL = 'channel'
+    COLLECTION = 'collection'
+    REPOST = 'repost'
+
+    __slots__ = ()
+
+    message_class = ClaimMessage
+
+    @property
+    def claim_type(self) -> str:
+        return self.message.WhichOneof('type')
+
+    def get_message(self, type_name):
+        message = getattr(self.message, type_name)
+        if self.claim_type is None:
+            message.SetInParent()
+        if self.claim_type != type_name:
+            raise ValueError(f'Claim is not a {type_name}.')
+        return message
+
+    @property
+    def is_stream(self):
+        return self.claim_type == self.STREAM
+
+    @property
+    def stream(self) -> 'Stream':
+        return Stream(self)
+
+    @property
+    def is_channel(self):
+        return self.claim_type == self.CHANNEL
+
+    @property
+    def channel(self) -> 'Channel':
+        return Channel(self)
+
+    @property
+    def is_repost(self):
+        return self.claim_type == self.REPOST
+
+    @property
+    def repost(self) -> 'Repost':
+        return Repost(self)
+
+    @property
+    def is_collection(self):
+        return self.claim_type == self.COLLECTION
+
+    @property
+    def collection(self) -> 'Collection':
+        return Collection(self)
+
+    @classmethod
+    def from_bytes(cls, data: bytes) -> 'Claim':
+        try:
+            return super().from_bytes(data)
+        except DecodeError:
+            claim = cls()
+            if data[0] == ord('{'):
+                claim.version = 0
+                compat.from_old_json_schema(claim, data)
+            elif data[0] not in (0, 1):
+                claim.version = 1
+                compat.from_types_v1(claim, data)
+            else:
+                raise
+            return claim
+
+
+class BaseClaim:
+
+    __slots__ = 'claim', 'message'
+
+    claim_type = None
+    object_fields = 'thumbnail',
+    repeat_fields = 'tags', 'languages', 'locations'
+
+    def __init__(self, claim: Claim = None):
+        self.claim = claim or Claim()
+        self.message = self.claim.get_message(self.claim_type)
+
+    def to_dict(self):
+        claim = MessageToDict(self.claim.message, preserving_proto_field_name=True)
+        claim.update(claim.pop(self.claim_type))
+        if 'languages' in claim:
+            claim['languages'] = self.langtags
+        if 'locations' in claim:
+            claim['locations'] = [l.to_dict() for l in self.locations]
+        return claim
+
+    def none_check(self, kwargs):
+        for key, value in kwargs.items():
+            if value is None:
+                raise InputValueIsNoneError(key)
+
+    def update(self, **kwargs):
+        self.none_check(kwargs)
+
+        for key in list(kwargs):
+            for field in self.object_fields:
+                if key.startswith(f'{field}_'):
+                    attr = getattr(self, field)
+                    setattr(attr, key[len(f'{field}_'):], kwargs.pop(key))
+                    continue
+
+        for l in self.repeat_fields:
+            field = getattr(self, l)
+            if kwargs.pop(f'clear_{l}', False):
+                del field[:]
+            items = kwargs.pop(l, None)
+            if items is not None:
+                if isinstance(items, str):
+                    field.append(items)
+                elif isinstance(items, list):
+                    field.extend(items)
+                else:
+                    raise ValueError(f"Unknown {l} value: {items}")
+
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+
+    @property
+    def title(self) -> str:
+        return self.claim.message.title
+
+    @title.setter
+    def title(self, title: str):
+        self.claim.message.title = title
+
+    @property
+    def description(self) -> str:
+        return self.claim.message.description
+
+    @description.setter
+    def description(self, description: str):
+        self.claim.message.description = description
+
+    @property
+    def thumbnail(self) -> Source:
+        return Source(self.claim.message.thumbnail)
+
+    @property
+    def tags(self) -> List[str]:
+        return TagList(self.claim.message.tags)
+
+    @property
+    def languages(self) -> LanguageList:
+        return LanguageList(self.claim.message.languages)
+
+    @property
+    def langtags(self) -> List[str]:
+        return [l.langtag for l in self.languages]
+
+    @property
+    def locations(self) -> LocationList:
+        return LocationList(self.claim.message.locations)
+
+
+class Stream(BaseClaim):
+
+    __slots__ = ()
+
+    claim_type = Claim.STREAM
+
+    object_fields = BaseClaim.object_fields + ('source',)
+
+    def to_dict(self):
+        claim = super().to_dict()
+        if 'source' in claim:
+            if 'hash' in claim['source']:
+                claim['source']['hash'] = self.source.file_hash
+            if 'sd_hash' in claim['source']:
+                claim['source']['sd_hash'] = self.source.sd_hash
+            elif 'bt_infohash' in claim['source']:
+                claim['source']['bt_infohash'] = self.source.bt_infohash
+            if 'media_type' in claim['source']:
+                claim['stream_type'] = guess_stream_type(claim['source']['media_type'])
+        fee = claim.get('fee', {})
+        if 'address' in fee:
+            fee['address'] = self.fee.address
+        if 'amount' in fee:
+            fee['amount'] = str(self.fee.amount)
+        return claim
+
+    def update(self, file_path=None, height=None, width=None, duration=None, **kwargs):
+
+        if kwargs.pop('clear_fee', False):
+            self.message.ClearField('fee')
+        else:
+            self.fee.update(
+                kwargs.pop('fee_address', None),
+                kwargs.pop('fee_currency', None),
+                kwargs.pop('fee_amount', None)
+            )
+
+        self.none_check(kwargs)
+
+        if 'sd_hash' in kwargs:
+            self.source.sd_hash = kwargs.pop('sd_hash')
+        elif 'bt_infohash' in kwargs:
+            self.source.bt_infohash = kwargs.pop('bt_infohash')
+        if 'file_name' in kwargs:
+            self.source.name = kwargs.pop('file_name')
+        if 'file_hash' in kwargs:
+            self.source.file_hash = kwargs.pop('file_hash')
+
+        stream_type = None
+        if file_path is not None:
+            stream_type = self.source.update(file_path=file_path)
+        elif self.source.name:
+            self.source.media_type, stream_type = guess_media_type(self.source.name)
+        elif self.source.media_type:
+            stream_type = guess_stream_type(self.source.media_type)
+
+        if 'file_size' in kwargs:
+            self.source.size = kwargs.pop('file_size')
+
+        if self.stream_type is not None and self.stream_type != stream_type:
+            self.message.ClearField(self.stream_type)
+
+        if stream_type in ('image', 'video', 'audio'):
+            media = getattr(self, stream_type)
+            media_args = {'file_metadata': None}
+            if file_path is not None and not all((duration, width, height)):
+                try:
+                    media_args['file_metadata'] = binary_file_metadata(binary_file_parser(file_path))
+                except:
+                    log.exception('Could not read file metadata.')
+            if isinstance(media, Playable):
+                media_args['duration'] = duration
+            if isinstance(media, Dimmensional):
+                media_args['height'] = height
+                media_args['width'] = width
+            media.update(**media_args)
+
+        super().update(**kwargs)
+
+    @property
+    def author(self) -> str:
+        return self.message.author
+
+    @author.setter
+    def author(self, author: str):
+        self.message.author = author
+
+    @property
+    def license(self) -> str:
+        return self.message.license
+
+    @license.setter
+    def license(self, license: str):
+        self.message.license = license
+
+    @property
+    def license_url(self) -> str:
+        return self.message.license_url
+
+    @license_url.setter
+    def license_url(self, license_url: str):
+        self.message.license_url = license_url
+
+    @property
+    def release_time(self) -> int:
+        return self.message.release_time
+
+    @release_time.setter
+    def release_time(self, release_time: int):
+        self.message.release_time = release_time
+
+    @property
+    def fee(self) -> Fee:
+        return Fee(self.message.fee)
+
+    @property
+    def has_fee(self) -> bool:
+        return self.message.HasField('fee')
+
+    @property
+    def has_source(self) -> bool:
+        return self.message.HasField('source')
+
+    @property
+    def source(self) -> Source:
+        return Source(self.message.source)
+
+    @property
+    def stream_type(self) -> str:
+        return self.message.WhichOneof('type')
+
+    @property
+    def image(self) -> Image:
+        return Image(self.message.image)
+
+    @property
+    def video(self) -> Video:
+        return Video(self.message.video)
+
+    @property
+    def audio(self) -> Audio:
+        return Audio(self.message.audio)
+
+
+class Channel(BaseClaim):
+
+    __slots__ = ()
+
+    claim_type = Claim.CHANNEL
+
+    object_fields = BaseClaim.object_fields + ('cover',)
+    repeat_fields = BaseClaim.repeat_fields + ('featured',)
+
+    def to_dict(self):
+        claim = super().to_dict()
+        claim['public_key'] = self.public_key
+        if 'featured' in claim:
+            claim['featured'] = self.featured.ids
+        return claim
+
+    @property
+    def public_key(self) -> str:
+        return hexlify(self.public_key_bytes).decode()
+
+    @public_key.setter
+    def public_key(self, sd_public_key: str):
+        self.message.public_key = unhexlify(sd_public_key.encode())
+
+    @property
+    def public_key_bytes(self) -> bytes:
+        if len(self.message.public_key) == 33:
+            return self.message.public_key
+        public_key_info = PublicKeyInfo.load(self.message.public_key)
+        public_key = cPublicKey(public_key_info.native['public_key'])
+        return public_key.format(compressed=True)
+
+    @public_key_bytes.setter
+    def public_key_bytes(self, public_key: bytes):
+        self.message.public_key = public_key
+
+    @property
+    def email(self) -> str:
+        return self.message.email
+
+    @email.setter
+    def email(self, email: str):
+        self.message.email = email
+
+    @property
+    def website_url(self) -> str:
+        return self.message.website_url
+
+    @website_url.setter
+    def website_url(self, website_url: str):
+        self.message.website_url = website_url
+
+    @property
+    def cover(self) -> Source:
+        return Source(self.message.cover)
+
+    @property
+    def featured(self) -> ClaimList:
+        return ClaimList(self.message.featured)
+
+
+class Repost(BaseClaim):
+
+    __slots__ = ()
+
+    claim_type = Claim.REPOST
+
+    @property
+    def reference(self) -> ClaimReference:
+        return ClaimReference(self.message)
+
+
+class Collection(BaseClaim):
+
+    __slots__ = ()
+
+    claim_type = Claim.COLLECTION
+
+    repeat_fields = BaseClaim.repeat_fields + ('claims',)
+
+    def to_dict(self):
+        claim = super().to_dict()
+        if claim.pop('claim_references', None):
+            claim['claims'] = self.claims.ids
+        return claim
+
+    @property
+    def claims(self) -> ClaimList:
+        return ClaimList(self.message)

scribe/schema/compat.py

@@ -0,0 +1,93 @@
+import json
+from decimal import Decimal
+
+from google.protobuf.message import DecodeError
+
+from scribe.schema.types.v1.legacy_claim_pb2 import Claim as OldClaimMessage
+from scribe.schema.types.v1.certificate_pb2 import KeyType
+from scribe.schema.types.v1.fee_pb2 import Fee as FeeMessage
+
+
+def from_old_json_schema(claim, payload: bytes):
+    try:
+        value = json.loads(payload)
+    except:
+        raise DecodeError('Could not parse JSON.')
+    stream = claim.stream
+    stream.source.sd_hash = value['sources']['lbry_sd_hash']
+    stream.source.media_type = (
+            value.get('content_type', value.get('content-type')) or
+            'application/octet-stream'
+    )
+    stream.title = value.get('title', '')
+    stream.description = value.get('description', '')
+    if value.get('thumbnail', ''):
+        stream.thumbnail.url = value.get('thumbnail', '')
+    stream.author = value.get('author', '')
+    stream.license = value.get('license', '')
+    stream.license_url = value.get('license_url', '')
+    language = value.get('language', '')
+    if language:
+        if language.lower() == 'english':
+            language = 'en'
+        try:
+            stream.languages.append(language)
+        except:
+            pass
+    if value.get('nsfw', False):
+        stream.tags.append('mature')
+    if "fee" in value and isinstance(value['fee'], dict):
+        fee = value["fee"]
+        currency = list(fee.keys())[0]
+        if currency == 'LBC':
+            stream.fee.lbc = Decimal(fee[currency]['amount'])
+        elif currency == 'USD':
+            stream.fee.usd = Decimal(fee[currency]['amount'])
+        elif currency == 'BTC':
+            stream.fee.btc = Decimal(fee[currency]['amount'])
+        else:
+            raise DecodeError(f'Unknown currency: {currency}')
+        stream.fee.address = fee[currency]['address']
+    return claim
+
+
+def from_types_v1(claim, payload: bytes):
+    old = OldClaimMessage()
+    old.ParseFromString(payload)
+    if old.claimType == 2:
+        channel = claim.channel
+        channel.public_key_bytes = old.certificate.publicKey
+    else:
+        stream = claim.stream
+        stream.title = old.stream.metadata.title
+        stream.description = old.stream.metadata.description
+        stream.author = old.stream.metadata.author
+        stream.license = old.stream.metadata.license
+        stream.license_url = old.stream.metadata.licenseUrl
+        stream.thumbnail.url = old.stream.metadata.thumbnail
+        if old.stream.metadata.HasField('language'):
+            stream.languages.add().message.language = old.stream.metadata.language
+        stream.source.media_type = old.stream.source.contentType
+        stream.source.sd_hash_bytes = old.stream.source.source
+        if old.stream.metadata.nsfw:
+            stream.tags.append('mature')
+        if old.stream.metadata.HasField('fee'):
+            fee = old.stream.metadata.fee
+            stream.fee.address_bytes = fee.address
+            currency = FeeMessage.Currency.Name(fee.currency)
+            if currency == 'LBC':
+                stream.fee.lbc = Decimal(fee.amount)
+            elif currency == 'USD':
+                stream.fee.usd = Decimal(fee.amount)
+            elif currency == 'BTC':
+                stream.fee.btc = Decimal(fee.amount)
+            else:
+                raise DecodeError(f'Unsupported currency: {currency}')
+        if old.HasField('publisherSignature'):
+            sig = old.publisherSignature
+            claim.signature = sig.signature
+            claim.signature_type = KeyType.Name(sig.signatureType)
+            claim.signing_channel_hash = sig.certificateId[::-1]
+            old.ClearField("publisherSignature")
+            claim.unsigned_payload = old.SerializeToString()
+    return claim

scribe/schema/mime_types.py

@@ -0,0 +1,214 @@
+import os
+import filetype
+import logging
+
+types_map = {
+    # http://www.iana.org/assignments/media-types
+    # Type mapping for automated metadata extraction (video, audio, image, document, binary, model)
+    '.a': ('application/octet-stream', 'binary'),
+    '.ai': ('application/postscript', 'image'),
+    '.aif': ('audio/x-aiff', 'audio'),
+    '.aifc': ('audio/x-aiff', 'audio'),
+    '.aiff': ('audio/x-aiff', 'audio'),
+    '.au': ('audio/basic', 'audio'),
+    '.avi': ('video/x-msvideo', 'video'),
+    '.bat': ('text/plain', 'document'),
+    '.bcpio': ('application/x-bcpio', 'binary'),
+    '.bin': ('application/octet-stream', 'binary'),
+    '.bmp': ('image/bmp', 'image'),
+    '.c': ('text/plain', 'document'),
+    '.cdf': ('application/x-netcdf', 'binary'),
+    '.cpio': ('application/x-cpio', 'binary'),
+    '.csh': ('application/x-csh', 'binary'),
+    '.css': ('text/css', 'document'),
+    '.csv': ('text/csv', 'document'),
+    '.dll': ('application/octet-stream', 'binary'),
+    '.doc': ('application/msword', 'document'),
+    '.dot': ('application/msword', 'document'),
+    '.dvi': ('application/x-dvi', 'binary'),
+    '.eml': ('message/rfc822', 'document'),
+    '.eps': ('application/postscript', 'document'),
+    '.epub': ('application/epub+zip', 'document'),
+    '.etx': ('text/x-setext', 'document'),
+    '.exe': ('application/octet-stream', 'binary'),
+    '.gif': ('image/gif', 'image'),
+    '.gtar': ('application/x-gtar', 'binary'),
+    '.h': ('text/plain', 'document'),
+    '.hdf': ('application/x-hdf', 'binary'),
+    '.htm': ('text/html', 'document'),
+    '.html': ('text/html', 'document'),
+    '.ico': ('image/vnd.microsoft.icon', 'image'),
+    '.ief': ('image/ief', 'image'),
+    '.iges': ('model/iges', 'model'),
+    '.jpe': ('image/jpeg', 'image'),
+    '.jpeg': ('image/jpeg', 'image'),
+    '.jpg': ('image/jpeg', 'image'),
+    '.js': ('application/javascript', 'document'),
+    '.json': ('application/json', 'document'),
+    '.ksh': ('text/plain', 'document'),
+    '.latex': ('application/x-latex', 'binary'),
+    '.m1v': ('video/mpeg', 'video'),
+    '.m3u': ('application/x-mpegurl', 'audio'),
+    '.m3u8': ('application/x-mpegurl', 'video'),
+    '.man': ('application/x-troff-man', 'document'),
+    '.markdown': ('text/markdown', 'document'),
+    '.md': ('text/markdown', 'document'),
+    '.me': ('application/x-troff-me', 'binary'),
+    '.mht': ('message/rfc822', 'document'),
+    '.mhtml': ('message/rfc822', 'document'),
+    '.mif': ('application/x-mif', 'binary'),
+    '.mov': ('video/quicktime', 'video'),
+    '.movie': ('video/x-sgi-movie', 'video'),
+    '.mp2': ('audio/mpeg', 'audio'),
+    '.mp3': ('audio/mpeg', 'audio'),
+    '.mp4': ('video/mp4', 'video'),
+    '.mpa': ('video/mpeg', 'video'),
+    '.mpd': ('application/dash+xml', 'video'),
+    '.mpe': ('video/mpeg', 'video'),
+    '.mpeg': ('video/mpeg', 'video'),
+    '.mpg': ('video/mpeg', 'video'),
+    '.ms': ('application/x-troff-ms', 'binary'),
+    '.m4s': ('video/iso.segment', 'binary'),
+    '.nc': ('application/x-netcdf', 'binary'),
+    '.nws': ('message/rfc822', 'document'),
+    '.o': ('application/octet-stream', 'binary'),
+    '.obj': ('application/octet-stream', 'model'),
+    '.oda': ('application/oda', 'binary'),
+    '.p12': ('application/x-pkcs12', 'binary'),
+    '.p7c': ('application/pkcs7-mime', 'binary'),
+    '.pbm': ('image/x-portable-bitmap', 'image'),
+    '.pdf': ('application/pdf', 'document'),
+    '.pfx': ('application/x-pkcs12', 'binary'),
+    '.pgm': ('image/x-portable-graymap', 'image'),
+    '.pl': ('text/plain', 'document'),
+    '.png': ('image/png', 'image'),
+    '.pnm': ('image/x-portable-anymap', 'image'),
+    '.pot': ('application/vnd.ms-powerpoint', 'document'),
+    '.ppa': ('application/vnd.ms-powerpoint', 'document'),
+    '.ppm': ('image/x-portable-pixmap', 'image'),
+    '.pps': ('application/vnd.ms-powerpoint', 'document'),
+    '.ppt': ('application/vnd.ms-powerpoint', 'document'),
+    '.ps': ('application/postscript', 'document'),
+    '.pwz': ('application/vnd.ms-powerpoint', 'document'),
+    '.py': ('text/x-python', 'document'),
+    '.pyc': ('application/x-python-code', 'binary'),
+    '.pyo': ('application/x-python-code', 'binary'),
+    '.qt': ('video/quicktime', 'video'),
+    '.ra': ('audio/x-pn-realaudio', 'audio'),
+    '.ram': ('application/x-pn-realaudio', 'audio'),
+    '.ras': ('image/x-cmu-raster', 'image'),
+    '.rdf': ('application/xml', 'binary'),
+    '.rgb': ('image/x-rgb', 'image'),
+    '.roff': ('application/x-troff', 'binary'),
+    '.rtx': ('text/richtext', 'document'),
+    '.sgm': ('text/x-sgml', 'document'),
+    '.sgml': ('text/x-sgml', 'document'),
+    '.sh': ('application/x-sh', 'document'),
+    '.shar': ('application/x-shar', 'binary'),
+    '.snd': ('audio/basic', 'audio'),
+    '.so': ('application/octet-stream', 'binary'),
+    '.src': ('application/x-wais-source', 'binary'),
+    '.stl': ('model/stl', 'model'),
+    '.sv4cpio': ('application/x-sv4cpio', 'binary'),
+    '.sv4crc': ('application/x-sv4crc', 'binary'),
+    '.svg': ('image/svg+xml', 'image'),
+    '.swf': ('application/x-shockwave-flash', 'binary'),
+    '.t': ('application/x-troff', 'binary'),
+    '.tar': ('application/x-tar', 'binary'),
+    '.tcl': ('application/x-tcl', 'binary'),
+    '.tex': ('application/x-tex', 'binary'),
+    '.texi': ('application/x-texinfo', 'binary'),
+    '.texinfo': ('application/x-texinfo', 'binary'),
+    '.tif': ('image/tiff', 'image'),
+    '.tiff': ('image/tiff', 'image'),
+    '.tr': ('application/x-troff', 'binary'),
+    '.ts': ('video/mp2t', 'video'),
+    '.tsv': ('text/tab-separated-values', 'document'),
+    '.txt': ('text/plain', 'document'),
+    '.ustar': ('application/x-ustar', 'binary'),
+    '.vcf': ('text/x-vcard', 'document'),
+    '.vtt': ('text/vtt', 'document'),
+    '.wav': ('audio/x-wav', 'audio'),
+    '.webm': ('video/webm', 'video'),
+    '.wiz': ('application/msword', 'document'),
+    '.wsdl': ('application/xml', 'document'),
+    '.xbm': ('image/x-xbitmap', 'image'),
+    '.xlb': ('application/vnd.ms-excel', 'document'),
+    '.xls': ('application/vnd.ms-excel', 'document'),
+    '.xml': ('text/xml', 'document'),
+    '.xpdl': ('application/xml', 'document'),
+    '.xpm': ('image/x-xpixmap', 'image'),
+    '.xsl': ('application/xml', 'document'),
+    '.xwd': ('image/x-xwindowdump', 'image'),
+    '.zip': ('application/zip', 'binary'),
+
+    # These are non-standard types, commonly found in the wild.
+    '.cbr': ('application/vnd.comicbook-rar', 'document'),
+    '.cbz': ('application/vnd.comicbook+zip', 'document'),
+    '.flac': ('audio/flac', 'audio'),
+    '.lbry': ('application/x-ext-lbry', 'document'),
+    '.m4a': ('audio/mp4', 'audio'),
+    '.m4v': ('video/m4v', 'video'),
+    '.mid': ('audio/midi', 'audio'),
+    '.midi': ('audio/midi', 'audio'),
+    '.mkv': ('video/x-matroska', 'video'),
+    '.mobi': ('application/x-mobipocket-ebook', 'document'),
+    '.oga': ('audio/ogg', 'audio'),
+    '.ogv': ('video/ogg', 'video'),
+    '.ogg': ('video/ogg', 'video'),
+    '.pct': ('image/pict', 'image'),
+    '.pic': ('image/pict', 'image'),
+    '.pict': ('image/pict', 'image'),
+    '.prc': ('application/x-mobipocket-ebook', 'document'),
+    '.rtf': ('application/rtf', 'document'),
+    '.xul': ('text/xul', 'document'),
+    
+    # microsoft is special and has its own 'standard'
+    # https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
+    '.wmv': ('video/x-ms-wmv', 'video')
+}
+
+# maps detected extensions to the possible analogs
+# i.e. .cbz file is actually a .zip
+synonyms_map = {
+    '.zip': ['.cbz'],
+    '.rar': ['.cbr'],
+    '.ar': ['.a']
+}
+
+log = logging.getLogger(__name__)
+
+
+def guess_media_type(path):
+    _, ext = os.path.splitext(path)
+    extension = ext.strip().lower()
+
+    try:
+        kind = filetype.guess(path)
+        if kind:
+            real_extension = f".{kind.extension}"
+
+            if extension != real_extension:
+                if extension:
+                    log.warning(f"file extension does not match it's contents: {path}, identified as {real_extension}")
+                else:
+                    log.debug(f"file {path} does not have extension, identified by it's contents as {real_extension}")
+
+                if extension not in synonyms_map.get(real_extension, []):
+                    extension = real_extension
+
+    except OSError as error:
+        pass
+
+    if extension[1:]:
+        if extension in types_map:
+            return types_map[extension]
+        return f'application/x-ext-{extension[1:]}', 'binary'
+    return 'application/octet-stream', 'binary'
+
+
+def guess_stream_type(media_type):
+    for media, stream in types_map.values():
+        if media == media_type:
+            return stream
+    return 'binary'

scribe/schema/purchase.py

@@ -0,0 +1,47 @@
+from google.protobuf.message import DecodeError
+from google.protobuf.json_format import MessageToDict
+from scribe.schema.types.v2.purchase_pb2 import Purchase as PurchaseMessage
+from .attrs import ClaimReference
+
+
+class Purchase(ClaimReference):
+
+    START_BYTE = ord('P')
+
+    __slots__ = ()
+
+    def __init__(self, claim_id=None):
+        super().__init__(PurchaseMessage())
+        if claim_id is not None:
+            self.claim_id = claim_id
+
+    def to_dict(self):
+        return MessageToDict(self.message)
+
+    def to_message_bytes(self) -> bytes:
+        return self.message.SerializeToString()
+
+    def to_bytes(self) -> bytes:
+        pieces = bytearray()
+        pieces.append(self.START_BYTE)
+        pieces.extend(self.to_message_bytes())
+        return bytes(pieces)
+
+    @classmethod
+    def has_start_byte(cls, data: bytes):
+        return data and data[0] == cls.START_BYTE
+
+    @classmethod
+    def from_bytes(cls, data: bytes):
+        purchase = cls()
+        if purchase.has_start_byte(data):
+            purchase.message.ParseFromString(data[1:])
+        else:
+            raise DecodeError('Message does not start with correct byte.')
+        return purchase
+
+    def __len__(self):
+        return len(self.to_bytes())
+
+    def __bytes__(self):
+        return self.to_bytes()

scribe/schema/result.py

@@ -0,0 +1,258 @@
+import base64
+from typing import List, TYPE_CHECKING, Union, Optional, NamedTuple
+from binascii import hexlify
+from itertools import chain
+
+from scribe.error import ResolveCensoredError
+from scribe.schema.types.v2.result_pb2 import Outputs as OutputsMessage
+from scribe.schema.types.v2.result_pb2 import Error as ErrorMessage
+# if TYPE_CHECKING:
+#     from lbry_schema.schema.claim import ResolveResult
+
+INVALID = ErrorMessage.Code.Name(ErrorMessage.INVALID)
+NOT_FOUND = ErrorMessage.Code.Name(ErrorMessage.NOT_FOUND)
+BLOCKED = ErrorMessage.Code.Name(ErrorMessage.BLOCKED)
+
+
+def set_reference(reference, claim_hash, rows):
+    if claim_hash:
+        for txo in rows:
+            if claim_hash == txo.claim_hash:
+                reference.tx_hash = txo.tx_hash
+                reference.nout = txo.position
+                reference.height = txo.height
+                return
+
+
+class Censor:
+
+    NOT_CENSORED = 0
+    SEARCH = 1
+    RESOLVE = 2
+
+    __slots__ = 'censor_type', 'censored'
+
+    def __init__(self, censor_type):
+        self.censor_type = censor_type
+        self.censored = {}
+
+    def is_censored(self, row):
+        return (row.get('censor_type') or self.NOT_CENSORED) >= self.censor_type
+
+    def apply(self, rows):
+        return [row for row in rows if not self.censor(row)]
+
+    def censor(self, row) -> Optional[bytes]:
+        if self.is_censored(row):
+            censoring_channel_hash = bytes.fromhex(row['censoring_channel_id'])[::-1]
+            self.censored.setdefault(censoring_channel_hash, set())
+            self.censored[censoring_channel_hash].add(row['tx_hash'])
+            return censoring_channel_hash
+        return None
+
+    def to_message(self, outputs: OutputsMessage, extra_txo_rows: dict):
+        for censoring_channel_hash, count in self.censored.items():
+            blocked = outputs.blocked.add()
+            blocked.count = len(count)
+            set_reference(blocked.channel, censoring_channel_hash, extra_txo_rows)
+            outputs.blocked_total += len(count)
+
+
+class ResolveResult(NamedTuple):
+    name: str
+    normalized_name: str
+    claim_hash: bytes
+    tx_num: int
+    position: int
+    tx_hash: bytes
+    height: int
+    amount: int
+    short_url: str
+    is_controlling: bool
+    canonical_url: str
+    creation_height: int
+    activation_height: int
+    expiration_height: int
+    effective_amount: int
+    support_amount: int
+    reposted: int
+    last_takeover_height: Optional[int]
+    claims_in_channel: Optional[int]
+    channel_hash: Optional[bytes]
+    reposted_claim_hash: Optional[bytes]
+    signature_valid: Optional[bool]
+
+
+
+class Outputs:
+
+    __slots__ = 'txos', 'extra_txos', 'txs', 'offset', 'total', 'blocked', 'blocked_total'
+
+    def __init__(self, txos: List, extra_txos: List, txs: set,
+                 offset: int, total: int, blocked: List, blocked_total: int):
+        self.txos = txos
+        self.txs = txs
+        self.extra_txos = extra_txos
+        self.offset = offset
+        self.total = total
+        self.blocked = blocked
+        self.blocked_total = blocked_total
+
+    def inflate(self, txs):
+        tx_map = {tx.hash: tx for tx in txs}
+        for txo_message in self.extra_txos:
+            self.message_to_txo(txo_message, tx_map)
+        txos = [self.message_to_txo(txo_message, tx_map) for txo_message in self.txos]
+        return txos, self.inflate_blocked(tx_map)
+
+    def inflate_blocked(self, tx_map):
+        return {
+            "total": self.blocked_total,
+            "channels": [{
+                'channel': self.message_to_txo(blocked.channel, tx_map),
+                'blocked': blocked.count
+            } for blocked in self.blocked]
+        }
+
+    def message_to_txo(self, txo_message, tx_map):
+        if txo_message.WhichOneof('meta') == 'error':
+            error = {
+                'error': {
+                    'name': txo_message.error.Code.Name(txo_message.error.code),
+                    'text': txo_message.error.text,
+                }
+            }
+            if error['error']['name'] == BLOCKED:
+                error['error']['censor'] = self.message_to_txo(
+                    txo_message.error.blocked.channel, tx_map
+                )
+            return error
+
+        tx = tx_map.get(txo_message.tx_hash)
+        if not tx:
+            return
+        txo = tx.outputs[txo_message.nout]
+        if txo_message.WhichOneof('meta') == 'claim':
+            claim = txo_message.claim
+            txo.meta = {
+                'short_url': f'lbry://{claim.short_url}',
+                'canonical_url': f'lbry://{claim.canonical_url or claim.short_url}',
+                'reposted': claim.reposted,
+                'is_controlling': claim.is_controlling,
+                'take_over_height': claim.take_over_height,
+                'creation_height': claim.creation_height,
+                'activation_height': claim.activation_height,
+                'expiration_height': claim.expiration_height,
+                'effective_amount': claim.effective_amount,
+                'support_amount': claim.support_amount,
+                # 'trending_group': claim.trending_group,
+                # 'trending_mixed': claim.trending_mixed,
+                # 'trending_local': claim.trending_local,
+                # 'trending_global': claim.trending_global,
+            }
+            if claim.HasField('channel'):
+                txo.channel = tx_map[claim.channel.tx_hash].outputs[claim.channel.nout]
+            if claim.HasField('repost'):
+                txo.reposted_claim = tx_map[claim.repost.tx_hash].outputs[claim.repost.nout]
+            try:
+                if txo.claim.is_channel:
+                    txo.meta['claims_in_channel'] = claim.claims_in_channel
+            except:
+                pass
+        return txo
+
+    @classmethod
+    def from_base64(cls, data: str) -> 'Outputs':
+        return cls.from_bytes(base64.b64decode(data))
+
+    @classmethod
+    def from_bytes(cls, data: bytes) -> 'Outputs':
+        outputs = OutputsMessage()
+        outputs.ParseFromString(data)
+        txs = set()
+        for txo_message in chain(outputs.txos, outputs.extra_txos):
+            if txo_message.WhichOneof('meta') == 'error':
+                continue
+            txs.add((hexlify(txo_message.tx_hash[::-1]).decode(), txo_message.height))
+        return cls(
+            outputs.txos, outputs.extra_txos, txs,
+            outputs.offset, outputs.total,
+            outputs.blocked, outputs.blocked_total
+        )
+
+    @classmethod
+    def from_grpc(cls, outputs: OutputsMessage) -> 'Outputs':
+        txs = set()
+        for txo_message in chain(outputs.txos, outputs.extra_txos):
+            if txo_message.WhichOneof('meta') == 'error':
+                continue
+            txs.add((hexlify(txo_message.tx_hash[::-1]).decode(), txo_message.height))
+        return cls(
+            outputs.txos, outputs.extra_txos, txs,
+            outputs.offset, outputs.total,
+            outputs.blocked, outputs.blocked_total
+        )
+
+    @classmethod
+    def to_base64(cls, txo_rows, extra_txo_rows, offset=0, total=None, blocked=None) -> str:
+        return base64.b64encode(cls.to_bytes(txo_rows, extra_txo_rows, offset, total, blocked)).decode()
+
+    @classmethod
+    def to_bytes(cls, txo_rows, extra_txo_rows, offset=0, total=None, blocked: Censor = None) -> bytes:
+        page = OutputsMessage()
+        page.offset = offset
+        if total is not None:
+            page.total = total
+        if blocked is not None:
+            blocked.to_message(page, extra_txo_rows)
+        for row in extra_txo_rows:
+            txo_message: 'OutputsMessage' = page.extra_txos.add()
+            if not isinstance(row, Exception):
+                if row.channel_hash:
+                    set_reference(txo_message.claim.channel, row.channel_hash, extra_txo_rows)
+                if row.reposted_claim_hash:
+                    set_reference(txo_message.claim.repost, row.reposted_claim_hash, extra_txo_rows)
+            cls.encode_txo(txo_message, row)
+
+        for row in txo_rows:
+            # cls.row_to_message(row, page.txos.add(), extra_txo_rows)
+            txo_message: 'OutputsMessage' = page.txos.add()
+            cls.encode_txo(txo_message, row)
+            if not isinstance(row, Exception):
+                if row.channel_hash:
+                    set_reference(txo_message.claim.channel, row.channel_hash, extra_txo_rows)
+                if row.reposted_claim_hash:
+                    set_reference(txo_message.claim.repost, row.reposted_claim_hash, extra_txo_rows)
+            elif isinstance(row, ResolveCensoredError):
+                set_reference(txo_message.error.blocked.channel, row.censor_id, extra_txo_rows)
+        return page.SerializeToString()
+
+    @classmethod
+    def encode_txo(cls, txo_message, resolve_result: Union[ResolveResult, Exception]):
+        if isinstance(resolve_result, Exception):
+            txo_message.error.text = resolve_result.args[0]
+            if isinstance(resolve_result, ValueError):
+                txo_message.error.code = ErrorMessage.INVALID
+            elif isinstance(resolve_result, LookupError):
+                txo_message.error.code = ErrorMessage.NOT_FOUND
+            elif isinstance(resolve_result, ResolveCensoredError):
+                txo_message.error.code = ErrorMessage.BLOCKED
+            return
+        txo_message.tx_hash = resolve_result.tx_hash
+        txo_message.nout = resolve_result.position
+        txo_message.height = resolve_result.height
+        txo_message.claim.short_url = resolve_result.short_url
+        txo_message.claim.reposted = resolve_result.reposted
+        txo_message.claim.is_controlling = resolve_result.is_controlling
+        txo_message.claim.creation_height = resolve_result.creation_height
+        txo_message.claim.activation_height = resolve_result.activation_height
+        txo_message.claim.expiration_height = resolve_result.expiration_height
+        txo_message.claim.effective_amount = resolve_result.effective_amount
+        txo_message.claim.support_amount = resolve_result.support_amount
+
+        if resolve_result.canonical_url is not None:
+            txo_message.claim.canonical_url = resolve_result.canonical_url
+        if resolve_result.last_takeover_height is not None:
+            txo_message.claim.take_over_height = resolve_result.last_takeover_height
+        if resolve_result.claims_in_channel is not None:
+            txo_message.claim.claims_in_channel = resolve_result.claims_in_channel

scribe/schema/support.py

@@ -0,0 +1,23 @@
+from scribe.schema.base import Signable
+from scribe.schema.types.v2.support_pb2 import Support as SupportMessage
+
+
+class Support(Signable):
+    __slots__ = ()
+    message_class = SupportMessage
+
+    @property
+    def emoji(self) -> str:
+        return self.message.emoji
+
+    @emoji.setter
+    def emoji(self, emoji: str):
+        self.message.emoji = emoji
+
+    @property
+    def comment(self) -> str:
+        return self.message.comment
+
+    @comment.setter
+    def comment(self, comment: str):
+        self.message.comment = comment

scribe/schema/tags.py

@@ -0,0 +1,13 @@
+from typing import List
+import re
+
+MULTI_SPACE_RE = re.compile(r"\s{2,}")
+WEIRD_CHARS_RE = re.compile(r"[#!~]")
+
+
+def normalize_tag(tag: str):
+    return MULTI_SPACE_RE.sub(' ', WEIRD_CHARS_RE.sub(' ', tag.lower().replace("'", ""))).strip()
+
+
+def clean_tags(tags: List[str]):
+    return [tag for tag in {normalize_tag(tag) for tag in tags} if tag]

scribe/schema/types/v1/certificate_pb2.py

@@ -0,0 +1,146 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: certificate.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf.internal import enum_type_wrapper
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='certificate.proto',
+  package='legacy_pb',
+  syntax='proto2',
+  serialized_options=None,
+  serialized_pb=_b('\n\x11\x63\x65rtificate.proto\x12\tlegacy_pb\"\xa2\x01\n\x0b\x43\x65rtificate\x12/\n\x07version\x18\x01 \x02(\x0e\x32\x1e.legacy_pb.Certificate.Version\x12#\n\x07keyType\x18\x02 \x02(\x0e\x32\x12.legacy_pb.KeyType\x12\x11\n\tpublicKey\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01*Q\n\x07KeyType\x12\x1b\n\x17UNKNOWN_PUBLIC_KEY_TYPE\x10\x00\x12\x0c\n\x08NIST256p\x10\x01\x12\x0c\n\x08NIST384p\x10\x02\x12\r\n\tSECP256k1\x10\x03')
+)
+
+_KEYTYPE = _descriptor.EnumDescriptor(
+  name='KeyType',
+  full_name='legacy_pb.KeyType',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='UNKNOWN_PUBLIC_KEY_TYPE', index=0, number=0,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='NIST256p', index=1, number=1,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='NIST384p', index=2, number=2,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='SECP256k1', index=3, number=3,
+      serialized_options=None,
+      type=None),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=197,
+  serialized_end=278,
+)
+_sym_db.RegisterEnumDescriptor(_KEYTYPE)
+
+KeyType = enum_type_wrapper.EnumTypeWrapper(_KEYTYPE)
+UNKNOWN_PUBLIC_KEY_TYPE = 0
+NIST256p = 1
+NIST384p = 2
+SECP256k1 = 3
+
+
+_CERTIFICATE_VERSION = _descriptor.EnumDescriptor(
+  name='Version',
+  full_name='legacy_pb.Certificate.Version',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='UNKNOWN_VERSION', index=0, number=0,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='_0_0_1', index=1, number=1,
+      serialized_options=None,
+      type=None),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=153,
+  serialized_end=195,
+)
+_sym_db.RegisterEnumDescriptor(_CERTIFICATE_VERSION)
+
+
+_CERTIFICATE = _descriptor.Descriptor(
+  name='Certificate',
+  full_name='legacy_pb.Certificate',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='version', full_name='legacy_pb.Certificate.version', index=0,
+      number=1, type=14, cpp_type=8, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='keyType', full_name='legacy_pb.Certificate.keyType', index=1,
+      number=2, type=14, cpp_type=8, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='publicKey', full_name='legacy_pb.Certificate.publicKey', index=2,
+      number=4, type=12, cpp_type=9, label=2,
+      has_default_value=False, default_value=_b(""),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+    _CERTIFICATE_VERSION,
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=33,
+  serialized_end=195,
+)
+
+_CERTIFICATE.fields_by_name['version'].enum_type = _CERTIFICATE_VERSION
+_CERTIFICATE.fields_by_name['keyType'].enum_type = _KEYTYPE
+_CERTIFICATE_VERSION.containing_type = _CERTIFICATE
+DESCRIPTOR.message_types_by_name['Certificate'] = _CERTIFICATE
+DESCRIPTOR.enum_types_by_name['KeyType'] = _KEYTYPE
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Certificate = _reflection.GeneratedProtocolMessageType('Certificate', (_message.Message,), dict(
+  DESCRIPTOR = _CERTIFICATE,
+  __module__ = 'certificate_pb2'
+  # @@protoc_insertion_point(class_scope:legacy_pb.Certificate)
+  ))
+_sym_db.RegisterMessage(Certificate)
+
+
+# @@protoc_insertion_point(module_scope)

scribe/schema/types/v1/fee_pb2.py

@@ -0,0 +1,148 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: fee.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='fee.proto',
+  package='legacy_pb',
+  syntax='proto2',
+  serialized_options=None,
+  serialized_pb=_b('\n\tfee.proto\x12\tlegacy_pb\"\xe3\x01\n\x03\x46\x65\x65\x12\'\n\x07version\x18\x01 \x02(\x0e\x32\x16.legacy_pb.Fee.Version\x12)\n\x08\x63urrency\x18\x02 \x02(\x0e\x32\x17.legacy_pb.Fee.Currency\x12\x0f\n\x07\x61\x64\x64ress\x18\x03 \x02(\x0c\x12\x0e\n\x06\x61mount\x18\x04 \x02(\x02\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\";\n\x08\x43urrency\x12\x14\n\x10UNKNOWN_CURRENCY\x10\x00\x12\x07\n\x03LBC\x10\x01\x12\x07\n\x03\x42TC\x10\x02\x12\x07\n\x03USD\x10\x03')
+)
+
+
+
+_FEE_VERSION = _descriptor.EnumDescriptor(
+  name='Version',
+  full_name='legacy_pb.Fee.Version',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='UNKNOWN_VERSION', index=0, number=0,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='_0_0_1', index=1, number=1,
+      serialized_options=None,
+      type=None),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=149,
+  serialized_end=191,
+)
+_sym_db.RegisterEnumDescriptor(_FEE_VERSION)
+
+_FEE_CURRENCY = _descriptor.EnumDescriptor(
+  name='Currency',
+  full_name='legacy_pb.Fee.Currency',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='UNKNOWN_CURRENCY', index=0, number=0,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='LBC', index=1, number=1,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='BTC', index=2, number=2,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='USD', index=3, number=3,
+      serialized_options=None,
+      type=None),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=193,
+  serialized_end=252,
+)
+_sym_db.RegisterEnumDescriptor(_FEE_CURRENCY)
+
+
+_FEE = _descriptor.Descriptor(
+  name='Fee',
+  full_name='legacy_pb.Fee',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='version', full_name='legacy_pb.Fee.version', index=0,
+      number=1, type=14, cpp_type=8, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='currency', full_name='legacy_pb.Fee.currency', index=1,
+      number=2, type=14, cpp_type=8, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='address', full_name='legacy_pb.Fee.address', index=2,
+      number=3, type=12, cpp_type=9, label=2,
+      has_default_value=False, default_value=_b(""),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='amount', full_name='legacy_pb.Fee.amount', index=3,
+      number=4, type=2, cpp_type=6, label=2,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+    _FEE_VERSION,
+    _FEE_CURRENCY,
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=25,
+  serialized_end=252,
+)
+
+_FEE.fields_by_name['version'].enum_type = _FEE_VERSION
+_FEE.fields_by_name['currency'].enum_type = _FEE_CURRENCY
+_FEE_VERSION.containing_type = _FEE
+_FEE_CURRENCY.containing_type = _FEE
+DESCRIPTOR.message_types_by_name['Fee'] = _FEE
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Fee = _reflection.GeneratedProtocolMessageType('Fee', (_message.Message,), dict(
+  DESCRIPTOR = _FEE,
+  __module__ = 'fee_pb2'
+  # @@protoc_insertion_point(class_scope:legacy_pb.Fee)
+  ))
+_sym_db.RegisterMessage(Fee)
+
+
+# @@protoc_insertion_point(module_scope)

scribe/schema/types/v1/legacy_claim_pb2.py

@@ -0,0 +1,158 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: legacy_claim.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from . import stream_pb2 as stream__pb2
+from . import certificate_pb2 as certificate__pb2
+from . import signature_pb2 as signature__pb2
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='legacy_claim.proto',
+  package='legacy_pb',
+  syntax='proto2',
+  serialized_options=None,
+  serialized_pb=_b('\n\x12legacy_claim.proto\x12\tlegacy_pb\x1a\x0cstream.proto\x1a\x11\x63\x65rtificate.proto\x1a\x0fsignature.proto\"\xd9\x02\n\x05\x43laim\x12)\n\x07version\x18\x01 \x02(\x0e\x32\x18.legacy_pb.Claim.Version\x12-\n\tclaimType\x18\x02 \x02(\x0e\x32\x1a.legacy_pb.Claim.ClaimType\x12!\n\x06stream\x18\x03 \x01(\x0b\x32\x11.legacy_pb.Stream\x12+\n\x0b\x63\x65rtificate\x18\x04 \x01(\x0b\x32\x16.legacy_pb.Certificate\x12\x30\n\x12publisherSignature\x18\x05 \x01(\x0b\x32\x14.legacy_pb.Signature\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"H\n\tClaimType\x12\x16\n\x12UNKNOWN_CLAIM_TYPE\x10\x00\x12\x0e\n\nstreamType\x10\x01\x12\x13\n\x0f\x63\x65rtificateType\x10\x02')
+  ,
+  dependencies=[stream__pb2.DESCRIPTOR,certificate__pb2.DESCRIPTOR,signature__pb2.DESCRIPTOR,])
+
+
+
+_CLAIM_VERSION = _descriptor.EnumDescriptor(
+  name='Version',
+  full_name='legacy_pb.Claim.Version',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='UNKNOWN_VERSION', index=0, number=0,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='_0_0_1', index=1, number=1,
+      serialized_options=None,
+      type=None),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=313,
+  serialized_end=355,
+)
+_sym_db.RegisterEnumDescriptor(_CLAIM_VERSION)
+
+_CLAIM_CLAIMTYPE = _descriptor.EnumDescriptor(
+  name='ClaimType',
+  full_name='legacy_pb.Claim.ClaimType',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='UNKNOWN_CLAIM_TYPE', index=0, number=0,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='streamType', index=1, number=1,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='certificateType', index=2, number=2,
+      serialized_options=None,
+      type=None),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=357,
+  serialized_end=429,
+)
+_sym_db.RegisterEnumDescriptor(_CLAIM_CLAIMTYPE)
+
+
+_CLAIM = _descriptor.Descriptor(
+  name='Claim',
+  full_name='legacy_pb.Claim',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='version', full_name='legacy_pb.Claim.version', index=0,
+      number=1, type=14, cpp_type=8, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='claimType', full_name='legacy_pb.Claim.claimType', index=1,
+      number=2, type=14, cpp_type=8, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='stream', full_name='legacy_pb.Claim.stream', index=2,
+      number=3, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='certificate', full_name='legacy_pb.Claim.certificate', index=3,
+      number=4, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='publisherSignature', full_name='legacy_pb.Claim.publisherSignature', index=4,
+      number=5, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+    _CLAIM_VERSION,
+    _CLAIM_CLAIMTYPE,
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=84,
+  serialized_end=429,
+)
+
+_CLAIM.fields_by_name['version'].enum_type = _CLAIM_VERSION
+_CLAIM.fields_by_name['claimType'].enum_type = _CLAIM_CLAIMTYPE
+_CLAIM.fields_by_name['stream'].message_type = stream__pb2._STREAM
+_CLAIM.fields_by_name['certificate'].message_type = certificate__pb2._CERTIFICATE
+_CLAIM.fields_by_name['publisherSignature'].message_type = signature__pb2._SIGNATURE
+_CLAIM_VERSION.containing_type = _CLAIM
+_CLAIM_CLAIMTYPE.containing_type = _CLAIM
+DESCRIPTOR.message_types_by_name['Claim'] = _CLAIM
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Claim = _reflection.GeneratedProtocolMessageType('Claim', (_message.Message,), dict(
+  DESCRIPTOR = _CLAIM,
+  __module__ = 'legacy_claim_pb2'
+  # @@protoc_insertion_point(class_scope:legacy_pb.Claim)
+  ))
+_sym_db.RegisterMessage(Claim)
+
+
+# @@protoc_insertion_point(module_scope)

scribe/schema/types/v1/signature_pb2.py

@@ -0,0 +1,118 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: signature.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from . import certificate_pb2 as certificate__pb2
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='signature.proto',
+  package='legacy_pb',
+  syntax='proto2',
+  serialized_options=None,
+  serialized_pb=_b('\n\x0fsignature.proto\x12\tlegacy_pb\x1a\x11\x63\x65rtificate.proto\"\xbb\x01\n\tSignature\x12-\n\x07version\x18\x01 \x02(\x0e\x32\x1c.legacy_pb.Signature.Version\x12)\n\rsignatureType\x18\x02 \x02(\x0e\x32\x12.legacy_pb.KeyType\x12\x11\n\tsignature\x18\x03 \x02(\x0c\x12\x15\n\rcertificateId\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
+  ,
+  dependencies=[certificate__pb2.DESCRIPTOR,])
+
+
+
+_SIGNATURE_VERSION = _descriptor.EnumDescriptor(
+  name='Version',
+  full_name='legacy_pb.Signature.Version',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='UNKNOWN_VERSION', index=0, number=0,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='_0_0_1', index=1, number=1,
+      serialized_options=None,
+      type=None),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=195,
+  serialized_end=237,
+)
+_sym_db.RegisterEnumDescriptor(_SIGNATURE_VERSION)
+
+
+_SIGNATURE = _descriptor.Descriptor(
+  name='Signature',
+  full_name='legacy_pb.Signature',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='version', full_name='legacy_pb.Signature.version', index=0,
+      number=1, type=14, cpp_type=8, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='signatureType', full_name='legacy_pb.Signature.signatureType', index=1,
+      number=2, type=14, cpp_type=8, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='signature', full_name='legacy_pb.Signature.signature', index=2,
+      number=3, type=12, cpp_type=9, label=2,
+      has_default_value=False, default_value=_b(""),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='certificateId', full_name='legacy_pb.Signature.certificateId', index=3,
+      number=4, type=12, cpp_type=9, label=2,
+      has_default_value=False, default_value=_b(""),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+    _SIGNATURE_VERSION,
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=50,
+  serialized_end=237,
+)
+
+_SIGNATURE.fields_by_name['version'].enum_type = _SIGNATURE_VERSION
+_SIGNATURE.fields_by_name['signatureType'].enum_type = certificate__pb2._KEYTYPE
+_SIGNATURE_VERSION.containing_type = _SIGNATURE
+DESCRIPTOR.message_types_by_name['Signature'] = _SIGNATURE
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Signature = _reflection.GeneratedProtocolMessageType('Signature', (_message.Message,), dict(
+  DESCRIPTOR = _SIGNATURE,
+  __module__ = 'signature_pb2'
+  # @@protoc_insertion_point(class_scope:legacy_pb.Signature)
+  ))
+_sym_db.RegisterMessage(Signature)
+
+
+# @@protoc_insertion_point(module_scope)

scribe/schema/types/v1/source_pb2.py

@@ -0,0 +1,140 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: source.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='source.proto',
+  package='legacy_pb',
+  syntax='proto2',
+  serialized_options=None,
+  serialized_pb=_b('\n\x0csource.proto\x12\tlegacy_pb\"\xf2\x01\n\x06Source\x12*\n\x07version\x18\x01 \x02(\x0e\x32\x19.legacy_pb.Source.Version\x12\x31\n\nsourceType\x18\x02 \x02(\x0e\x32\x1d.legacy_pb.Source.SourceTypes\x12\x0e\n\x06source\x18\x03 \x02(\x0c\x12\x13\n\x0b\x63ontentType\x18\x04 \x02(\t\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"8\n\x0bSourceTypes\x12\x17\n\x13UNKNOWN_SOURCE_TYPE\x10\x00\x12\x10\n\x0clbry_sd_hash\x10\x01')
+)
+
+
+
+_SOURCE_VERSION = _descriptor.EnumDescriptor(
+  name='Version',
+  full_name='legacy_pb.Source.Version',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='UNKNOWN_VERSION', index=0, number=0,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='_0_0_1', index=1, number=1,
+      serialized_options=None,
+      type=None),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=170,
+  serialized_end=212,
+)
+_sym_db.RegisterEnumDescriptor(_SOURCE_VERSION)
+
+_SOURCE_SOURCETYPES = _descriptor.EnumDescriptor(
+  name='SourceTypes',
+  full_name='legacy_pb.Source.SourceTypes',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='UNKNOWN_SOURCE_TYPE', index=0, number=0,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='lbry_sd_hash', index=1, number=1,
+      serialized_options=None,
+      type=None),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=214,
+  serialized_end=270,
+)
+_sym_db.RegisterEnumDescriptor(_SOURCE_SOURCETYPES)
+
+
+_SOURCE = _descriptor.Descriptor(
+  name='Source',
+  full_name='legacy_pb.Source',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='version', full_name='legacy_pb.Source.version', index=0,
+      number=1, type=14, cpp_type=8, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='sourceType', full_name='legacy_pb.Source.sourceType', index=1,
+      number=2, type=14, cpp_type=8, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='source', full_name='legacy_pb.Source.source', index=2,
+      number=3, type=12, cpp_type=9, label=2,
+      has_default_value=False, default_value=_b(""),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='contentType', full_name='legacy_pb.Source.contentType', index=3,
+      number=4, type=9, cpp_type=9, label=2,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+    _SOURCE_VERSION,
+    _SOURCE_SOURCETYPES,
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=28,
+  serialized_end=270,
+)
+
+_SOURCE.fields_by_name['version'].enum_type = _SOURCE_VERSION
+_SOURCE.fields_by_name['sourceType'].enum_type = _SOURCE_SOURCETYPES
+_SOURCE_VERSION.containing_type = _SOURCE
+_SOURCE_SOURCETYPES.containing_type = _SOURCE
+DESCRIPTOR.message_types_by_name['Source'] = _SOURCE
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Source = _reflection.GeneratedProtocolMessageType('Source', (_message.Message,), dict(
+  DESCRIPTOR = _SOURCE,
+  __module__ = 'source_pb2'
+  # @@protoc_insertion_point(class_scope:legacy_pb.Source)
+  ))
+_sym_db.RegisterMessage(Source)
+
+
+# @@protoc_insertion_point(module_scope)

scribe/schema/types/v1/stream_pb2.py

@@ -0,0 +1,113 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: stream.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from . import metadata_pb2 as metadata__pb2
+from . import source_pb2 as source__pb2
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='stream.proto',
+  package='legacy_pb',
+  syntax='proto2',
+  serialized_options=None,
+  serialized_pb=_b('\n\x0cstream.proto\x12\tlegacy_pb\x1a\x0emetadata.proto\x1a\x0csource.proto\"\xaa\x01\n\x06Stream\x12*\n\x07version\x18\x01 \x02(\x0e\x32\x19.legacy_pb.Stream.Version\x12%\n\x08metadata\x18\x02 \x02(\x0b\x32\x13.legacy_pb.Metadata\x12!\n\x06source\x18\x03 \x02(\x0b\x32\x11.legacy_pb.Source\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
+  ,
+  dependencies=[metadata__pb2.DESCRIPTOR,source__pb2.DESCRIPTOR,])
+
+
+
+_STREAM_VERSION = _descriptor.EnumDescriptor(
+  name='Version',
+  full_name='legacy_pb.Stream.Version',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='UNKNOWN_VERSION', index=0, number=0,
+      serialized_options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='_0_0_1', index=1, number=1,
+      serialized_options=None,
+      type=None),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=186,
+  serialized_end=228,
+)
+_sym_db.RegisterEnumDescriptor(_STREAM_VERSION)
+
+
+_STREAM = _descriptor.Descriptor(
+  name='Stream',
+  full_name='legacy_pb.Stream',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='version', full_name='legacy_pb.Stream.version', index=0,
+      number=1, type=14, cpp_type=8, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='metadata', full_name='legacy_pb.Stream.metadata', index=1,
+      number=2, type=11, cpp_type=10, label=2,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='source', full_name='legacy_pb.Stream.source', index=2,
+      number=3, type=11, cpp_type=10, label=2,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+    _STREAM_VERSION,
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=58,
+  serialized_end=228,
+)
+
+_STREAM.fields_by_name['version'].enum_type = _STREAM_VERSION
+_STREAM.fields_by_name['metadata'].message_type = metadata__pb2._METADATA
+_STREAM.fields_by_name['source'].message_type = source__pb2._SOURCE
+_STREAM_VERSION.containing_type = _STREAM
+DESCRIPTOR.message_types_by_name['Stream'] = _STREAM
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Stream = _reflection.GeneratedProtocolMessageType('Stream', (_message.Message,), dict(
+  DESCRIPTOR = _STREAM,
+  __module__ = 'stream_pb2'
+  # @@protoc_insertion_point(class_scope:legacy_pb.Stream)
+  ))
+_sym_db.RegisterMessage(Stream)
+
+
+# @@protoc_insertion_point(module_scope)

scribe/schema/types/v2/hub_pb2.py

@@ -0,0 +1,960 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: hub.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from . import result_pb2 as result__pb2
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='hub.proto',
+  package='pb',
+  syntax='proto3',
+  serialized_options=b'Z$github.com/lbryio/hub/protobuf/go/pb',
+  create_key=_descriptor._internal_create_key,
+  serialized_pb=b'\n\thub.proto\x12\x02pb\x1a\x0cresult.proto\"\x0e\n\x0c\x45mptyMessage\".\n\rServerMessage\x12\x0f\n\x07\x61\x64\x64ress\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\t\"N\n\x0cHelloMessage\x12\x0c\n\x04port\x18\x01 \x01(\t\x12\x0c\n\x04host\x18\x02 \x01(\t\x12\"\n\x07servers\x18\x03 \x03(\x0b\x32\x11.pb.ServerMessage\"0\n\x0fInvertibleField\x12\x0e\n\x06invert\x18\x01 \x01(\x08\x12\r\n\x05value\x18\x02 \x03(\t\"\x1c\n\x0bStringValue\x12\r\n\x05value\x18\x01 \x01(\t\"\x1a\n\tBoolValue\x12\r\n\x05value\x18\x01 \x01(\x08\"\x1c\n\x0bUInt32Value\x12\r\n\x05value\x18\x01 \x01(\r\"j\n\nRangeField\x12\x1d\n\x02op\x18\x01 \x01(\x0e\x32\x11.pb.RangeField.Op\x12\r\n\x05value\x18\x02 \x03(\x05\".\n\x02Op\x12\x06\n\x02\x45Q\x10\x00\x12\x07\n\x03LTE\x10\x01\x12\x07\n\x03GTE\x10\x02\x12\x06\n\x02LT\x10\x03\x12\x06\n\x02GT\x10\x04\"\x8e\x0c\n\rSearchRequest\x12%\n\x08\x63laim_id\x18\x01 \x01(\x0b\x32\x13.pb.InvertibleField\x12\'\n\nchannel_id\x18\x02 \x01(\x0b\x32\x13.pb.InvertibleField\x12\x0c\n\x04text\x18\x03 \x01(\t\x12\r\n\x05limit\x18\x04 \x01(\x05\x12\x10\n\x08order_by\x18\x05 \x03(\t\x12\x0e\n\x06offset\x18\x06 \x01(\r\x12\x16\n\x0eis_controlling\x18\x07 \x01(\x08\x12\x1d\n\x15last_take_over_height\x18\x08 \x01(\t\x12\x12\n\nclaim_name\x18\t \x01(\t\x12\x17\n\x0fnormalized_name\x18\n \x01(\t\x12#\n\x0btx_position\x18\x0b \x03(\x0b\x32\x0e.pb.RangeField\x12\x1e\n\x06\x61mount\x18\x0c \x03(\x0b\x32\x0e.pb.RangeField\x12!\n\ttimestamp\x18\r \x03(\x0b\x32\x0e.pb.RangeField\x12*\n\x12\x63reation_timestamp\x18\x0e \x03(\x0b\x32\x0e.pb.RangeField\x12\x1e\n\x06height\x18\x0f \x03(\x0b\x32\x0e.pb.RangeField\x12\'\n\x0f\x63reation_height\x18\x10 \x03(\x0b\x32\x0e.pb.RangeField\x12)\n\x11\x61\x63tivation_height\x18\x11 \x03(\x0b\x32\x0e.pb.RangeField\x12)\n\x11\x65xpiration_height\x18\x12 \x03(\x0b\x32\x0e.pb.RangeField\x12$\n\x0crelease_time\x18\x13 \x03(\x0b\x32\x0e.pb.RangeField\x12\x11\n\tshort_url\x18\x14 \x01(\t\x12\x15\n\rcanonical_url\x18\x15 \x01(\t\x12\r\n\x05title\x18\x16 \x01(\t\x12\x0e\n\x06\x61uthor\x18\x17 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x18 \x01(\t\x12\x12\n\nclaim_type\x18\x19 \x03(\t\x12$\n\x0crepost_count\x18\x1a \x03(\x0b\x32\x0e.pb.RangeField\x12\x13\n\x0bstream_type\x18\x1b \x03(\t\x12\x12\n\nmedia_type\x18\x1c \x03(\t\x12\"\n\nfee_amount\x18\x1d \x03(\x0b\x32\x0e.pb.RangeField\x12\x14\n\x0c\x66\x65\x65_currency\x18\x1e \x01(\t\x12 \n\x08\x64uration\x18\x1f \x03(\x0b\x32\x0e.pb.RangeField\x12\x19\n\x11reposted_claim_id\x18  \x01(\t\x12#\n\x0b\x63\x65nsor_type\x18! \x03(\x0b\x32\x0e.pb.RangeField\x12\x19\n\x11\x63laims_in_channel\x18\" \x01(\t\x12)\n\x12is_signature_valid\x18$ \x01(\x0b\x32\r.pb.BoolValue\x12(\n\x10\x65\x66\x66\x65\x63tive_amount\x18% \x03(\x0b\x32\x0e.pb.RangeField\x12&\n\x0esupport_amount\x18& \x03(\x0b\x32\x0e.pb.RangeField\x12&\n\x0etrending_score\x18\' \x03(\x0b\x32\x0e.pb.RangeField\x12\r\n\x05tx_id\x18+ \x01(\t\x12 \n\x07tx_nout\x18, \x01(\x0b\x32\x0f.pb.UInt32Value\x12\x11\n\tsignature\x18- \x01(\t\x12\x18\n\x10signature_digest\x18. \x01(\t\x12\x18\n\x10public_key_bytes\x18/ \x01(\t\x12\x15\n\rpublic_key_id\x18\x30 \x01(\t\x12\x10\n\x08\x61ny_tags\x18\x31 \x03(\t\x12\x10\n\x08\x61ll_tags\x18\x32 \x03(\t\x12\x10\n\x08not_tags\x18\x33 \x03(\t\x12\x1d\n\x15has_channel_signature\x18\x34 \x01(\x08\x12!\n\nhas_source\x18\x35 \x01(\x0b\x32\r.pb.BoolValue\x12 \n\x18limit_claims_per_channel\x18\x36 \x01(\x05\x12\x15\n\rany_languages\x18\x37 \x03(\t\x12\x15\n\rall_languages\x18\x38 \x03(\t\x12\x19\n\x11remove_duplicates\x18\x39 \x01(\x08\x12\x11\n\tno_totals\x18: \x01(\x08\x12\x0f\n\x07sd_hash\x18; \x01(\t2\x88\x03\n\x03Hub\x12*\n\x06Search\x12\x11.pb.SearchRequest\x1a\x0b.pb.Outputs\"\x00\x12+\n\x04Ping\x12\x10.pb.EmptyMessage\x1a\x0f.pb.StringValue\"\x00\x12-\n\x05Hello\x12\x10.pb.HelloMessage\x1a\x10.pb.HelloMessage\"\x00\x12/\n\x07\x41\x64\x64Peer\x12\x11.pb.ServerMessage\x1a\x0f.pb.StringValue\"\x00\x12\x35\n\rPeerSubscribe\x12\x11.pb.ServerMessage\x1a\x0f.pb.StringValue\"\x00\x12.\n\x07Version\x12\x10.pb.EmptyMessage\x1a\x0f.pb.StringValue\"\x00\x12/\n\x08\x46\x65\x61tures\x12\x10.pb.EmptyMessage\x1a\x0f.pb.StringValue\"\x00\x12\x30\n\tBroadcast\x12\x10.pb.EmptyMessage\x1a\x0f.pb.UInt32Value\"\x00\x42&Z$github.com/lbryio/hub/protobuf/go/pbb\x06proto3'
+  ,
+  dependencies=[result__pb2.DESCRIPTOR,])
+
+
+
+_RANGEFIELD_OP = _descriptor.EnumDescriptor(
+  name='Op',
+  full_name='pb.RangeField.Op',
+  filename=None,
+  file=DESCRIPTOR,
+  create_key=_descriptor._internal_create_key,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='EQ', index=0, number=0,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+    _descriptor.EnumValueDescriptor(
+      name='LTE', index=1, number=1,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+    _descriptor.EnumValueDescriptor(
+      name='GTE', index=2, number=2,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+    _descriptor.EnumValueDescriptor(
+      name='LT', index=3, number=3,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+    _descriptor.EnumValueDescriptor(
+      name='GT', index=4, number=4,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=373,
+  serialized_end=419,
+)
+_sym_db.RegisterEnumDescriptor(_RANGEFIELD_OP)
+
+
+_EMPTYMESSAGE = _descriptor.Descriptor(
+  name='EmptyMessage',
+  full_name='pb.EmptyMessage',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=31,
+  serialized_end=45,
+)
+
+
+_SERVERMESSAGE = _descriptor.Descriptor(
+  name='ServerMessage',
+  full_name='pb.ServerMessage',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='address', full_name='pb.ServerMessage.address', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='port', full_name='pb.ServerMessage.port', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=47,
+  serialized_end=93,
+)
+
+
+_HELLOMESSAGE = _descriptor.Descriptor(
+  name='HelloMessage',
+  full_name='pb.HelloMessage',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='port', full_name='pb.HelloMessage.port', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='host', full_name='pb.HelloMessage.host', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='servers', full_name='pb.HelloMessage.servers', index=2,
+      number=3, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=95,
+  serialized_end=173,
+)
+
+
+_INVERTIBLEFIELD = _descriptor.Descriptor(
+  name='InvertibleField',
+  full_name='pb.InvertibleField',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='invert', full_name='pb.InvertibleField.invert', index=0,
+      number=1, type=8, cpp_type=7, label=1,
+      has_default_value=False, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='value', full_name='pb.InvertibleField.value', index=1,
+      number=2, type=9, cpp_type=9, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=175,
+  serialized_end=223,
+)
+
+
+_STRINGVALUE = _descriptor.Descriptor(
+  name='StringValue',
+  full_name='pb.StringValue',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='value', full_name='pb.StringValue.value', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=225,
+  serialized_end=253,
+)
+
+
+_BOOLVALUE = _descriptor.Descriptor(
+  name='BoolValue',
+  full_name='pb.BoolValue',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='value', full_name='pb.BoolValue.value', index=0,
+      number=1, type=8, cpp_type=7, label=1,
+      has_default_value=False, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=255,
+  serialized_end=281,
+)
+
+
+_UINT32VALUE = _descriptor.Descriptor(
+  name='UInt32Value',
+  full_name='pb.UInt32Value',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='value', full_name='pb.UInt32Value.value', index=0,
+      number=1, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=283,
+  serialized_end=311,
+)
+
+
+_RANGEFIELD = _descriptor.Descriptor(
+  name='RangeField',
+  full_name='pb.RangeField',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='op', full_name='pb.RangeField.op', index=0,
+      number=1, type=14, cpp_type=8, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='value', full_name='pb.RangeField.value', index=1,
+      number=2, type=5, cpp_type=1, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+    _RANGEFIELD_OP,
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=313,
+  serialized_end=419,
+)
+
+
+_SEARCHREQUEST = _descriptor.Descriptor(
+  name='SearchRequest',
+  full_name='pb.SearchRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='claim_id', full_name='pb.SearchRequest.claim_id', index=0,
+      number=1, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='channel_id', full_name='pb.SearchRequest.channel_id', index=1,
+      number=2, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='text', full_name='pb.SearchRequest.text', index=2,
+      number=3, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='limit', full_name='pb.SearchRequest.limit', index=3,
+      number=4, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='order_by', full_name='pb.SearchRequest.order_by', index=4,
+      number=5, type=9, cpp_type=9, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='offset', full_name='pb.SearchRequest.offset', index=5,
+      number=6, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='is_controlling', full_name='pb.SearchRequest.is_controlling', index=6,
+      number=7, type=8, cpp_type=7, label=1,
+      has_default_value=False, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='last_take_over_height', full_name='pb.SearchRequest.last_take_over_height', index=7,
+      number=8, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='claim_name', full_name='pb.SearchRequest.claim_name', index=8,
+      number=9, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='normalized_name', full_name='pb.SearchRequest.normalized_name', index=9,
+      number=10, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='tx_position', full_name='pb.SearchRequest.tx_position', index=10,
+      number=11, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='amount', full_name='pb.SearchRequest.amount', index=11,
+      number=12, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='timestamp', full_name='pb.SearchRequest.timestamp', index=12,
+      number=13, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='creation_timestamp', full_name='pb.SearchRequest.creation_timestamp', index=13,
+      number=14, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='height', full_name='pb.SearchRequest.height', index=14,
+      number=15, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='creation_height', full_name='pb.SearchRequest.creation_height', index=15,
+      number=16, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='activation_height', full_name='pb.SearchRequest.activation_height', index=16,
+      number=17, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='expiration_height', full_name='pb.SearchRequest.expiration_height', index=17,
+      number=18, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='release_time', full_name='pb.SearchRequest.release_time', index=18,
+      number=19, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='short_url', full_name='pb.SearchRequest.short_url', index=19,
+      number=20, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='canonical_url', full_name='pb.SearchRequest.canonical_url', index=20,
+      number=21, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='title', full_name='pb.SearchRequest.title', index=21,
+      number=22, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='author', full_name='pb.SearchRequest.author', index=22,
+      number=23, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='description', full_name='pb.SearchRequest.description', index=23,
+      number=24, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='claim_type', full_name='pb.SearchRequest.claim_type', index=24,
+      number=25, type=9, cpp_type=9, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='repost_count', full_name='pb.SearchRequest.repost_count', index=25,
+      number=26, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='stream_type', full_name='pb.SearchRequest.stream_type', index=26,
+      number=27, type=9, cpp_type=9, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='media_type', full_name='pb.SearchRequest.media_type', index=27,
+      number=28, type=9, cpp_type=9, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='fee_amount', full_name='pb.SearchRequest.fee_amount', index=28,
+      number=29, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='fee_currency', full_name='pb.SearchRequest.fee_currency', index=29,
+      number=30, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='duration', full_name='pb.SearchRequest.duration', index=30,
+      number=31, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='reposted_claim_id', full_name='pb.SearchRequest.reposted_claim_id', index=31,
+      number=32, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='censor_type', full_name='pb.SearchRequest.censor_type', index=32,
+      number=33, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='claims_in_channel', full_name='pb.SearchRequest.claims_in_channel', index=33,
+      number=34, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='is_signature_valid', full_name='pb.SearchRequest.is_signature_valid', index=34,
+      number=36, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='effective_amount', full_name='pb.SearchRequest.effective_amount', index=35,
+      number=37, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='support_amount', full_name='pb.SearchRequest.support_amount', index=36,
+      number=38, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='trending_score', full_name='pb.SearchRequest.trending_score', index=37,
+      number=39, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='tx_id', full_name='pb.SearchRequest.tx_id', index=38,
+      number=43, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='tx_nout', full_name='pb.SearchRequest.tx_nout', index=39,
+      number=44, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='signature', full_name='pb.SearchRequest.signature', index=40,
+      number=45, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='signature_digest', full_name='pb.SearchRequest.signature_digest', index=41,
+      number=46, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='public_key_bytes', full_name='pb.SearchRequest.public_key_bytes', index=42,
+      number=47, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='public_key_id', full_name='pb.SearchRequest.public_key_id', index=43,
+      number=48, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='any_tags', full_name='pb.SearchRequest.any_tags', index=44,
+      number=49, type=9, cpp_type=9, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='all_tags', full_name='pb.SearchRequest.all_tags', index=45,
+      number=50, type=9, cpp_type=9, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='not_tags', full_name='pb.SearchRequest.not_tags', index=46,
+      number=51, type=9, cpp_type=9, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='has_channel_signature', full_name='pb.SearchRequest.has_channel_signature', index=47,
+      number=52, type=8, cpp_type=7, label=1,
+      has_default_value=False, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='has_source', full_name='pb.SearchRequest.has_source', index=48,
+      number=53, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='limit_claims_per_channel', full_name='pb.SearchRequest.limit_claims_per_channel', index=49,
+      number=54, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='any_languages', full_name='pb.SearchRequest.any_languages', index=50,
+      number=55, type=9, cpp_type=9, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='all_languages', full_name='pb.SearchRequest.all_languages', index=51,
+      number=56, type=9, cpp_type=9, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='remove_duplicates', full_name='pb.SearchRequest.remove_duplicates', index=52,
+      number=57, type=8, cpp_type=7, label=1,
+      has_default_value=False, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='no_totals', full_name='pb.SearchRequest.no_totals', index=53,
+      number=58, type=8, cpp_type=7, label=1,
+      has_default_value=False, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='sd_hash', full_name='pb.SearchRequest.sd_hash', index=54,
+      number=59, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=422,
+  serialized_end=1972,
+)
+
+_HELLOMESSAGE.fields_by_name['servers'].message_type = _SERVERMESSAGE
+_RANGEFIELD.fields_by_name['op'].enum_type = _RANGEFIELD_OP
+_RANGEFIELD_OP.containing_type = _RANGEFIELD
+_SEARCHREQUEST.fields_by_name['claim_id'].message_type = _INVERTIBLEFIELD
+_SEARCHREQUEST.fields_by_name['channel_id'].message_type = _INVERTIBLEFIELD
+_SEARCHREQUEST.fields_by_name['tx_position'].message_type = _RANGEFIELD
+_SEARCHREQUEST.fields_by_name['amount'].message_type = _RANGEFIELD
+_SEARCHREQUEST.fields_by_name['timestamp'].message_type = _RANGEFIELD
+_SEARCHREQUEST.fields_by_name['creation_timestamp'].message_type = _RANGEFIELD
+_SEARCHREQUEST.fields_by_name['height'].message_type = _RANGEFIELD
+_SEARCHREQUEST.fields_by_name['creation_height'].message_type = _RANGEFIELD
+_SEARCHREQUEST.fields_by_name['activation_height'].message_type = _RANGEFIELD
+_SEARCHREQUEST.fields_by_name['expiration_height'].message_type = _RANGEFIELD
+_SEARCHREQUEST.fields_by_name['release_time'].message_type = _RANGEFIELD
+_SEARCHREQUEST.fields_by_name['repost_count'].message_type = _RANGEFIELD
+_SEARCHREQUEST.fields_by_name['fee_amount'].message_type = _RANGEFIELD
+_SEARCHREQUEST.fields_by_name['duration'].message_type = _RANGEFIELD
+_SEARCHREQUEST.fields_by_name['censor_type'].message_type = _RANGEFIELD
+_SEARCHREQUEST.fields_by_name['is_signature_valid'].message_type = _BOOLVALUE
+_SEARCHREQUEST.fields_by_name['effective_amount'].message_type = _RANGEFIELD
+_SEARCHREQUEST.fields_by_name['support_amount'].message_type = _RANGEFIELD
+_SEARCHREQUEST.fields_by_name['trending_score'].message_type = _RANGEFIELD
+_SEARCHREQUEST.fields_by_name['tx_nout'].message_type = _UINT32VALUE
+_SEARCHREQUEST.fields_by_name['has_source'].message_type = _BOOLVALUE
+DESCRIPTOR.message_types_by_name['EmptyMessage'] = _EMPTYMESSAGE
+DESCRIPTOR.message_types_by_name['ServerMessage'] = _SERVERMESSAGE
+DESCRIPTOR.message_types_by_name['HelloMessage'] = _HELLOMESSAGE
+DESCRIPTOR.message_types_by_name['InvertibleField'] = _INVERTIBLEFIELD
+DESCRIPTOR.message_types_by_name['StringValue'] = _STRINGVALUE
+DESCRIPTOR.message_types_by_name['BoolValue'] = _BOOLVALUE
+DESCRIPTOR.message_types_by_name['UInt32Value'] = _UINT32VALUE
+DESCRIPTOR.message_types_by_name['RangeField'] = _RANGEFIELD
+DESCRIPTOR.message_types_by_name['SearchRequest'] = _SEARCHREQUEST
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+EmptyMessage = _reflection.GeneratedProtocolMessageType('EmptyMessage', (_message.Message,), {
+  'DESCRIPTOR' : _EMPTYMESSAGE,
+  '__module__' : 'hub_pb2'
+  # @@protoc_insertion_point(class_scope:pb.EmptyMessage)
+  })
+_sym_db.RegisterMessage(EmptyMessage)
+
+ServerMessage = _reflection.GeneratedProtocolMessageType('ServerMessage', (_message.Message,), {
+  'DESCRIPTOR' : _SERVERMESSAGE,
+  '__module__' : 'hub_pb2'
+  # @@protoc_insertion_point(class_scope:pb.ServerMessage)
+  })
+_sym_db.RegisterMessage(ServerMessage)
+
+HelloMessage = _reflection.GeneratedProtocolMessageType('HelloMessage', (_message.Message,), {
+  'DESCRIPTOR' : _HELLOMESSAGE,
+  '__module__' : 'hub_pb2'
+  # @@protoc_insertion_point(class_scope:pb.HelloMessage)
+  })
+_sym_db.RegisterMessage(HelloMessage)
+
+InvertibleField = _reflection.GeneratedProtocolMessageType('InvertibleField', (_message.Message,), {
+  'DESCRIPTOR' : _INVERTIBLEFIELD,
+  '__module__' : 'hub_pb2'
+  # @@protoc_insertion_point(class_scope:pb.InvertibleField)
+  })
+_sym_db.RegisterMessage(InvertibleField)
+
+StringValue = _reflection.GeneratedProtocolMessageType('StringValue', (_message.Message,), {
+  'DESCRIPTOR' : _STRINGVALUE,
+  '__module__' : 'hub_pb2'
+  # @@protoc_insertion_point(class_scope:pb.StringValue)
+  })
+_sym_db.RegisterMessage(StringValue)
+
+BoolValue = _reflection.GeneratedProtocolMessageType('BoolValue', (_message.Message,), {
+  'DESCRIPTOR' : _BOOLVALUE,
+  '__module__' : 'hub_pb2'
+  # @@protoc_insertion_point(class_scope:pb.BoolValue)
+  })
+_sym_db.RegisterMessage(BoolValue)
+
+UInt32Value = _reflection.GeneratedProtocolMessageType('UInt32Value', (_message.Message,), {
+  'DESCRIPTOR' : _UINT32VALUE,
+  '__module__' : 'hub_pb2'
+  # @@protoc_insertion_point(class_scope:pb.UInt32Value)
+  })
+_sym_db.RegisterMessage(UInt32Value)
+
+RangeField = _reflection.GeneratedProtocolMessageType('RangeField', (_message.Message,), {
+  'DESCRIPTOR' : _RANGEFIELD,
+  '__module__' : 'hub_pb2'
+  # @@protoc_insertion_point(class_scope:pb.RangeField)
+  })
+_sym_db.RegisterMessage(RangeField)
+
+SearchRequest = _reflection.GeneratedProtocolMessageType('SearchRequest', (_message.Message,), {
+  'DESCRIPTOR' : _SEARCHREQUEST,
+  '__module__' : 'hub_pb2'
+  # @@protoc_insertion_point(class_scope:pb.SearchRequest)
+  })
+_sym_db.RegisterMessage(SearchRequest)
+
+
+DESCRIPTOR._options = None
+
+_HUB = _descriptor.ServiceDescriptor(
+  name='Hub',
+  full_name='pb.Hub',
+  file=DESCRIPTOR,
+  index=0,
+  serialized_options=None,
+  create_key=_descriptor._internal_create_key,
+  serialized_start=1975,
+  serialized_end=2367,
+  methods=[
+  _descriptor.MethodDescriptor(
+    name='Search',
+    full_name='pb.Hub.Search',
+    index=0,
+    containing_service=None,
+    input_type=_SEARCHREQUEST,
+    output_type=result__pb2._OUTPUTS,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+  _descriptor.MethodDescriptor(
+    name='Ping',
+    full_name='pb.Hub.Ping',
+    index=1,
+    containing_service=None,
+    input_type=_EMPTYMESSAGE,
+    output_type=_STRINGVALUE,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+  _descriptor.MethodDescriptor(
+    name='Hello',
+    full_name='pb.Hub.Hello',
+    index=2,
+    containing_service=None,
+    input_type=_HELLOMESSAGE,
+    output_type=_HELLOMESSAGE,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+  _descriptor.MethodDescriptor(
+    name='AddPeer',
+    full_name='pb.Hub.AddPeer',
+    index=3,
+    containing_service=None,
+    input_type=_SERVERMESSAGE,
+    output_type=_STRINGVALUE,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+  _descriptor.MethodDescriptor(
+    name='PeerSubscribe',
+    full_name='pb.Hub.PeerSubscribe',
+    index=4,
+    containing_service=None,
+    input_type=_SERVERMESSAGE,
+    output_type=_STRINGVALUE,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+  _descriptor.MethodDescriptor(
+    name='Version',
+    full_name='pb.Hub.Version',
+    index=5,
+    containing_service=None,
+    input_type=_EMPTYMESSAGE,
+    output_type=_STRINGVALUE,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+  _descriptor.MethodDescriptor(
+    name='Features',
+    full_name='pb.Hub.Features',
+    index=6,
+    containing_service=None,
+    input_type=_EMPTYMESSAGE,
+    output_type=_STRINGVALUE,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+  _descriptor.MethodDescriptor(
+    name='Broadcast',
+    full_name='pb.Hub.Broadcast',
+    index=7,
+    containing_service=None,
+    input_type=_EMPTYMESSAGE,
+    output_type=_UINT32VALUE,
+    serialized_options=None,
+    create_key=_descriptor._internal_create_key,
+  ),
+])
+_sym_db.RegisterServiceDescriptor(_HUB)
+
+DESCRIPTOR.services_by_name['Hub'] = _HUB
+
+# @@protoc_insertion_point(module_scope)

scribe/schema/types/v2/hub_pb2_grpc.py

@@ -0,0 +1,298 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+
+from . import hub_pb2 as hub__pb2
+from . import result_pb2 as result__pb2
+
+
+class HubStub(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.Search = channel.unary_unary(
+                '/pb.Hub/Search',
+                request_serializer=hub__pb2.SearchRequest.SerializeToString,
+                response_deserializer=result__pb2.Outputs.FromString,
+                )
+        self.Ping = channel.unary_unary(
+                '/pb.Hub/Ping',
+                request_serializer=hub__pb2.EmptyMessage.SerializeToString,
+                response_deserializer=hub__pb2.StringValue.FromString,
+                )
+        self.Hello = channel.unary_unary(
+                '/pb.Hub/Hello',
+                request_serializer=hub__pb2.HelloMessage.SerializeToString,
+                response_deserializer=hub__pb2.HelloMessage.FromString,
+                )
+        self.AddPeer = channel.unary_unary(
+                '/pb.Hub/AddPeer',
+                request_serializer=hub__pb2.ServerMessage.SerializeToString,
+                response_deserializer=hub__pb2.StringValue.FromString,
+                )
+        self.PeerSubscribe = channel.unary_unary(
+                '/pb.Hub/PeerSubscribe',
+                request_serializer=hub__pb2.ServerMessage.SerializeToString,
+                response_deserializer=hub__pb2.StringValue.FromString,
+                )
+        self.Version = channel.unary_unary(
+                '/pb.Hub/Version',
+                request_serializer=hub__pb2.EmptyMessage.SerializeToString,
+                response_deserializer=hub__pb2.StringValue.FromString,
+                )
+        self.Features = channel.unary_unary(
+                '/pb.Hub/Features',
+                request_serializer=hub__pb2.EmptyMessage.SerializeToString,
+                response_deserializer=hub__pb2.StringValue.FromString,
+                )
+        self.Broadcast = channel.unary_unary(
+                '/pb.Hub/Broadcast',
+                request_serializer=hub__pb2.EmptyMessage.SerializeToString,
+                response_deserializer=hub__pb2.UInt32Value.FromString,
+                )
+
+
+class HubServicer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def Search(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def Ping(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def Hello(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def AddPeer(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def PeerSubscribe(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def Version(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def Features(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def Broadcast(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_HubServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'Search': grpc.unary_unary_rpc_method_handler(
+                    servicer.Search,
+                    request_deserializer=hub__pb2.SearchRequest.FromString,
+                    response_serializer=result__pb2.Outputs.SerializeToString,
+            ),
+            'Ping': grpc.unary_unary_rpc_method_handler(
+                    servicer.Ping,
+                    request_deserializer=hub__pb2.EmptyMessage.FromString,
+                    response_serializer=hub__pb2.StringValue.SerializeToString,
+            ),
+            'Hello': grpc.unary_unary_rpc_method_handler(
+                    servicer.Hello,
+                    request_deserializer=hub__pb2.HelloMessage.FromString,
+                    response_serializer=hub__pb2.HelloMessage.SerializeToString,
+            ),
+            'AddPeer': grpc.unary_unary_rpc_method_handler(
+                    servicer.AddPeer,
+                    request_deserializer=hub__pb2.ServerMessage.FromString,
+                    response_serializer=hub__pb2.StringValue.SerializeToString,
+            ),
+            'PeerSubscribe': grpc.unary_unary_rpc_method_handler(
+                    servicer.PeerSubscribe,
+                    request_deserializer=hub__pb2.ServerMessage.FromString,
+                    response_serializer=hub__pb2.StringValue.SerializeToString,
+            ),
+            'Version': grpc.unary_unary_rpc_method_handler(
+                    servicer.Version,
+                    request_deserializer=hub__pb2.EmptyMessage.FromString,
+                    response_serializer=hub__pb2.StringValue.SerializeToString,
+            ),
+            'Features': grpc.unary_unary_rpc_method_handler(
+                    servicer.Features,
+                    request_deserializer=hub__pb2.EmptyMessage.FromString,
+                    response_serializer=hub__pb2.StringValue.SerializeToString,
+            ),
+            'Broadcast': grpc.unary_unary_rpc_method_handler(
+                    servicer.Broadcast,
+                    request_deserializer=hub__pb2.EmptyMessage.FromString,
+                    response_serializer=hub__pb2.UInt32Value.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'pb.Hub', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+ # This class is part of an EXPERIMENTAL API.
+class Hub(object):
+    """Missing associated documentation comment in .proto file."""
+
+    @staticmethod
+    def Search(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/pb.Hub/Search',
+            hub__pb2.SearchRequest.SerializeToString,
+            result__pb2.Outputs.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def Ping(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/pb.Hub/Ping',
+            hub__pb2.EmptyMessage.SerializeToString,
+            hub__pb2.StringValue.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def Hello(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/pb.Hub/Hello',
+            hub__pb2.HelloMessage.SerializeToString,
+            hub__pb2.HelloMessage.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def AddPeer(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/pb.Hub/AddPeer',
+            hub__pb2.ServerMessage.SerializeToString,
+            hub__pb2.StringValue.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def PeerSubscribe(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/pb.Hub/PeerSubscribe',
+            hub__pb2.ServerMessage.SerializeToString,
+            hub__pb2.StringValue.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def Version(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/pb.Hub/Version',
+            hub__pb2.EmptyMessage.SerializeToString,
+            hub__pb2.StringValue.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def Features(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/pb.Hub/Features',
+            hub__pb2.EmptyMessage.SerializeToString,
+            hub__pb2.StringValue.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def Broadcast(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/pb.Hub/Broadcast',
+            hub__pb2.EmptyMessage.SerializeToString,
+            hub__pb2.UInt32Value.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

scribe/schema/types/v2/purchase_pb2.py

@@ -0,0 +1,69 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: purchase.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf import descriptor_pb2
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='purchase.proto',
+  package='pb',
+  syntax='proto3',
+  serialized_pb=_b('\n\x0epurchase.proto\x12\x02pb\"\x1e\n\x08Purchase\x12\x12\n\nclaim_hash\x18\x01 \x01(\x0c\x62\x06proto3')
+)
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+
+
+
+_PURCHASE = _descriptor.Descriptor(
+  name='Purchase',
+  full_name='pb.Purchase',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='claim_hash', full_name='pb.Purchase.claim_hash', index=0,
+      number=1, type=12, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b(""),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=22,
+  serialized_end=52,
+)
+
+DESCRIPTOR.message_types_by_name['Purchase'] = _PURCHASE
+
+Purchase = _reflection.GeneratedProtocolMessageType('Purchase', (_message.Message,), dict(
+  DESCRIPTOR = _PURCHASE,
+  __module__ = 'purchase_pb2'
+  # @@protoc_insertion_point(class_scope:pb.Purchase)
+  ))
+_sym_db.RegisterMessage(Purchase)
+
+
+# @@protoc_insertion_point(module_scope)

scribe/schema/types/v2/result_pb2.py

@@ -0,0 +1,464 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: result.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='result.proto',
+  package='pb',
+  syntax='proto3',
+  serialized_options=b'Z$github.com/lbryio/hub/protobuf/go/pb',
+  create_key=_descriptor._internal_create_key,
+  serialized_pb=b'\n\x0cresult.proto\x12\x02pb\"\x97\x01\n\x07Outputs\x12\x18\n\x04txos\x18\x01 \x03(\x0b\x32\n.pb.Output\x12\x1e\n\nextra_txos\x18\x02 \x03(\x0b\x32\n.pb.Output\x12\r\n\x05total\x18\x03 \x01(\r\x12\x0e\n\x06offset\x18\x04 \x01(\r\x12\x1c\n\x07\x62locked\x18\x05 \x03(\x0b\x32\x0b.pb.Blocked\x12\x15\n\rblocked_total\x18\x06 \x01(\r\"{\n\x06Output\x12\x0f\n\x07tx_hash\x18\x01 \x01(\x0c\x12\x0c\n\x04nout\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\x1e\n\x05\x63laim\x18\x07 \x01(\x0b\x32\r.pb.ClaimMetaH\x00\x12\x1a\n\x05\x65rror\x18\x0f \x01(\x0b\x32\t.pb.ErrorH\x00\x42\x06\n\x04meta\"\xe6\x02\n\tClaimMeta\x12\x1b\n\x07\x63hannel\x18\x01 \x01(\x0b\x32\n.pb.Output\x12\x1a\n\x06repost\x18\x02 \x01(\x0b\x32\n.pb.Output\x12\x11\n\tshort_url\x18\x03 \x01(\t\x12\x15\n\rcanonical_url\x18\x04 \x01(\t\x12\x16\n\x0eis_controlling\x18\x05 \x01(\x08\x12\x18\n\x10take_over_height\x18\x06 \x01(\r\x12\x17\n\x0f\x63reation_height\x18\x07 \x01(\r\x12\x19\n\x11\x61\x63tivation_height\x18\x08 \x01(\r\x12\x19\n\x11\x65xpiration_height\x18\t \x01(\r\x12\x19\n\x11\x63laims_in_channel\x18\n \x01(\r\x12\x10\n\x08reposted\x18\x0b \x01(\r\x12\x18\n\x10\x65\x66\x66\x65\x63tive_amount\x18\x14 \x01(\x04\x12\x16\n\x0esupport_amount\x18\x15 \x01(\x04\x12\x16\n\x0etrending_score\x18\x16 \x01(\x01\"\x94\x01\n\x05\x45rror\x12\x1c\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x0e.pb.Error.Code\x12\x0c\n\x04text\x18\x02 \x01(\t\x12\x1c\n\x07\x62locked\x18\x03 \x01(\x0b\x32\x0b.pb.Blocked\"A\n\x04\x43ode\x12\x10\n\x0cUNKNOWN_CODE\x10\x00\x12\r\n\tNOT_FOUND\x10\x01\x12\x0b\n\x07INVALID\x10\x02\x12\x0b\n\x07\x42LOCKED\x10\x03\"5\n\x07\x42locked\x12\r\n\x05\x63ount\x18\x01 \x01(\r\x12\x1b\n\x07\x63hannel\x18\x02 \x01(\x0b\x32\n.pb.OutputB&Z$github.com/lbryio/hub/protobuf/go/pbb\x06proto3'
+)
+
+
+
+_ERROR_CODE = _descriptor.EnumDescriptor(
+  name='Code',
+  full_name='pb.Error.Code',
+  filename=None,
+  file=DESCRIPTOR,
+  create_key=_descriptor._internal_create_key,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='UNKNOWN_CODE', index=0, number=0,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+    _descriptor.EnumValueDescriptor(
+      name='NOT_FOUND', index=1, number=1,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+    _descriptor.EnumValueDescriptor(
+      name='INVALID', index=2, number=2,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+    _descriptor.EnumValueDescriptor(
+      name='BLOCKED', index=3, number=3,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=744,
+  serialized_end=809,
+)
+_sym_db.RegisterEnumDescriptor(_ERROR_CODE)
+
+
+_OUTPUTS = _descriptor.Descriptor(
+  name='Outputs',
+  full_name='pb.Outputs',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='txos', full_name='pb.Outputs.txos', index=0,
+      number=1, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='extra_txos', full_name='pb.Outputs.extra_txos', index=1,
+      number=2, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='total', full_name='pb.Outputs.total', index=2,
+      number=3, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='offset', full_name='pb.Outputs.offset', index=3,
+      number=4, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='blocked', full_name='pb.Outputs.blocked', index=4,
+      number=5, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='blocked_total', full_name='pb.Outputs.blocked_total', index=5,
+      number=6, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=21,
+  serialized_end=172,
+)
+
+
+_OUTPUT = _descriptor.Descriptor(
+  name='Output',
+  full_name='pb.Output',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='tx_hash', full_name='pb.Output.tx_hash', index=0,
+      number=1, type=12, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"",
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='nout', full_name='pb.Output.nout', index=1,
+      number=2, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='height', full_name='pb.Output.height', index=2,
+      number=3, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='claim', full_name='pb.Output.claim', index=3,
+      number=7, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='error', full_name='pb.Output.error', index=4,
+      number=15, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+    _descriptor.OneofDescriptor(
+      name='meta', full_name='pb.Output.meta',
+      index=0, containing_type=None,
+      create_key=_descriptor._internal_create_key,
+    fields=[]),
+  ],
+  serialized_start=174,
+  serialized_end=297,
+)
+
+
+_CLAIMMETA = _descriptor.Descriptor(
+  name='ClaimMeta',
+  full_name='pb.ClaimMeta',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='channel', full_name='pb.ClaimMeta.channel', index=0,
+      number=1, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='repost', full_name='pb.ClaimMeta.repost', index=1,
+      number=2, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='short_url', full_name='pb.ClaimMeta.short_url', index=2,
+      number=3, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='canonical_url', full_name='pb.ClaimMeta.canonical_url', index=3,
+      number=4, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='is_controlling', full_name='pb.ClaimMeta.is_controlling', index=4,
+      number=5, type=8, cpp_type=7, label=1,
+      has_default_value=False, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='take_over_height', full_name='pb.ClaimMeta.take_over_height', index=5,
+      number=6, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='creation_height', full_name='pb.ClaimMeta.creation_height', index=6,
+      number=7, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='activation_height', full_name='pb.ClaimMeta.activation_height', index=7,
+      number=8, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='expiration_height', full_name='pb.ClaimMeta.expiration_height', index=8,
+      number=9, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='claims_in_channel', full_name='pb.ClaimMeta.claims_in_channel', index=9,
+      number=10, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='reposted', full_name='pb.ClaimMeta.reposted', index=10,
+      number=11, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='effective_amount', full_name='pb.ClaimMeta.effective_amount', index=11,
+      number=20, type=4, cpp_type=4, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='support_amount', full_name='pb.ClaimMeta.support_amount', index=12,
+      number=21, type=4, cpp_type=4, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='trending_score', full_name='pb.ClaimMeta.trending_score', index=13,
+      number=22, type=1, cpp_type=5, label=1,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=300,
+  serialized_end=658,
+)
+
+
+_ERROR = _descriptor.Descriptor(
+  name='Error',
+  full_name='pb.Error',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='code', full_name='pb.Error.code', index=0,
+      number=1, type=14, cpp_type=8, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='text', full_name='pb.Error.text', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='blocked', full_name='pb.Error.blocked', index=2,
+      number=3, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+    _ERROR_CODE,
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=661,
+  serialized_end=809,
+)
+
+
+_BLOCKED = _descriptor.Descriptor(
+  name='Blocked',
+  full_name='pb.Blocked',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='count', full_name='pb.Blocked.count', index=0,
+      number=1, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='channel', full_name='pb.Blocked.channel', index=1,
+      number=2, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=811,
+  serialized_end=864,
+)
+
+_OUTPUTS.fields_by_name['txos'].message_type = _OUTPUT
+_OUTPUTS.fields_by_name['extra_txos'].message_type = _OUTPUT
+_OUTPUTS.fields_by_name['blocked'].message_type = _BLOCKED
+_OUTPUT.fields_by_name['claim'].message_type = _CLAIMMETA
+_OUTPUT.fields_by_name['error'].message_type = _ERROR
+_OUTPUT.oneofs_by_name['meta'].fields.append(
+  _OUTPUT.fields_by_name['claim'])
+_OUTPUT.fields_by_name['claim'].containing_oneof = _OUTPUT.oneofs_by_name['meta']
+_OUTPUT.oneofs_by_name['meta'].fields.append(
+  _OUTPUT.fields_by_name['error'])
+_OUTPUT.fields_by_name['error'].containing_oneof = _OUTPUT.oneofs_by_name['meta']
+_CLAIMMETA.fields_by_name['channel'].message_type = _OUTPUT
+_CLAIMMETA.fields_by_name['repost'].message_type = _OUTPUT
+_ERROR.fields_by_name['code'].enum_type = _ERROR_CODE
+_ERROR.fields_by_name['blocked'].message_type = _BLOCKED
+_ERROR_CODE.containing_type = _ERROR
+_BLOCKED.fields_by_name['channel'].message_type = _OUTPUT
+DESCRIPTOR.message_types_by_name['Outputs'] = _OUTPUTS
+DESCRIPTOR.message_types_by_name['Output'] = _OUTPUT
+DESCRIPTOR.message_types_by_name['ClaimMeta'] = _CLAIMMETA
+DESCRIPTOR.message_types_by_name['Error'] = _ERROR
+DESCRIPTOR.message_types_by_name['Blocked'] = _BLOCKED
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Outputs = _reflection.GeneratedProtocolMessageType('Outputs', (_message.Message,), {
+  'DESCRIPTOR' : _OUTPUTS,
+  '__module__' : 'result_pb2'
+  # @@protoc_insertion_point(class_scope:pb.Outputs)
+  })
+_sym_db.RegisterMessage(Outputs)
+
+Output = _reflection.GeneratedProtocolMessageType('Output', (_message.Message,), {
+  'DESCRIPTOR' : _OUTPUT,
+  '__module__' : 'result_pb2'
+  # @@protoc_insertion_point(class_scope:pb.Output)
+  })
+_sym_db.RegisterMessage(Output)
+
+ClaimMeta = _reflection.GeneratedProtocolMessageType('ClaimMeta', (_message.Message,), {
+  'DESCRIPTOR' : _CLAIMMETA,
+  '__module__' : 'result_pb2'
+  # @@protoc_insertion_point(class_scope:pb.ClaimMeta)
+  })
+_sym_db.RegisterMessage(ClaimMeta)
+
+Error = _reflection.GeneratedProtocolMessageType('Error', (_message.Message,), {
+  'DESCRIPTOR' : _ERROR,
+  '__module__' : 'result_pb2'
+  # @@protoc_insertion_point(class_scope:pb.Error)
+  })
+_sym_db.RegisterMessage(Error)
+
+Blocked = _reflection.GeneratedProtocolMessageType('Blocked', (_message.Message,), {
+  'DESCRIPTOR' : _BLOCKED,
+  '__module__' : 'result_pb2'
+  # @@protoc_insertion_point(class_scope:pb.Blocked)
+  })
+_sym_db.RegisterMessage(Blocked)
+
+
+DESCRIPTOR._options = None
+# @@protoc_insertion_point(module_scope)

scribe/schema/types/v2/result_pb2_grpc.py

@@ -0,0 +1,4 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+

scribe/schema/types/v2/support_pb2.py

@@ -0,0 +1,76 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: support.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf import descriptor_pb2
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='support.proto',
+  package='pb',
+  syntax='proto3',
+  serialized_pb=_b('\n\rsupport.proto\x12\x02pb\")\n\x07Support\x12\r\n\x05\x65moji\x18\x01 \x01(\t\x12\x0f\n\x07\x63omment\x18\x02 \x01(\tb\x06proto3')
+)
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+
+
+
+_SUPPORT = _descriptor.Descriptor(
+  name='Support',
+  full_name='pb.Support',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='emoji', full_name='pb.Support.emoji', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='comment', full_name='pb.Support.comment', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=21,
+  serialized_end=62,
+)
+
+DESCRIPTOR.message_types_by_name['Support'] = _SUPPORT
+
+Support = _reflection.GeneratedProtocolMessageType('Support', (_message.Message,), dict(
+  DESCRIPTOR = _SUPPORT,
+  __module__ = 'support_pb2'
+  # @@protoc_insertion_point(class_scope:pb.Support)
+  ))
+_sym_db.RegisterMessage(Support)
+
+
+# @@protoc_insertion_point(module_scope)

scribe/schema/url.py

@@ -0,0 +1,130 @@
+import re
+import unicodedata
+from typing import NamedTuple, Tuple
+
+
+def _create_url_regex():
+    # see https://spec.lbry.com/ and test_url.py
+    invalid_names_regex = \
+        r"[^=&#:$@%?;\"/\\<>%{}|^~`\[\]" \
+        r"\u0000-\u0020\uD800-\uDFFF\uFFFE-\uFFFF]+"
+
+    def _named(name, regex):
+        return "(?P<" + name + ">" + regex + ")"
+
+    def _group(regex):
+        return "(?:" + regex + ")"
+
+    def _oneof(*choices):
+        return _group('|'.join(choices))
+
+    def _claim(name, prefix=""):
+        return _group(
+            _named(name+"_name", prefix + invalid_names_regex) +
+            _oneof(
+                _group('[:#]' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
+                _group(r'\$' + _named(name+"_amount_order", '[1-9][0-9]*'))
+            ) + '?'
+        )
+
+    return (
+        '^' +
+        _named("scheme", "lbry://") + '?' +
+        _oneof(
+            _group(_claim("channel_with_stream", "@") + "/" + _claim("stream_in_channel")),
+            _claim("channel", "@"),
+            _claim("stream")
+        ) +
+        '$'
+    )
+
+
+URL_REGEX = _create_url_regex()
+
+
+def normalize_name(name):
+    return unicodedata.normalize('NFD', name).casefold()
+
+
+class PathSegment(NamedTuple):
+    name: str
+    claim_id: str = None
+    amount_order: int = None
+
+    @property
+    def normalized(self):
+        return normalize_name(self.name)
+
+    @property
+    def is_shortid(self):
+        return self.claim_id is not None and len(self.claim_id) < 40
+
+    @property
+    def is_fullid(self):
+        return self.claim_id is not None and len(self.claim_id) == 40
+
+    def to_dict(self):
+        q = {'name': self.name}
+        if self.claim_id is not None:
+            q['claim_id'] = self.claim_id
+        if self.amount_order is not None:
+            q['amount_order'] = self.amount_order
+        return q
+
+    def __str__(self):
+        if self.claim_id is not None:
+            return f"{self.name}:{self.claim_id}"
+        elif self.amount_order is not None:
+            return f"{self.name}${self.amount_order}"
+        return self.name
+
+
+class URL(NamedTuple):
+    stream: PathSegment
+    channel: PathSegment
+
+    @property
+    def has_channel(self):
+        return self.channel is not None
+
+    @property
+    def has_stream(self):
+        return self.stream is not None
+
+    @property
+    def has_stream_in_channel(self):
+        return self.has_channel and self.has_stream
+
+    @property
+    def parts(self) -> Tuple:
+        if self.has_stream_in_channel:
+            return self.channel, self.stream
+        if self.has_channel:
+            return self.channel,
+        return self.stream,
+
+    def __str__(self):
+        return f"lbry://{'/'.join(str(p) for p in self.parts)}"
+
+    @classmethod
+    def parse(cls, url):
+        match = re.match(URL_REGEX, url)
+
+        if match is None:
+            raise ValueError('Invalid LBRY URL')
+
+        segments = {}
+        parts = match.groupdict()
+        for segment in ('channel', 'stream', 'channel_with_stream', 'stream_in_channel'):
+            if parts[f'{segment}_name'] is not None:
+                segments[segment] = PathSegment(
+                    parts[f'{segment}_name'],
+                    parts[f'{segment}_claim_id'],
+                    parts[f'{segment}_amount_order']
+                )
+
+        if 'channel_with_stream' in segments:
+            segments['channel'] = segments['channel_with_stream']
+            segments['stream'] = segments['stream_in_channel']
+
+        return cls(segments.get('stream', None), segments.get('channel', None))

setup.py

@@ -0,0 +1,69 @@
+import os
+from scribe import __name__, __version__
+from setuptools import setup, find_packages
+
+BASE = os.path.dirname(__file__)
+with open(os.path.join(BASE, 'README.md'), encoding='utf-8') as fh:
+    long_description = fh.read()
+
+
+setup(
+    name=__name__,
+    version=__version__,
+    author="LBRY Inc.",
+    author_email="hello@lbry.com",
+    url="https://lbry.com",
+    description="A decentralized media library and marketplace",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    keywords="lbry protocol electrum spv",
+    license='MIT',
+    python_requires='>=3.7',
+    packages=find_packages(exclude=('tests',)),
+    zip_safe=False,
+    entry_points={
+        'console_scripts': [
+            'scribe=scribe.cli:run_writer_forever',
+            'scribe-hub=scribe.cli:run_server_forever',
+            'scribe-elastic-sync=scribe.cli:run_es_sync_forever',
+        ],
+    },
+    install_requires=[
+        'aiohttp==3.5.4',
+        'certifi>=2018.11.29',
+        # 'colorama==0.3.7',
+        # 'distro==1.4.0',
+        'base58==1.0.0',
+        'cffi==1.13.2',
+        'cryptography==2.5',
+        'protobuf==3.17.2',
+        'msgpack==0.6.1',
+        'prometheus_client==0.7.1',
+        'ecdsa==0.13.3',
+        'pyyaml==5.3.1',
+        'prometheus_client==0.7.1',
+        'coincurve==15.0.0',
+        'pbkdf2==1.3',
+        'attrs==18.2.0',
+        'elasticsearch==7.10.1',
+        'lbry-rocksdb==0.8.2',
+        'uvloop'
+    ],
+    extras_require={
+        'lint': ['pylint==2.10.0'],
+        'test': ['coverage'],
+    },
+    classifiers=[
+        'Framework :: AsyncIO',
+        'Intended Audience :: Developers',
+        'Intended Audience :: System Administrators',
+        'License :: OSI Approved :: MIT License',
+        'Programming Language :: Python :: 3',
+        'Operating System :: OS Independent',
+        'Topic :: Internet',
+        'Topic :: Software Development :: Testing',
+        'Topic :: Software Development :: Libraries :: Python Modules',
+        'Topic :: System :: Distributed Computing',
+        'Topic :: Utilities',
+    ],
+)

tests/test_revertable.py

@@ -0,0 +1,237 @@
+import unittest
+import tempfile
+import shutil
+from scribe.db.revertable import RevertableOpStack, RevertableDelete, RevertablePut, OpStackIntegrity
+from scribe.db.prefixes import ClaimToTXOPrefixRow, PrefixDB
+
+
+class TestRevertableOpStack(unittest.TestCase):
+    def setUp(self):
+        self.fake_db = {}
+        self.stack = RevertableOpStack(self.fake_db.get)
+
+    def tearDown(self) -> None:
+        self.stack.clear()
+        self.fake_db.clear()
+
+    def process_stack(self):
+        for op in self.stack:
+            if op.is_put:
+                self.fake_db[op.key] = op.value
+            else:
+                self.fake_db.pop(op.key)
+        self.stack.clear()
+
+    def update(self, key1: bytes, value1: bytes, key2: bytes, value2: bytes):
+        self.stack.append_op(RevertableDelete(key1, value1))
+        self.stack.append_op(RevertablePut(key2, value2))
+
+    def test_simplify(self):
+        key1 = ClaimToTXOPrefixRow.pack_key(b'\x01' * 20)
+        key2 = ClaimToTXOPrefixRow.pack_key(b'\x02' * 20)
+        key3 = ClaimToTXOPrefixRow.pack_key(b'\x03' * 20)
+        key4 = ClaimToTXOPrefixRow.pack_key(b'\x04' * 20)
+
+        val1 = ClaimToTXOPrefixRow.pack_value(1, 0, 1, 0, 1, False, 'derp')
+        val2 = ClaimToTXOPrefixRow.pack_value(1, 0, 1, 0, 1, False, 'oops')
+        val3 = ClaimToTXOPrefixRow.pack_value(1, 0, 1, 0, 1, False, 'other')
+
+        # check that we can't delete a non existent value
+        with self.assertRaises(OpStackIntegrity):
+            self.stack.append_op(RevertableDelete(key1, val1))
+
+        self.stack.append_op(RevertablePut(key1, val1))
+        self.assertEqual(1, len(self.stack))
+        self.stack.append_op(RevertableDelete(key1, val1))
+        self.assertEqual(0, len(self.stack))
+
+        self.stack.append_op(RevertablePut(key1, val1))
+        self.assertEqual(1, len(self.stack))
+        # try to delete the wrong value
+        with self.assertRaises(OpStackIntegrity):
+            self.stack.append_op(RevertableDelete(key2, val2))
+
+        self.stack.append_op(RevertableDelete(key1, val1))
+        self.assertEqual(0, len(self.stack))
+        self.stack.append_op(RevertablePut(key2, val3))
+        self.assertEqual(1, len(self.stack))
+
+        self.process_stack()
+
+        self.assertDictEqual({key2: val3}, self.fake_db)
+
+        # check that we can't put on top of the existing stored value
+        with self.assertRaises(OpStackIntegrity):
+            self.stack.append_op(RevertablePut(key2, val1))
+
+        self.assertEqual(0, len(self.stack))
+        self.stack.append_op(RevertableDelete(key2, val3))
+        self.assertEqual(1, len(self.stack))
+        self.stack.append_op(RevertablePut(key2, val3))
+        self.assertEqual(0, len(self.stack))
+
+        self.update(key2, val3, key2, val1)
+        self.assertEqual(2, len(self.stack))
+
+        self.process_stack()
+        self.assertDictEqual({key2: val1}, self.fake_db)
+
+        self.update(key2, val1, key2, val2)
+        self.assertEqual(2, len(self.stack))
+        self.update(key2, val2, key2, val3)
+        self.update(key2, val3, key2, val2)
+        self.update(key2, val2, key2, val3)
+        self.update(key2, val3, key2, val2)
+        with self.assertRaises(OpStackIntegrity):
+            self.update(key2, val3, key2, val2)
+        self.update(key2, val2, key2, val3)
+        self.assertEqual(2, len(self.stack))
+        self.stack.append_op(RevertableDelete(key2, val3))
+        self.process_stack()
+        self.assertDictEqual({}, self.fake_db)
+
+        self.stack.append_op(RevertablePut(key2, val3))
+        self.process_stack()
+        with self.assertRaises(OpStackIntegrity):
+            self.update(key2, val2, key2, val2)
+        self.update(key2, val3, key2, val2)
+        self.assertDictEqual({key2: val3}, self.fake_db)
+        undo = self.stack.get_undo_ops()
+        self.process_stack()
+        self.assertDictEqual({key2: val2}, self.fake_db)
+        self.stack.apply_packed_undo_ops(undo)
+        self.process_stack()
+        self.assertDictEqual({key2: val3}, self.fake_db)
+
+
+class TestRevertablePrefixDB(unittest.TestCase):
+    def setUp(self):
+        self.tmp_dir = tempfile.mkdtemp()
+        self.db = PrefixDB(self.tmp_dir, cache_mb=1, max_open_files=32)
+
+    def tearDown(self) -> None:
+        self.db.close()
+        shutil.rmtree(self.tmp_dir)
+
+    def test_rollback(self):
+        name = 'derp'
+        claim_hash1 = 20 * b'\x00'
+        claim_hash2 = 20 * b'\x01'
+        claim_hash3 = 20 * b'\x02'
+
+        takeover_height = 10000000
+
+        self.assertIsNone(self.db.claim_takeover.get(name))
+        self.db.claim_takeover.stage_put((name,), (claim_hash1, takeover_height))
+        self.assertIsNone(self.db.claim_takeover.get(name))
+        self.assertEqual(10000000, self.db.claim_takeover.get_pending(name).height)
+
+        self.db.commit(10000000, b'\x00' * 32)
+        self.assertEqual(10000000, self.db.claim_takeover.get(name).height)
+
+        self.db.claim_takeover.stage_delete((name,), (claim_hash1, takeover_height))
+        self.db.claim_takeover.stage_put((name,), (claim_hash2, takeover_height + 1))
+        self.db.claim_takeover.stage_delete((name,), (claim_hash2, takeover_height + 1))
+        self.db.commit(10000001, b'\x01' * 32)
+        self.assertIsNone(self.db.claim_takeover.get(name))
+        self.db.claim_takeover.stage_put((name,), (claim_hash3, takeover_height + 2))
+        self.db.commit(10000002, b'\x02' * 32)
+        self.assertEqual(10000002, self.db.claim_takeover.get(name).height)
+
+        self.db.claim_takeover.stage_delete((name,), (claim_hash3, takeover_height + 2))
+        self.db.claim_takeover.stage_put((name,), (claim_hash2, takeover_height + 3))
+        self.db.commit(10000003, b'\x03' * 32)
+        self.assertEqual(10000003, self.db.claim_takeover.get(name).height)
+
+        self.db.rollback(10000003, b'\x03' * 32)
+        self.assertEqual(10000002, self.db.claim_takeover.get(name).height)
+        self.db.rollback(10000002, b'\x02' * 32)
+        self.assertIsNone(self.db.claim_takeover.get(name))
+        self.db.rollback(10000001, b'\x01' * 32)
+        self.assertEqual(10000000, self.db.claim_takeover.get(name).height)
+        self.db.rollback(10000000, b'\x00' * 32)
+        self.assertIsNone(self.db.claim_takeover.get(name))
+
+    def test_hub_db_iterator(self):
+        name = 'derp'
+        claim_hash0 = 20 * b'\x00'
+        claim_hash1 = 20 * b'\x01'
+        claim_hash2 = 20 * b'\x02'
+        claim_hash3 = 20 * b'\x03'
+        overflow_value = 0xffffffff
+        self.db.claim_expiration.stage_put((99, 999, 0), (claim_hash0, name))
+        self.db.claim_expiration.stage_put((100, 1000, 0), (claim_hash1, name))
+        self.db.claim_expiration.stage_put((100, 1001, 0), (claim_hash2, name))
+        self.db.claim_expiration.stage_put((101, 1002, 0), (claim_hash3, name))
+        self.db.claim_expiration.stage_put((overflow_value - 1, 1003, 0), (claim_hash3, name))
+        self.db.claim_expiration.stage_put((overflow_value, 1004, 0), (claim_hash3, name))
+        self.db.tx_num.stage_put((b'\x00' * 32,), (101,))
+        self.db.claim_takeover.stage_put((name,), (claim_hash3, 101))
+        self.db.db_state.stage_put((), (b'n?\xcf\x12\x99\xd4\xec]y\xc3\xa4\xc9\x1dbJJ\xcf\x9e.\x17=\x95\xa1\xa0POgvihuV', 0, 1, b'VuhivgOP\xa0\xa1\x95=\x17.\x9e\xcfJJb\x1d\xc9\xa4\xc3y]\xec\xd4\x99\x12\xcf?n', 1, 0, 1, 7, 1, -1, -1, 0))
+        self.db.unsafe_commit()
+
+        state = self.db.db_state.get()
+        self.assertEqual(b'n?\xcf\x12\x99\xd4\xec]y\xc3\xa4\xc9\x1dbJJ\xcf\x9e.\x17=\x95\xa1\xa0POgvihuV', state.genesis)
+
+        self.assertListEqual(
+            [], list(self.db.claim_expiration.iterate(prefix=(98,)))
+        )
+        self.assertListEqual(
+            list(self.db.claim_expiration.iterate(start=(98,), stop=(99,))),
+            list(self.db.claim_expiration.iterate(prefix=(98,)))
+        )
+        self.assertListEqual(
+            list(self.db.claim_expiration.iterate(start=(99,), stop=(100,))),
+            list(self.db.claim_expiration.iterate(prefix=(99,)))
+        )
+        self.assertListEqual(
+            [
+                ((99, 999, 0), (claim_hash0, name)),
+            ], list(self.db.claim_expiration.iterate(prefix=(99,)))
+        )
+        self.assertListEqual(
+            [
+                ((100, 1000, 0), (claim_hash1, name)),
+                ((100, 1001, 0), (claim_hash2, name))
+            ], list(self.db.claim_expiration.iterate(prefix=(100,)))
+        )
+        self.assertListEqual(
+            list(self.db.claim_expiration.iterate(start=(100,), stop=(101,))),
+            list(self.db.claim_expiration.iterate(prefix=(100,)))
+        )
+        self.assertListEqual(
+            [
+                ((overflow_value - 1, 1003, 0), (claim_hash3, name))
+            ], list(self.db.claim_expiration.iterate(prefix=(overflow_value - 1,)))
+        )
+        self.assertListEqual(
+            [
+                ((overflow_value, 1004, 0), (claim_hash3, name))
+            ], list(self.db.claim_expiration.iterate(prefix=(overflow_value,)))
+        )
+
+    def test_hub_db_iterator_start_stop(self):
+        tx_num = 101
+        for x in range(255):
+            claim_hash = 20 * chr(x).encode()
+            self.db.active_amount.stage_put((claim_hash, 1, 200, tx_num, 1), (100000,))
+            self.db.active_amount.stage_put((claim_hash, 1, 201, tx_num + 1, 1), (200000,))
+            self.db.active_amount.stage_put((claim_hash, 1, 202, tx_num + 2, 1), (300000,))
+            tx_num += 3
+        self.db.unsafe_commit()
+
+        def get_active_amount_as_of_height(claim_hash: bytes, height: int) -> int:
+            for v in self.db.active_amount.iterate(
+                    start=(claim_hash, 1, 0), stop=(claim_hash, 1, height + 1),
+                    include_key=False, reverse=True):
+                return v.amount
+            return 0
+
+        for x in range(255):
+            claim_hash = 20 * chr(x).encode()
+            self.assertEqual(300000, get_active_amount_as_of_height(claim_hash, 300))
+            self.assertEqual(300000, get_active_amount_as_of_height(claim_hash, 203))
+            self.assertEqual(300000, get_active_amount_as_of_height(claim_hash, 202))
+            self.assertEqual(200000, get_active_amount_as_of_height(claim_hash, 201))
+            self.assertEqual(100000, get_active_amount_as_of_height(claim_hash, 200))
+            self.assertEqual(0, get_active_amount_as_of_height(claim_hash, 199))

tests/testcase.py

@@ -0,0 +1,748 @@
+import os
+import sys
+import json
+import shutil
+import logging
+import tempfile
+import functools
+import asyncio
+from asyncio.runners import _cancel_all_tasks  # type: ignore
+import unittest
+from unittest.case import _Outcome
+from typing import Optional
+from time import time, perf_counter
+from binascii import unhexlify
+from functools import partial
+
+from lbry.wallet import WalletManager, Wallet, Ledger, Account, Transaction
+from lbry.conf import Config
+from lbry.wallet.util import satoshis_to_coins
+from lbry.wallet.dewies import lbc_to_dewies
+from lbry.wallet.orchstr8 import Conductor
+from lbry.wallet.orchstr8.node import LBCWalletNode, WalletNode, HubNode
+from scribe.schema.claim import Claim
+
+from lbry.extras.daemon.daemon import Daemon, jsonrpc_dumps_pretty
+from lbry.extras.daemon.components import Component, WalletComponent
+from lbry.extras.daemon.components import (
+    DHT_COMPONENT,
+    HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
+    UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, LIBTORRENT_COMPONENT
+)
+from lbry.extras.daemon.componentmanager import ComponentManager
+from lbry.extras.daemon.exchange_rate_manager import (
+    ExchangeRateManager, ExchangeRate, BittrexBTCFeed, BittrexUSDFeed
+)
+from lbry.extras.daemon.storage import SQLiteStorage
+from lbry.blob.blob_manager import BlobManager
+from lbry.stream.reflector.server import ReflectorServer
+from lbry.blob_exchange.server import BlobServer
+
+
+class ColorHandler(logging.StreamHandler):
+
+    level_color = {
+        logging.DEBUG: "black",
+        logging.INFO: "light_gray",
+        logging.WARNING: "yellow",
+        logging.ERROR: "red"
+    }
+
+    color_code = dict(
+        black=30,
+        red=31,
+        green=32,
+        yellow=33,
+        blue=34,
+        magenta=35,
+        cyan=36,
+        white=37,
+        light_gray='0;37',
+        dark_gray='1;30'
+    )
+
+    def emit(self, record):
+        try:
+            msg = self.format(record)
+            color_name = self.level_color.get(record.levelno, "black")
+            color_code = self.color_code[color_name]
+            stream = self.stream
+            stream.write(f'\x1b[{color_code}m{msg}\x1b[0m')
+            stream.write(self.terminator)
+            self.flush()
+        except Exception:
+            self.handleError(record)
+
+
+HANDLER = ColorHandler(sys.stdout)
+HANDLER.setFormatter(
+    logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+)
+logging.getLogger().addHandler(HANDLER)
+
+
+class AsyncioTestCase(unittest.TestCase):
+    # Implementation inspired by discussion:
+    #  https://bugs.python.org/issue32972
+
+    LOOP_SLOW_CALLBACK_DURATION = 0.2
+    TIMEOUT = 120.0
+
+    maxDiff = None
+
+    async def asyncSetUp(self):  # pylint: disable=C0103
+        pass
+
+    async def asyncTearDown(self):  # pylint: disable=C0103
+        pass
+
+    def run(self, result=None):  # pylint: disable=R0915
+        orig_result = result
+        if result is None:
+            result = self.defaultTestResult()
+            startTestRun = getattr(result, 'startTestRun', None)  # pylint: disable=C0103
+            if startTestRun is not None:
+                startTestRun()
+
+        result.startTest(self)
+
+        testMethod = getattr(self, self._testMethodName)  # pylint: disable=C0103
+        if (getattr(self.__class__, "__unittest_skip__", False) or
+                getattr(testMethod, "__unittest_skip__", False)):
+            # If the class or method was skipped.
+            try:
+                skip_why = (getattr(self.__class__, '__unittest_skip_why__', '')
+                            or getattr(testMethod, '__unittest_skip_why__', ''))
+                self._addSkip(result, self, skip_why)
+            finally:
+                result.stopTest(self)
+            return
+        expecting_failure_method = getattr(testMethod,
+                                           "__unittest_expecting_failure__", False)
+        expecting_failure_class = getattr(self,
+                                          "__unittest_expecting_failure__", False)
+        expecting_failure = expecting_failure_class or expecting_failure_method
+        outcome = _Outcome(result)
+
+        self.loop = asyncio.new_event_loop()  # pylint: disable=W0201
+        asyncio.set_event_loop(self.loop)
+        self.loop.set_debug(True)
+        self.loop.slow_callback_duration = self.LOOP_SLOW_CALLBACK_DURATION
+
+        try:
+            self._outcome = outcome
+
+            with outcome.testPartExecutor(self):
+                self.setUp()
+                self.add_timeout()
+                self.loop.run_until_complete(self.asyncSetUp())
+            if outcome.success:
+                outcome.expecting_failure = expecting_failure
+                with outcome.testPartExecutor(self, isTest=True):
+                    maybe_coroutine = testMethod()
+                    if asyncio.iscoroutine(maybe_coroutine):
+                        self.add_timeout()
+                        self.loop.run_until_complete(maybe_coroutine)
+                outcome.expecting_failure = False
+                with outcome.testPartExecutor(self):
+                    self.add_timeout()
+                    self.loop.run_until_complete(self.asyncTearDown())
+                    self.tearDown()
+
+            self.doAsyncCleanups()
+
+            try:
+                _cancel_all_tasks(self.loop)
+                self.loop.run_until_complete(self.loop.shutdown_asyncgens())
+            finally:
+                asyncio.set_event_loop(None)
+                self.loop.close()
+
+            for test, reason in outcome.skipped:
+                self._addSkip(result, test, reason)
+            self._feedErrorsToResult(result, outcome.errors)
+            if outcome.success:
+                if expecting_failure:
+                    if outcome.expectedFailure:
+                        self._addExpectedFailure(result, outcome.expectedFailure)
+                    else:
+                        self._addUnexpectedSuccess(result)
+                else:
+                    result.addSuccess(self)
+            return result
+        finally:
+            result.stopTest(self)
+            if orig_result is None:
+                stopTestRun = getattr(result, 'stopTestRun', None)  # pylint: disable=C0103
+                if stopTestRun is not None:
+                    stopTestRun()  # pylint: disable=E1102
+
+            # explicitly break reference cycles:
+            # outcome.errors -> frame -> outcome -> outcome.errors
+            # outcome.expectedFailure -> frame -> outcome -> outcome.expectedFailure
+            outcome.errors.clear()
+            outcome.expectedFailure = None
+
+            # clear the outcome, no more needed
+            self._outcome = None
+
+    def doAsyncCleanups(self):  # pylint: disable=C0103
+        outcome = self._outcome or _Outcome()
+        while self._cleanups:
+            function, args, kwargs = self._cleanups.pop()
+            with outcome.testPartExecutor(self):
+                maybe_coroutine = function(*args, **kwargs)
+                if asyncio.iscoroutine(maybe_coroutine):
+                    self.add_timeout()
+                    self.loop.run_until_complete(maybe_coroutine)
+
+    def cancel(self):
+        for task in asyncio.all_tasks(self.loop):
+            if not task.done():
+                task.print_stack()
+                task.cancel()
+
+    def add_timeout(self):
+        if self.TIMEOUT:
+            self.loop.call_later(self.TIMEOUT, self.cancel)
+
+
+class AdvanceTimeTestCase(AsyncioTestCase):
+
+    async def asyncSetUp(self):
+        self._time = 0  # pylint: disable=W0201
+        self.loop.time = functools.wraps(self.loop.time)(lambda: self._time)
+        await super().asyncSetUp()
+
+    async def advance(self, seconds):
+        while self.loop._ready:
+            await asyncio.sleep(0)
+        self._time += seconds
+        await asyncio.sleep(0)
+        while self.loop._ready:
+            await asyncio.sleep(0)
+
+
+class IntegrationTestCase(AsyncioTestCase):
+
+    SEED = None
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.conductor: Optional[Conductor] = None
+        self.blockchain: Optional[LBCWalletNode] = None
+        self.hub: Optional[HubNode] = None
+        self.wallet_node: Optional[WalletNode] = None
+        self.manager: Optional[WalletManager] = None
+        self.ledger: Optional[Ledger] = None
+        self.wallet: Optional[Wallet] = None
+        self.account: Optional[Account] = None
+
+    async def asyncSetUp(self):
+        from time import perf_counter
+        start = perf_counter()
+        self.conductor = Conductor(seed=self.SEED)
+
+        await self.conductor.start_lbcd()
+        self.addCleanup(self.conductor.stop_lbcd)
+        print(f"{perf_counter() - start}s to start lbcd")
+        start = perf_counter()
+        await self.conductor.start_lbcwallet()
+        self.addCleanup(self.conductor.stop_lbcwallet)
+        print(f"{perf_counter() - start}s to start lbcwallet")
+        start = perf_counter()
+        await self.conductor.start_spv()
+        self.addCleanup(self.conductor.stop_spv)
+        print(f"{perf_counter() - start}s to start spv")
+        start = perf_counter()
+        await self.conductor.start_wallet()
+        self.addCleanup(self.conductor.stop_wallet)
+        print(f"{perf_counter() - start}s to start wallet")
+        start = perf_counter()
+        await self.conductor.start_hub()
+        self.addCleanup(self.conductor.stop_hub)
+        print(f"{perf_counter() - start}s to start go hub")
+
+        self.blockchain = self.conductor.lbcwallet_node
+        self.hub = self.conductor.hub_node
+        self.wallet_node = self.conductor.wallet_node
+        self.manager = self.wallet_node.manager
+        self.ledger = self.wallet_node.ledger
+        self.wallet = self.wallet_node.wallet
+        self.account = self.wallet_node.wallet.default_account
+
+    async def assertBalance(self, account, expected_balance: str):  # pylint: disable=C0103
+        balance = await account.get_balance()
+        self.assertEqual(satoshis_to_coins(balance), expected_balance)
+
+    def broadcast(self, tx):
+        return self.ledger.broadcast(tx)
+
+    async def broadcast_and_confirm(self, tx, ledger=None):
+        ledger = ledger or self.ledger
+        notifications = asyncio.create_task(ledger.wait(tx))
+        await ledger.broadcast(tx)
+        await notifications
+        await self.generate_and_wait(1, [tx.id], ledger)
+
+    async def on_header(self, height):
+        if self.ledger.headers.height < height:
+            await self.ledger.on_header.where(
+                lambda e: e.height == height
+            )
+        return True
+
+    async def send_to_address_and_wait(self, address, amount, blocks_to_generate=0, ledger=None):
+        tx_watch = []
+        txid = None
+        done = False
+        watcher = (ledger or self.ledger).on_transaction.where(
+            lambda e: e.tx.id == txid or done or tx_watch.append(e.tx.id)
+        )
+
+        txid = await self.blockchain.send_to_address(address, amount)
+        done = txid in tx_watch
+        await watcher
+
+        await self.generate_and_wait(blocks_to_generate, [txid], ledger)
+        return txid
+
+    async def generate_and_wait(self, blocks_to_generate, txids, ledger=None):
+        if blocks_to_generate > 0:
+            watcher = (ledger or self.ledger).on_transaction.where(
+                lambda e: ((e.tx.id in txids and txids.remove(e.tx.id)), len(txids) <= 0)[-1]  # multi-statement lambda
+            )
+            self.conductor.spv_node.server.synchronized.clear()
+            await self.blockchain.generate(blocks_to_generate)
+            height = self.blockchain.block_expected
+            await watcher
+            while True:
+                await self.conductor.spv_node.server.synchronized.wait()
+                self.conductor.spv_node.server.synchronized.clear()
+                if self.conductor.spv_node.server.db.db_height >= height:
+                    break
+
+    def on_address_update(self, address):
+        return self.ledger.on_transaction.where(
+            lambda e: e.address == address
+        )
+
+    def on_transaction_address(self, tx, address):
+        return self.ledger.on_transaction.where(
+            lambda e: e.tx.id == tx.id and e.address == address
+        )
+
+    async def generate(self, blocks):
+        """ Ask lbrycrd to generate some blocks and wait until ledger has them. """
+        prepare = self.ledger.on_header.where(self.blockchain.is_expected_block)
+        height = self.blockchain.block_expected
+        self.conductor.spv_node.server.synchronized.clear()
+        await self.blockchain.generate(blocks)
+        await prepare  # no guarantee that it didn't happen already, so start waiting from before calling generate
+        while True:
+            await self.conductor.spv_node.server.synchronized.wait()
+            self.conductor.spv_node.server.synchronized.clear()
+            if self.conductor.spv_node.server.db.db_height >= height:
+                break
+
+
+class FakeExchangeRateManager(ExchangeRateManager):
+
+    def __init__(self, market_feeds, rates):  # pylint: disable=super-init-not-called
+        self.market_feeds = market_feeds
+        for feed in self.market_feeds:
+            feed.last_check = time()
+            feed.rate = ExchangeRate(feed.market, rates[feed.market], time())
+
+    def start(self):
+        pass
+
+    def stop(self):
+        pass
+
+
+def get_fake_exchange_rate_manager(rates=None):
+    return FakeExchangeRateManager(
+        [BittrexBTCFeed(), BittrexUSDFeed()],
+        rates or {'BTCLBC': 3.0, 'USDLBC': 2.0}
+    )
+
+
+class ExchangeRateManagerComponent(Component):
+    component_name = EXCHANGE_RATE_MANAGER_COMPONENT
+
+    def __init__(self, component_manager, rates=None):
+        super().__init__(component_manager)
+        self.exchange_rate_manager = get_fake_exchange_rate_manager(rates)
+
+    @property
+    def component(self) -> ExchangeRateManager:
+        return self.exchange_rate_manager
+
+    async def start(self):
+        self.exchange_rate_manager.start()
+
+    async def stop(self):
+        self.exchange_rate_manager.stop()
+
+
+class CommandTestCase(IntegrationTestCase):
+
+    VERBOSITY = logging.WARN
+    blob_lru_cache_size = 0
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.daemon = None
+        self.daemons = []
+        self.server_config = None
+        self.server_storage = None
+        self.extra_wallet_nodes = []
+        self.extra_wallet_node_port = 5280
+        self.server_blob_manager = None
+        self.server = None
+        self.reflector = None
+        self.skip_libtorrent = True
+
+    async def asyncSetUp(self):
+        start = perf_counter()
+
+        logging.getLogger('lbry.blob_exchange').setLevel(self.VERBOSITY)
+        logging.getLogger('lbry.daemon').setLevel(self.VERBOSITY)
+        logging.getLogger('lbry.stream').setLevel(self.VERBOSITY)
+        logging.getLogger('lbry.wallet').setLevel(self.VERBOSITY)
+
+        await super().asyncSetUp()
+        print("first setup", perf_counter() - start)
+        start = perf_counter()
+
+        self.daemon = await self.add_daemon(self.wallet_node)
+
+        await self.account.ensure_address_gap()
+        address = (await self.account.receiving.get_addresses(limit=1, only_usable=True))[0]
+        await self.send_to_address_and_wait(address, 10, 6)
+        print("sent to address and waited", perf_counter() - start)
+
+        server_tmp_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, server_tmp_dir)
+        self.server_config = Config(
+            data_dir=server_tmp_dir,
+            wallet_dir=server_tmp_dir,
+            save_files=True,
+            download_dir=server_tmp_dir
+        )
+        self.server_config.transaction_cache_size = 10000
+        self.server_storage = SQLiteStorage(self.server_config, ':memory:')
+        await self.server_storage.open()
+
+        self.server_blob_manager = BlobManager(self.loop, server_tmp_dir, self.server_storage, self.server_config)
+        self.server = BlobServer(self.loop, self.server_blob_manager, 'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP')
+        self.server.start_server(5567, '127.0.0.1')
+        await self.server.started_listening.wait()
+
+        self.reflector = ReflectorServer(self.server_blob_manager)
+        self.reflector.start_server(5566, '127.0.0.1')
+        await self.reflector.started_listening.wait()
+        self.addCleanup(self.reflector.stop_server)
+
+    async def asyncTearDown(self):
+        await super().asyncTearDown()
+        for wallet_node in self.extra_wallet_nodes:
+            await wallet_node.stop(cleanup=True)
+        for daemon in self.daemons:
+            daemon.component_manager.get_component('wallet')._running = False
+            await daemon.stop()
+
+    async def add_daemon(self, wallet_node=None, seed=None):
+        start_wallet_node = False
+        if wallet_node is None:
+            wallet_node = WalletNode(
+                self.wallet_node.manager_class,
+                self.wallet_node.ledger_class,
+                port=self.extra_wallet_node_port
+            )
+            self.extra_wallet_node_port += 1
+            start_wallet_node = True
+
+        upload_dir = os.path.join(wallet_node.data_path, 'uploads')
+        os.mkdir(upload_dir)
+
+        conf = Config(
+            # needed during instantiation to access known_hubs path
+            data_dir=wallet_node.data_path,
+            wallet_dir=wallet_node.data_path,
+            save_files=True,
+            download_dir=wallet_node.data_path
+        )
+        conf.upload_dir = upload_dir  # not a real conf setting
+        conf.share_usage_data = False
+        conf.use_upnp = False
+        conf.reflect_streams = True
+        conf.blockchain_name = 'lbrycrd_regtest'
+        conf.lbryum_servers = [(self.conductor.spv_node.hostname, self.conductor.spv_node.port)]
+        conf.reflector_servers = [('127.0.0.1', 5566)]
+        conf.fixed_peers = [('127.0.0.1', 5567)]
+        conf.known_dht_nodes = []
+        conf.blob_lru_cache_size = self.blob_lru_cache_size
+        conf.transaction_cache_size = 10000
+        conf.components_to_skip = [
+            DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
+            PEER_PROTOCOL_SERVER_COMPONENT
+        ]
+        if self.skip_libtorrent:
+            conf.components_to_skip.append(LIBTORRENT_COMPONENT)
+
+        if start_wallet_node:
+            await wallet_node.start(self.conductor.spv_node, seed=seed, config=conf)
+            self.extra_wallet_nodes.append(wallet_node)
+        else:
+            wallet_node.manager.config = conf
+            wallet_node.manager.ledger.config['known_hubs'] = conf.known_hubs
+
+        def wallet_maker(component_manager):
+            wallet_component = WalletComponent(component_manager)
+            wallet_component.wallet_manager = wallet_node.manager
+            wallet_component._running = True
+            return wallet_component
+
+        daemon = Daemon(conf, ComponentManager(
+            conf, skip_components=conf.components_to_skip, wallet=wallet_maker,
+            exchange_rate_manager=partial(ExchangeRateManagerComponent, rates={
+                'BTCLBC': 1.0, 'USDLBC': 2.0
+            })
+        ))
+        await daemon.initialize()
+        self.daemons.append(daemon)
+        wallet_node.manager.old_db = daemon.storage
+        return daemon
+
+    async def confirm_tx(self, txid, ledger=None):
+        """ Wait for tx to be in mempool, then generate a block, wait for tx to be in a block. """
+        # await (ledger or self.ledger).on_transaction.where(lambda e: e.tx.id == txid)
+        on_tx = (ledger or self.ledger).on_transaction.where(lambda e: e.tx.id == txid)
+        await asyncio.wait([self.generate(1), on_tx], timeout=5)
+
+        # # actually, if it's in the mempool or in the block we're fine
+        # await self.generate_and_wait(1, [txid], ledger=ledger)
+        # return txid
+
+        return txid
+
+    async def on_transaction_dict(self, tx):
+        await self.ledger.wait(Transaction(unhexlify(tx['hex'])))
+
+    @staticmethod
+    def get_all_addresses(tx):
+        addresses = set()
+        for txi in tx['inputs']:
+            addresses.add(txi['address'])
+        for txo in tx['outputs']:
+            addresses.add(txo['address'])
+        return list(addresses)
+
+    async def blockchain_claim_name(self, name: str, value: str, amount: str, confirm=True):
+        txid = await self.blockchain._cli_cmnd('claimname', name, value, amount)
+        if confirm:
+            await self.generate(1)
+        return txid
+
+    async def blockchain_update_name(self, txid: str, value: str, amount: str, confirm=True):
+        txid = await self.blockchain._cli_cmnd('updateclaim', txid, value, amount)
+        if confirm:
+            await self.generate(1)
+        return txid
+
+    async def out(self, awaitable):
+        """ Serializes lbrynet API results to JSON then loads and returns it as dictionary. """
+        return json.loads(jsonrpc_dumps_pretty(await awaitable, ledger=self.ledger))['result']
+
+    def sout(self, value):
+        """ Synchronous version of `out` method. """
+        return json.loads(jsonrpc_dumps_pretty(value, ledger=self.ledger))['result']
+
+    async def confirm_and_render(self, awaitable, confirm, return_tx=False) -> Transaction:
+        tx = await awaitable
+        if confirm:
+            await self.ledger.wait(tx)
+            await self.generate(1)
+            await self.ledger.wait(tx, self.blockchain.block_expected)
+        if not return_tx:
+            return self.sout(tx)
+        return tx
+
+    async def create_nondeterministic_channel(self, name, price, pubkey_bytes, daemon=None, blocking=False):
+        account = (daemon or self.daemon).wallet_manager.default_account
+        claim_address = await account.receiving.get_or_create_usable_address()
+        claim = Claim()
+        claim.channel.public_key_bytes = pubkey_bytes
+        tx = await Transaction.claim_create(
+            name, claim, lbc_to_dewies(price),
+            claim_address, [self.account], self.account
+        )
+        await tx.sign([self.account])
+        await (daemon or self.daemon).broadcast_or_release(tx, blocking)
+        return self.sout(tx)
+
+    def create_upload_file(self, data, prefix=None, suffix=None):
+        file_path = tempfile.mktemp(prefix=prefix or "tmp", suffix=suffix or "", dir=self.daemon.conf.upload_dir)
+        with open(file_path, 'w+b') as file:
+            file.write(data)
+            file.flush()
+            return file.name
+
+    async def stream_create(
+            self, name='hovercraft', bid='1.0', file_path=None,
+            data=b'hi!', confirm=True, prefix=None, suffix=None, return_tx=False, **kwargs):
+        if file_path is None and data is not None:
+            file_path = self.create_upload_file(data=data, prefix=prefix, suffix=suffix)
+        return await self.confirm_and_render(
+            self.daemon.jsonrpc_stream_create(name, bid, file_path=file_path, **kwargs), confirm, return_tx
+        )
+
+    async def stream_update(
+            self, claim_id, data=None, prefix=None, suffix=None, confirm=True, return_tx=False, **kwargs):
+        if data is not None:
+            file_path = self.create_upload_file(data=data, prefix=prefix, suffix=suffix)
+            return await self.confirm_and_render(
+                self.daemon.jsonrpc_stream_update(claim_id, file_path=file_path, **kwargs), confirm, return_tx
+            )
+        return await self.confirm_and_render(
+            self.daemon.jsonrpc_stream_update(claim_id, **kwargs), confirm
+        )
+
+    async def stream_repost(self, claim_id, name='repost', bid='1.0', confirm=True, **kwargs):
+        return await self.confirm_and_render(
+            self.daemon.jsonrpc_stream_repost(claim_id=claim_id, name=name, bid=bid, **kwargs), confirm
+        )
+
+    async def stream_abandon(self, *args, confirm=True, **kwargs):
+        if 'blocking' not in kwargs:
+            kwargs['blocking'] = False
+        return await self.confirm_and_render(
+            self.daemon.jsonrpc_stream_abandon(*args, **kwargs), confirm
+        )
+
+    async def purchase_create(self, *args, confirm=True, **kwargs):
+        return await self.confirm_and_render(
+            self.daemon.jsonrpc_purchase_create(*args, **kwargs), confirm
+        )
+
+    async def publish(self, name, *args, confirm=True, **kwargs):
+        return await self.confirm_and_render(
+            self.daemon.jsonrpc_publish(name, *args, **kwargs), confirm
+        )
+
+    async def channel_create(self, name='@arena', bid='1.0', confirm=True, **kwargs):
+        return await self.confirm_and_render(
+            self.daemon.jsonrpc_channel_create(name, bid, **kwargs), confirm
+        )
+
+    async def channel_update(self, claim_id, confirm=True, **kwargs):
+        return await self.confirm_and_render(
+            self.daemon.jsonrpc_channel_update(claim_id, **kwargs), confirm
+        )
+
+    async def channel_abandon(self, *args, confirm=True, **kwargs):
+        if 'blocking' not in kwargs:
+            kwargs['blocking'] = False
+        return await self.confirm_and_render(
+            self.daemon.jsonrpc_channel_abandon(*args, **kwargs), confirm
+        )
+
+    async def collection_create(
+            self, name='firstcollection', bid='1.0', confirm=True, **kwargs):
+        return await self.confirm_and_render(
+            self.daemon.jsonrpc_collection_create(name, bid, **kwargs), confirm
+        )
+
+    async def collection_update(
+            self, claim_id, confirm=True, **kwargs):
+        return await self.confirm_and_render(
+            self.daemon.jsonrpc_collection_update(claim_id, **kwargs), confirm
+        )
+
+    async def collection_abandon(self, *args, confirm=True, **kwargs):
+        if 'blocking' not in kwargs:
+            kwargs['blocking'] = False
+        return await self.confirm_and_render(
+            self.daemon.jsonrpc_stream_abandon(*args, **kwargs), confirm
+        )
+
+    async def support_create(self, claim_id, bid='1.0', confirm=True, **kwargs):
+        return await self.confirm_and_render(
+            self.daemon.jsonrpc_support_create(claim_id, bid, **kwargs), confirm
+        )
+
+    async def support_abandon(self, *args, confirm=True, **kwargs):
+        if 'blocking' not in kwargs:
+            kwargs['blocking'] = False
+        return await self.confirm_and_render(
+            self.daemon.jsonrpc_support_abandon(*args, **kwargs), confirm
+        )
+
+    async def account_send(self, *args, confirm=True, **kwargs):
+        return await self.confirm_and_render(
+            self.daemon.jsonrpc_account_send(*args, **kwargs), confirm
+        )
+
+    async def wallet_send(self, *args, confirm=True, **kwargs):
+        return await self.confirm_and_render(
+            self.daemon.jsonrpc_wallet_send(*args, **kwargs), confirm
+        )
+
+    async def txo_spend(self, *args, confirm=True, **kwargs):
+        txs = await self.daemon.jsonrpc_txo_spend(*args, **kwargs)
+        if confirm:
+            await asyncio.wait([self.ledger.wait(tx) for tx in txs])
+            await self.generate(1)
+            await asyncio.wait([self.ledger.wait(tx, self.blockchain.block_expected) for tx in txs])
+        return self.sout(txs)
+
+    async def blob_clean(self):
+        return await self.out(self.daemon.jsonrpc_blob_clean())
+
+    async def status(self):
+        return await self.out(self.daemon.jsonrpc_status())
+
+    async def resolve(self, uri, **kwargs):
+        return (await self.out(self.daemon.jsonrpc_resolve(uri, **kwargs)))[uri]
+
+    async def claim_search(self, **kwargs):
+        return (await self.out(self.daemon.jsonrpc_claim_search(**kwargs)))['items']
+
+    async def get_claim_by_claim_id(self, claim_id):
+        return await self.out(self.ledger.get_claim_by_claim_id(claim_id))
+
+    async def file_list(self, *args, **kwargs):
+        return (await self.out(self.daemon.jsonrpc_file_list(*args, **kwargs)))['items']
+
+    async def txo_list(self, *args, **kwargs):
+        return (await self.out(self.daemon.jsonrpc_txo_list(*args, **kwargs)))['items']
+
+    async def txo_sum(self, *args, **kwargs):
+        return await self.out(self.daemon.jsonrpc_txo_sum(*args, **kwargs))
+
+    async def txo_plot(self, *args, **kwargs):
+        return await self.out(self.daemon.jsonrpc_txo_plot(*args, **kwargs))
+
+    async def claim_list(self, *args, **kwargs):
+        return (await self.out(self.daemon.jsonrpc_claim_list(*args, **kwargs)))['items']
+
+    async def stream_list(self, *args, **kwargs):
+        return (await self.out(self.daemon.jsonrpc_stream_list(*args, **kwargs)))['items']
+
+    async def channel_list(self, *args, **kwargs):
+        return (await self.out(self.daemon.jsonrpc_channel_list(*args, **kwargs)))['items']
+
+    async def transaction_list(self, *args, **kwargs):
+        return (await self.out(self.daemon.jsonrpc_transaction_list(*args, **kwargs)))['items']
+
+    async def blob_list(self, *args, **kwargs):
+        return (await self.out(self.daemon.jsonrpc_blob_list(*args, **kwargs)))['items']
+
+    @staticmethod
+    def get_claim_id(tx):
+        return tx['outputs'][0]['claim_id']
+
+    def assertItemCount(self, result, count):  # pylint: disable=invalid-name
+        self.assertEqual(count, result['total_items'])