From ca34d703a293efa970b1ad8ccef6c17380f7b417 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 8 Mar 2022 20:40:09 -0500 Subject: [PATCH] move lbry.wallet.server and lbry.schema to scribe --- lbry/extras/daemon/daemon.py | 6 +- lbry/extras/daemon/json_response_encoder.py | 6 +- lbry/extras/daemon/migrator/migrate5to6.py | 2 +- lbry/extras/daemon/storage.py | 2 +- lbry/file/file_manager.py | 2 +- lbry/schema/Makefile | 5 - lbry/schema/README.md | 24 - lbry/schema/__init__.py | 1 - lbry/schema/attrs.py | 571 -- lbry/schema/base.py | 124 - lbry/schema/claim.py | 422 -- lbry/schema/compat.py | 93 - lbry/schema/mime_types.py | 214 - lbry/schema/purchase.py | 47 - lbry/schema/result.py | 256 - lbry/schema/support.py | 23 - lbry/schema/tags.py | 13 - lbry/schema/types/__init__.py | 0 lbry/schema/types/v1/__init__.py | 0 lbry/schema/types/v1/certificate_pb2.py | 146 - lbry/schema/types/v1/fee_pb2.py | 148 - lbry/schema/types/v1/legacy_claim_pb2.py | 158 - lbry/schema/types/v1/metadata_pb2.py | 936 ---- lbry/schema/types/v1/signature_pb2.py | 118 - lbry/schema/types/v1/source_pb2.py | 140 - lbry/schema/types/v1/stream_pb2.py | 113 - lbry/schema/types/v2/__init__.py | 0 lbry/schema/types/v2/claim_pb2.py | 4692 ----------------- lbry/schema/types/v2/hub_pb2.py | 960 ---- lbry/schema/types/v2/hub_pb2_grpc.py | 298 -- lbry/schema/types/v2/purchase_pb2.py | 69 - lbry/schema/types/v2/result_pb2.py | 464 -- lbry/schema/types/v2/result_pb2_grpc.py | 4 - lbry/schema/types/v2/support_pb2.py | 76 - lbry/schema/url.py | 130 - lbry/stream/managed_stream.py | 4 +- lbry/testcase.py | 2 +- lbry/utils.py | 2 +- lbry/wallet/account.py | 2 +- lbry/wallet/bip32.py | 338 -- lbry/wallet/database.py | 2 +- lbry/wallet/ledger.py | 6 +- lbry/wallet/network.py | 4 +- lbry/wallet/orchstr8/node.py | 9 +- lbry/wallet/server/__init__.py | 0 lbry/wallet/server/block_processor.py | 1697 ------ lbry/wallet/server/chain_reader.py | 285 - lbry/wallet/server/cli.py | 64 - lbry/wallet/server/coin.py | 381 -- lbry/wallet/server/daemon.py | 345 -- lbry/wallet/server/db/__init__.py | 49 - lbry/wallet/server/db/common.py | 477 -- lbry/wallet/server/db/db.py | 1152 ---- .../server/db/elasticsearch/__init__.py | 1 - lbry/wallet/server/db/elasticsearch/common.py | 150 - .../server/db/elasticsearch/constants.py | 100 - .../db/elasticsearch/fast_ar_trending.py | 117 - .../server/db/elasticsearch/notifier.py | 55 - lbry/wallet/server/db/elasticsearch/search.py | 727 --- lbry/wallet/server/db/elasticsearch/sync.py | 416 -- lbry/wallet/server/db/interface.py | 145 - lbry/wallet/server/db/prefixes.py | 1800 ------- lbry/wallet/server/db/revertable.py | 175 - lbry/wallet/server/env.py | 397 -- lbry/wallet/server/hash.py | 160 - lbry/wallet/server/mempool.py | 207 - lbry/wallet/server/merkle.py | 258 - lbry/wallet/server/prefetcher.py | 119 - lbry/wallet/server/script.py | 289 - lbry/wallet/server/server.py | 0 lbry/wallet/server/session.py | 1525 ------ lbry/wallet/server/tx.py | 626 --- lbry/wallet/server/util.py | 361 -- lbry/wallet/server/version.py | 3 - lbry/wallet/server/websocket.py | 55 - lbry/wallet/transaction.py | 12 +- lbry/wallet/udp.py | 2 +- setup.py | 3 - test_rocksdb.py | 47 - tests/integration/blockchain/test_network.py | 12 +- .../blockchain/test_purchase_command.py | 2 +- .../blockchain/test_wallet_server_sessions.py | 8 +- .../datanetwork/test_file_commands.py | 2 +- .../takeovers/test_resolve_command.py | 2 +- .../test_internal_transaction_api.py | 2 +- .../test_exchange_rate_manager.py | 2 +- tests/unit/lbrynet_daemon/test_mime_types.py | 2 +- tests/unit/schema/test_claim_from_bytes.py | 2 +- tests/unit/schema/test_mime_types.py | 2 +- tests/unit/schema/test_models.py | 2 +- tests/unit/schema/test_tags.py | 2 +- tests/unit/schema/test_url.py | 2 +- tests/unit/stream/test_stream_manager.py | 2 +- tests/unit/wallet/test_bip32.py | 2 +- tests/unit/wallet/test_schema_signing.py | 4 +- 95 files changed, 54 insertions(+), 22828 deletions(-) delete mode 100644 lbry/schema/Makefile delete mode 100644 lbry/schema/README.md delete mode 100644 lbry/schema/__init__.py delete mode 100644 lbry/schema/attrs.py delete mode 100644 lbry/schema/base.py delete mode 100644 lbry/schema/claim.py delete mode 100644 lbry/schema/compat.py delete mode 100644 lbry/schema/mime_types.py delete mode 100644 lbry/schema/purchase.py delete mode 100644 lbry/schema/result.py delete mode 100644 lbry/schema/support.py delete mode 100644 lbry/schema/tags.py delete mode 100644 lbry/schema/types/__init__.py delete mode 100644 lbry/schema/types/v1/__init__.py delete mode 100644 lbry/schema/types/v1/certificate_pb2.py delete mode 100644 lbry/schema/types/v1/fee_pb2.py delete mode 100644 lbry/schema/types/v1/legacy_claim_pb2.py delete mode 100644 lbry/schema/types/v1/metadata_pb2.py delete mode 100644 lbry/schema/types/v1/signature_pb2.py delete mode 100644 lbry/schema/types/v1/source_pb2.py delete mode 100644 lbry/schema/types/v1/stream_pb2.py delete mode 100644 lbry/schema/types/v2/__init__.py delete mode 100644 lbry/schema/types/v2/claim_pb2.py delete mode 100644 lbry/schema/types/v2/hub_pb2.py delete mode 100644 lbry/schema/types/v2/hub_pb2_grpc.py delete mode 100644 lbry/schema/types/v2/purchase_pb2.py delete mode 100644 lbry/schema/types/v2/result_pb2.py delete mode 100644 lbry/schema/types/v2/result_pb2_grpc.py delete mode 100644 lbry/schema/types/v2/support_pb2.py delete mode 100644 lbry/schema/url.py delete mode 100644 lbry/wallet/bip32.py delete mode 100644 lbry/wallet/server/__init__.py delete mode 100644 lbry/wallet/server/block_processor.py delete mode 100644 lbry/wallet/server/chain_reader.py delete mode 100644 lbry/wallet/server/cli.py delete mode 100644 lbry/wallet/server/coin.py delete mode 100644 lbry/wallet/server/daemon.py delete mode 100644 lbry/wallet/server/db/__init__.py delete mode 100644 lbry/wallet/server/db/common.py delete mode 100644 lbry/wallet/server/db/db.py delete mode 100644 lbry/wallet/server/db/elasticsearch/__init__.py delete mode 100644 lbry/wallet/server/db/elasticsearch/common.py delete mode 100644 lbry/wallet/server/db/elasticsearch/constants.py delete mode 100644 lbry/wallet/server/db/elasticsearch/fast_ar_trending.py delete mode 100644 lbry/wallet/server/db/elasticsearch/notifier.py delete mode 100644 lbry/wallet/server/db/elasticsearch/search.py delete mode 100644 lbry/wallet/server/db/elasticsearch/sync.py delete mode 100644 lbry/wallet/server/db/interface.py delete mode 100644 lbry/wallet/server/db/prefixes.py delete mode 100644 lbry/wallet/server/db/revertable.py delete mode 100644 lbry/wallet/server/env.py delete mode 100644 lbry/wallet/server/hash.py delete mode 100644 lbry/wallet/server/mempool.py delete mode 100644 lbry/wallet/server/merkle.py delete mode 100644 lbry/wallet/server/prefetcher.py delete mode 100644 lbry/wallet/server/script.py delete mode 100644 lbry/wallet/server/server.py delete mode 100644 lbry/wallet/server/session.py delete mode 100644 lbry/wallet/server/tx.py delete mode 100644 lbry/wallet/server/util.py delete mode 100644 lbry/wallet/server/version.py delete mode 100644 lbry/wallet/server/websocket.py delete mode 100755 test_rocksdb.py diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index 1240eb407..45d3ba68d 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -27,7 +27,7 @@ from lbry.wallet import ( ) from lbry.wallet.dewies import dewies_to_lbc, lbc_to_dewies, dict_values_to_lbc from lbry.wallet.constants import TXO_TYPES, CLAIM_TYPE_NAMES -from lbry.wallet.bip32 import PrivateKey +from scribe.bip32 import PrivateKey from lbry import utils from lbry.conf import Config, Setting, NOT_SET @@ -51,8 +51,8 @@ from lbry.extras.daemon.json_response_encoder import JSONResponseEncoder from lbry.extras.daemon.undecorated import undecorated from lbry.extras.daemon.security import ensure_request_allowed from lbry.file_analysis import VideoFileAnalyzer -from lbry.schema.claim import Claim -from lbry.schema.url import URL, normalize_name +from scribe.schema.claim import Claim +from scribe.schema.url import URL, normalize_name if typing.TYPE_CHECKING: diff --git a/lbry/extras/daemon/json_response_encoder.py b/lbry/extras/daemon/json_response_encoder.py index c75e7d512..39d896c49 100644 --- a/lbry/extras/daemon/json_response_encoder.py +++ b/lbry/extras/daemon/json_response_encoder.py @@ -6,11 +6,11 @@ from json import JSONEncoder from google.protobuf.message import DecodeError -from lbry.schema.claim import Claim -from lbry.schema.support import Support +from scribe.schema.claim import Claim +from scribe.schema.support import Support +from scribe.bip32 import PublicKey from lbry.torrent.torrent_manager import TorrentSource from lbry.wallet import Wallet, Ledger, Account, Transaction, Output -from lbry.wallet.bip32 import PublicKey from lbry.wallet.dewies import dewies_to_lbc from lbry.stream.managed_stream import ManagedStream diff --git a/lbry/extras/daemon/migrator/migrate5to6.py b/lbry/extras/daemon/migrator/migrate5to6.py index d74ddd15e..160c61424 100644 --- a/lbry/extras/daemon/migrator/migrate5to6.py +++ b/lbry/extras/daemon/migrator/migrate5to6.py @@ -3,7 +3,7 @@ import os import json import logging from binascii import hexlify -from lbry.schema.claim import Claim +from scribe.schema.claim import Claim log = logging.getLogger(__name__) diff --git a/lbry/extras/daemon/storage.py b/lbry/extras/daemon/storage.py index e5c9ec67c..8982e4464 100644 --- a/lbry/extras/daemon/storage.py +++ b/lbry/extras/daemon/storage.py @@ -10,7 +10,7 @@ from lbry.wallet import SQLiteMixin from lbry.conf import Config from lbry.wallet.dewies import dewies_to_lbc, lbc_to_dewies from lbry.wallet.transaction import Transaction, Output -from lbry.schema.claim import Claim +from scribe.schema.claim import Claim from lbry.dht.constants import DATA_EXPIRATION from lbry.blob.blob_info import BlobInfo diff --git a/lbry/file/file_manager.py b/lbry/file/file_manager.py index 6b398d975..e32ce05cb 100644 --- a/lbry/file/file_manager.py +++ b/lbry/file/file_manager.py @@ -9,7 +9,7 @@ from lbry.error import InvalidStreamURLError from lbry.stream.managed_stream import ManagedStream from lbry.torrent.torrent_manager import TorrentSource from lbry.utils import cache_concurrent -from lbry.schema.url import URL +from scribe.schema.url import URL from lbry.wallet.dewies import dewies_to_lbc from lbry.file.source_manager import SourceManager from lbry.file.source import ManagedDownloadSource diff --git a/lbry/schema/Makefile b/lbry/schema/Makefile deleted file mode 100644 index 917b2a8e2..000000000 --- a/lbry/schema/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -build: - rm types/v2/* -rf - touch types/v2/__init__.py - cd types/v2/ && protoc --python_out=. -I ../../../../../types/v2/proto/ ../../../../../types/v2/proto/*.proto - sed -e 's/^import\ \(.*\)_pb2\ /from . import\ \1_pb2\ /g' -i types/v2/*.py diff --git a/lbry/schema/README.md b/lbry/schema/README.md deleted file mode 100644 index 12af8c720..000000000 --- a/lbry/schema/README.md +++ /dev/null @@ -1,24 +0,0 @@ -Schema -===== - -Those files are generated from the [types repo](https://github.com/lbryio/types). If you are modifying/adding a new type, make sure it is cloned in the same root folder as the SDK repo, like: - -``` -repos/ - - lbry-sdk/ - - types/ -``` - -Then, [download protoc 3.2.0](https://github.com/protocolbuffers/protobuf/releases/tag/v3.2.0), add it to your PATH. On linux it is: - -```bash -cd ~/.local/bin -wget https://github.com/protocolbuffers/protobuf/releases/download/v3.2.0/protoc-3.2.0-linux-x86_64.zip -unzip protoc-3.2.0-linux-x86_64.zip bin/protoc -d.. -``` - -Finally, `make` should update everything in place. - - -### Why protoc 3.2.0? -Different/newer versions will generate larger diffs and we need to make sure they are good. In theory, we can just update to latest and it will all work, but it is a good practice to check blockchain data and retro compatibility before bumping versions (if you do, please update this section!). diff --git a/lbry/schema/__init__.py b/lbry/schema/__init__.py deleted file mode 100644 index b5ab2a718..000000000 --- a/lbry/schema/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .claim import Claim \ No newline at end of file diff --git a/lbry/schema/attrs.py b/lbry/schema/attrs.py deleted file mode 100644 index 522dc3413..000000000 --- a/lbry/schema/attrs.py +++ /dev/null @@ -1,571 +0,0 @@ -import json -import logging -import os.path -import hashlib -from typing import Tuple, List -from string import ascii_letters -from decimal import Decimal, ROUND_UP -from binascii import hexlify, unhexlify -from google.protobuf.json_format import MessageToDict - -from lbry.crypto.base58 import Base58 -from lbry.constants import COIN -from lbry.error import MissingPublishedFileError, EmptyPublishedFileError - -from lbry.schema.mime_types import guess_media_type -from lbry.schema.base import Metadata, BaseMessageList -from lbry.schema.tags import clean_tags, normalize_tag -from lbry.schema.types.v2.claim_pb2 import ( - Fee as FeeMessage, - Location as LocationMessage, - Language as LanguageMessage -) - - -log = logging.getLogger(__name__) - - -def calculate_sha384_file_hash(file_path): - sha384 = hashlib.sha384() - with open(file_path, 'rb') as f: - for chunk in iter(lambda: f.read(128 * sha384.block_size), b''): - sha384.update(chunk) - return sha384.digest() - - -def country_int_to_str(country: int) -> str: - r = LocationMessage.Country.Name(country) - return r[1:] if r.startswith('R') else r - - -def country_str_to_int(country: str) -> int: - if len(country) == 3: - country = 'R' + country - return LocationMessage.Country.Value(country) - - -class Dimmensional(Metadata): - - __slots__ = () - - @property - def width(self) -> int: - return self.message.width - - @width.setter - def width(self, width: int): - self.message.width = width - - @property - def height(self) -> int: - return self.message.height - - @height.setter - def height(self, height: int): - self.message.height = height - - @property - def dimensions(self) -> Tuple[int, int]: - return self.width, self.height - - @dimensions.setter - def dimensions(self, dimensions: Tuple[int, int]): - self.message.width, self.message.height = dimensions - - def _extract(self, file_metadata, field): - try: - setattr(self, field, file_metadata.getValues(field)[0]) - except: - log.exception(f'Could not extract {field} from file metadata.') - - def update(self, file_metadata=None, height=None, width=None): - if height is not None: - self.height = height - elif file_metadata: - self._extract(file_metadata, 'height') - - if width is not None: - self.width = width - elif file_metadata: - self._extract(file_metadata, 'width') - - -class Playable(Metadata): - - __slots__ = () - - @property - def duration(self) -> int: - return self.message.duration - - @duration.setter - def duration(self, duration: int): - self.message.duration = duration - - def update(self, file_metadata=None, duration=None): - if duration is not None: - self.duration = duration - elif file_metadata: - try: - self.duration = file_metadata.getValues('duration')[0].seconds - except: - log.exception('Could not extract duration from file metadata.') - - -class Image(Dimmensional): - - __slots__ = () - - -class Audio(Playable): - - __slots__ = () - - -class Video(Dimmensional, Playable): - - __slots__ = () - - def update(self, file_metadata=None, height=None, width=None, duration=None): - Dimmensional.update(self, file_metadata, height, width) - Playable.update(self, file_metadata, duration) - - -class Source(Metadata): - - __slots__ = () - - def update(self, file_path=None): - if file_path is not None: - self.name = os.path.basename(file_path) - self.media_type, stream_type = guess_media_type(file_path) - if not os.path.isfile(file_path): - raise MissingPublishedFileError(file_path) - self.size = os.path.getsize(file_path) - if self.size == 0: - raise EmptyPublishedFileError(file_path) - self.file_hash_bytes = calculate_sha384_file_hash(file_path) - return stream_type - - @property - def name(self) -> str: - return self.message.name - - @name.setter - def name(self, name: str): - self.message.name = name - - @property - def size(self) -> int: - return self.message.size - - @size.setter - def size(self, size: int): - self.message.size = size - - @property - def media_type(self) -> str: - return self.message.media_type - - @media_type.setter - def media_type(self, media_type: str): - self.message.media_type = media_type - - @property - def file_hash(self) -> str: - return hexlify(self.message.hash).decode() - - @file_hash.setter - def file_hash(self, file_hash: str): - self.message.hash = unhexlify(file_hash.encode()) - - @property - def file_hash_bytes(self) -> bytes: - return self.message.hash - - @file_hash_bytes.setter - def file_hash_bytes(self, file_hash_bytes: bytes): - self.message.hash = file_hash_bytes - - @property - def sd_hash(self) -> str: - return hexlify(self.message.sd_hash).decode() - - @sd_hash.setter - def sd_hash(self, sd_hash: str): - self.message.sd_hash = unhexlify(sd_hash.encode()) - - @property - def sd_hash_bytes(self) -> bytes: - return self.message.sd_hash - - @sd_hash_bytes.setter - def sd_hash_bytes(self, sd_hash: bytes): - self.message.sd_hash = sd_hash - - @property - def bt_infohash(self) -> str: - return hexlify(self.message.bt_infohash).decode() - - @bt_infohash.setter - def bt_infohash(self, bt_infohash: str): - self.message.bt_infohash = unhexlify(bt_infohash.encode()) - - @property - def bt_infohash_bytes(self) -> bytes: - return self.message.bt_infohash.decode() - - @bt_infohash_bytes.setter - def bt_infohash_bytes(self, bt_infohash: bytes): - self.message.bt_infohash = bt_infohash - - @property - def url(self) -> str: - return self.message.url - - @url.setter - def url(self, url: str): - self.message.url = url - - -class Fee(Metadata): - - __slots__ = () - - def update(self, address: str = None, currency: str = None, amount=None): - if amount: - currency = (currency or self.currency or '').lower() - if not currency: - raise Exception('In order to set a fee amount, please specify a fee currency.') - if currency not in ('lbc', 'btc', 'usd'): - raise Exception(f'Missing or unknown currency provided: {currency}') - setattr(self, currency, Decimal(amount)) - elif currency: - raise Exception('In order to set a fee currency, please specify a fee amount.') - if address: - if not self.currency: - raise Exception('In order to set a fee address, please specify a fee amount and currency.') - self.address = address - - @property - def currency(self) -> str: - if self.message.currency: - return FeeMessage.Currency.Name(self.message.currency) - - @property - def address(self) -> str: - if self.address_bytes: - return Base58.encode(self.address_bytes) - - @address.setter - def address(self, address: str): - self.address_bytes = Base58.decode(address) - - @property - def address_bytes(self) -> bytes: - return self.message.address - - @address_bytes.setter - def address_bytes(self, address: bytes): - self.message.address = address - - @property - def amount(self) -> Decimal: - if self.currency == 'LBC': - return self.lbc - if self.currency == 'BTC': - return self.btc - if self.currency == 'USD': - return self.usd - - DEWIES = Decimal(COIN) - - @property - def lbc(self) -> Decimal: - if self.message.currency != FeeMessage.LBC: - raise ValueError('LBC can only be returned for LBC fees.') - return Decimal(self.message.amount / self.DEWIES) - - @lbc.setter - def lbc(self, amount: Decimal): - self.dewies = int(amount * self.DEWIES) - - @property - def dewies(self) -> int: - if self.message.currency != FeeMessage.LBC: - raise ValueError('Dewies can only be returned for LBC fees.') - return self.message.amount - - @dewies.setter - def dewies(self, amount: int): - self.message.amount = amount - self.message.currency = FeeMessage.LBC - - SATOSHIES = Decimal(COIN) - - @property - def btc(self) -> Decimal: - if self.message.currency != FeeMessage.BTC: - raise ValueError('BTC can only be returned for BTC fees.') - return Decimal(self.message.amount / self.SATOSHIES) - - @btc.setter - def btc(self, amount: Decimal): - self.satoshis = int(amount * self.SATOSHIES) - - @property - def satoshis(self) -> int: - if self.message.currency != FeeMessage.BTC: - raise ValueError('Satoshies can only be returned for BTC fees.') - return self.message.amount - - @satoshis.setter - def satoshis(self, amount: int): - self.message.amount = amount - self.message.currency = FeeMessage.BTC - - PENNIES = Decimal('100.0') - PENNY = Decimal('0.01') - - @property - def usd(self) -> Decimal: - if self.message.currency != FeeMessage.USD: - raise ValueError('USD can only be returned for USD fees.') - return Decimal(self.message.amount / self.PENNIES) - - @usd.setter - def usd(self, amount: Decimal): - self.pennies = int(amount.quantize(self.PENNY, ROUND_UP) * self.PENNIES) - - @property - def pennies(self) -> int: - if self.message.currency != FeeMessage.USD: - raise ValueError('Pennies can only be returned for USD fees.') - return self.message.amount - - @pennies.setter - def pennies(self, amount: int): - self.message.amount = amount - self.message.currency = FeeMessage.USD - - -class ClaimReference(Metadata): - - __slots__ = () - - @property - def claim_id(self) -> str: - return hexlify(self.claim_hash[::-1]).decode() - - @claim_id.setter - def claim_id(self, claim_id: str): - self.claim_hash = unhexlify(claim_id)[::-1] - - @property - def claim_hash(self) -> bytes: - return self.message.claim_hash - - @claim_hash.setter - def claim_hash(self, claim_hash: bytes): - self.message.claim_hash = claim_hash - - -class ClaimList(BaseMessageList[ClaimReference]): - - __slots__ = () - item_class = ClaimReference - - @property - def _message(self): - return self.message.claim_references - - def append(self, value): - self.add().claim_id = value - - @property - def ids(self) -> List[str]: - return [c.claim_id for c in self] - - -class Language(Metadata): - - __slots__ = () - - @property - def langtag(self) -> str: - langtag = [] - if self.language: - langtag.append(self.language) - if self.script: - langtag.append(self.script) - if self.region: - langtag.append(self.region) - return '-'.join(langtag) - - @langtag.setter - def langtag(self, langtag: str): - parts = langtag.split('-') - self.language = parts.pop(0) - if parts and len(parts[0]) == 4: - self.script = parts.pop(0) - if parts and len(parts[0]) == 2 and parts[0].isalpha(): - self.region = parts.pop(0) - if parts and len(parts[0]) == 3 and parts[0].isdigit(): - self.region = parts.pop(0) - assert not parts, f"Failed to parse language tag: {langtag}" - - @property - def language(self) -> str: - if self.message.language: - return LanguageMessage.Language.Name(self.message.language) - - @language.setter - def language(self, language: str): - self.message.language = LanguageMessage.Language.Value(language) - - @property - def script(self) -> str: - if self.message.script: - return LanguageMessage.Script.Name(self.message.script) - - @script.setter - def script(self, script: str): - self.message.script = LanguageMessage.Script.Value(script) - - @property - def region(self) -> str: - if self.message.region: - return country_int_to_str(self.message.region) - - @region.setter - def region(self, region: str): - self.message.region = country_str_to_int(region) - - -class LanguageList(BaseMessageList[Language]): - __slots__ = () - item_class = Language - - def append(self, value: str): - self.add().langtag = value - - -class Location(Metadata): - - __slots__ = () - - def from_value(self, value): - if isinstance(value, str) and value.startswith('{'): - value = json.loads(value) - - if isinstance(value, dict): - for key, val in value.items(): - setattr(self, key, val) - - elif isinstance(value, str): - parts = value.split(':') - if len(parts) > 2 or (parts[0] and parts[0][0] in ascii_letters): - country = parts and parts.pop(0) - if country: - self.country = country - state = parts and parts.pop(0) - if state: - self.state = state - city = parts and parts.pop(0) - if city: - self.city = city - code = parts and parts.pop(0) - if code: - self.code = code - latitude = parts and parts.pop(0) - if latitude: - self.latitude = latitude - longitude = parts and parts.pop(0) - if longitude: - self.longitude = longitude - - else: - raise ValueError(f'Could not parse country value: {value}') - - def to_dict(self): - d = MessageToDict(self.message) - if self.message.longitude: - d['longitude'] = self.longitude - if self.message.latitude: - d['latitude'] = self.latitude - return d - - @property - def country(self) -> str: - if self.message.country: - return LocationMessage.Country.Name(self.message.country) - - @country.setter - def country(self, country: str): - self.message.country = LocationMessage.Country.Value(country) - - @property - def state(self) -> str: - return self.message.state - - @state.setter - def state(self, state: str): - self.message.state = state - - @property - def city(self) -> str: - return self.message.city - - @city.setter - def city(self, city: str): - self.message.city = city - - @property - def code(self) -> str: - return self.message.code - - @code.setter - def code(self, code: str): - self.message.code = code - - GPS_PRECISION = Decimal('10000000') - - @property - def latitude(self) -> str: - if self.message.latitude: - return str(Decimal(self.message.latitude) / self.GPS_PRECISION) - - @latitude.setter - def latitude(self, latitude: str): - latitude = Decimal(latitude) - assert -90 <= latitude <= 90, "Latitude must be between -90 and 90 degrees." - self.message.latitude = int(latitude * self.GPS_PRECISION) - - @property - def longitude(self) -> str: - if self.message.longitude: - return str(Decimal(self.message.longitude) / self.GPS_PRECISION) - - @longitude.setter - def longitude(self, longitude: str): - longitude = Decimal(longitude) - assert -180 <= longitude <= 180, "Longitude must be between -180 and 180 degrees." - self.message.longitude = int(longitude * self.GPS_PRECISION) - - -class LocationList(BaseMessageList[Location]): - __slots__ = () - item_class = Location - - def append(self, value): - self.add().from_value(value) - - -class TagList(BaseMessageList[str]): - __slots__ = () - item_class = str - - def append(self, tag: str): - tag = normalize_tag(tag) - if tag and tag not in self.message: - self.message.append(tag) diff --git a/lbry/schema/base.py b/lbry/schema/base.py deleted file mode 100644 index 6e9bab4e3..000000000 --- a/lbry/schema/base.py +++ /dev/null @@ -1,124 +0,0 @@ -from binascii import hexlify, unhexlify -from typing import List, Iterator, TypeVar, Generic - -from google.protobuf.message import DecodeError -from google.protobuf.json_format import MessageToDict - - -class Signable: - - __slots__ = ( - 'message', 'version', 'signature', - 'signature_type', 'unsigned_payload', 'signing_channel_hash' - ) - - message_class = None - - def __init__(self, message=None): - self.message = message or self.message_class() - self.version = 2 - self.signature = None - self.signature_type = 'SECP256k1' - self.unsigned_payload = None - self.signing_channel_hash = None - - def clear_signature(self): - self.signature = None - self.unsigned_payload = None - self.signing_channel_hash = None - - @property - def signing_channel_id(self): - return hexlify(self.signing_channel_hash[::-1]).decode() if self.signing_channel_hash else None - - @signing_channel_id.setter - def signing_channel_id(self, channel_id: str): - self.signing_channel_hash = unhexlify(channel_id)[::-1] - - @property - def is_signed(self): - return self.signature is not None - - def to_dict(self): - return MessageToDict(self.message) - - def to_message_bytes(self) -> bytes: - return self.message.SerializeToString() - - def to_bytes(self) -> bytes: - pieces = bytearray() - if self.is_signed: - pieces.append(1) - pieces.extend(self.signing_channel_hash) - pieces.extend(self.signature) - else: - pieces.append(0) - pieces.extend(self.to_message_bytes()) - return bytes(pieces) - - @classmethod - def from_bytes(cls, data: bytes): - signable = cls() - if data[0] == 0: - signable.message.ParseFromString(data[1:]) - elif data[0] == 1: - signable.signing_channel_hash = data[1:21] - signable.signature = data[21:85] - signable.message.ParseFromString(data[85:]) - else: - raise DecodeError('Could not determine message format version.') - return signable - - def __len__(self): - return len(self.to_bytes()) - - def __bytes__(self): - return self.to_bytes() - - -class Metadata: - - __slots__ = 'message', - - def __init__(self, message): - self.message = message - - -I = TypeVar('I') - - -class BaseMessageList(Metadata, Generic[I]): - - __slots__ = () - - item_class = None - - @property - def _message(self): - return self.message - - def add(self) -> I: - return self.item_class(self._message.add()) - - def extend(self, values: List[str]): - for value in values: - self.append(value) - - def append(self, value: str): - raise NotImplemented - - def __len__(self): - return len(self._message) - - def __iter__(self) -> Iterator[I]: - for item in self._message: - yield self.item_class(item) - - def __getitem__(self, item) -> I: - return self.item_class(self._message[item]) - - def __delitem__(self, key): - del self._message[key] - - def __eq__(self, other) -> bool: - return self._message == other diff --git a/lbry/schema/claim.py b/lbry/schema/claim.py deleted file mode 100644 index 53565c1e1..000000000 --- a/lbry/schema/claim.py +++ /dev/null @@ -1,422 +0,0 @@ -import logging -from typing import List -from binascii import hexlify, unhexlify - -from asn1crypto.keys import PublicKeyInfo -from coincurve import PublicKey as cPublicKey - -from google.protobuf.json_format import MessageToDict -from google.protobuf.message import DecodeError -from hachoir.core.log import log as hachoir_log -from hachoir.parser import createParser as binary_file_parser -from hachoir.metadata import extractMetadata as binary_file_metadata - -from lbry.schema import compat -from lbry.schema.base import Signable -from lbry.schema.mime_types import guess_media_type, guess_stream_type -from lbry.schema.attrs import ( - Source, Playable, Dimmensional, Fee, Image, Video, Audio, - LanguageList, LocationList, ClaimList, ClaimReference, TagList -) -from lbry.schema.types.v2.claim_pb2 import Claim as ClaimMessage -from lbry.error import InputValueIsNoneError - - -hachoir_log.use_print = False -log = logging.getLogger(__name__) - - -class Claim(Signable): - - STREAM = 'stream' - CHANNEL = 'channel' - COLLECTION = 'collection' - REPOST = 'repost' - - __slots__ = () - - message_class = ClaimMessage - - @property - def claim_type(self) -> str: - return self.message.WhichOneof('type') - - def get_message(self, type_name): - message = getattr(self.message, type_name) - if self.claim_type is None: - message.SetInParent() - if self.claim_type != type_name: - raise ValueError(f'Claim is not a {type_name}.') - return message - - @property - def is_stream(self): - return self.claim_type == self.STREAM - - @property - def stream(self) -> 'Stream': - return Stream(self) - - @property - def is_channel(self): - return self.claim_type == self.CHANNEL - - @property - def channel(self) -> 'Channel': - return Channel(self) - - @property - def is_repost(self): - return self.claim_type == self.REPOST - - @property - def repost(self) -> 'Repost': - return Repost(self) - - @property - def is_collection(self): - return self.claim_type == self.COLLECTION - - @property - def collection(self) -> 'Collection': - return Collection(self) - - @classmethod - def from_bytes(cls, data: bytes) -> 'Claim': - try: - return super().from_bytes(data) - except DecodeError: - claim = cls() - if data[0] == ord('{'): - claim.version = 0 - compat.from_old_json_schema(claim, data) - elif data[0] not in (0, 1): - claim.version = 1 - compat.from_types_v1(claim, data) - else: - raise - return claim - - -class BaseClaim: - - __slots__ = 'claim', 'message' - - claim_type = None - object_fields = 'thumbnail', - repeat_fields = 'tags', 'languages', 'locations' - - def __init__(self, claim: Claim = None): - self.claim = claim or Claim() - self.message = self.claim.get_message(self.claim_type) - - def to_dict(self): - claim = MessageToDict(self.claim.message, preserving_proto_field_name=True) - claim.update(claim.pop(self.claim_type)) - if 'languages' in claim: - claim['languages'] = self.langtags - if 'locations' in claim: - claim['locations'] = [l.to_dict() for l in self.locations] - return claim - - def none_check(self, kwargs): - for key, value in kwargs.items(): - if value is None: - raise InputValueIsNoneError(key) - - def update(self, **kwargs): - self.none_check(kwargs) - - for key in list(kwargs): - for field in self.object_fields: - if key.startswith(f'{field}_'): - attr = getattr(self, field) - setattr(attr, key[len(f'{field}_'):], kwargs.pop(key)) - continue - - for l in self.repeat_fields: - field = getattr(self, l) - if kwargs.pop(f'clear_{l}', False): - del field[:] - items = kwargs.pop(l, None) - if items is not None: - if isinstance(items, str): - field.append(items) - elif isinstance(items, list): - field.extend(items) - else: - raise ValueError(f"Unknown {l} value: {items}") - - for key, value in kwargs.items(): - setattr(self, key, value) - - @property - def title(self) -> str: - return self.claim.message.title - - @title.setter - def title(self, title: str): - self.claim.message.title = title - - @property - def description(self) -> str: - return self.claim.message.description - - @description.setter - def description(self, description: str): - self.claim.message.description = description - - @property - def thumbnail(self) -> Source: - return Source(self.claim.message.thumbnail) - - @property - def tags(self) -> List[str]: - return TagList(self.claim.message.tags) - - @property - def languages(self) -> LanguageList: - return LanguageList(self.claim.message.languages) - - @property - def langtags(self) -> List[str]: - return [l.langtag for l in self.languages] - - @property - def locations(self) -> LocationList: - return LocationList(self.claim.message.locations) - - -class Stream(BaseClaim): - - __slots__ = () - - claim_type = Claim.STREAM - - object_fields = BaseClaim.object_fields + ('source',) - - def to_dict(self): - claim = super().to_dict() - if 'source' in claim: - if 'hash' in claim['source']: - claim['source']['hash'] = self.source.file_hash - if 'sd_hash' in claim['source']: - claim['source']['sd_hash'] = self.source.sd_hash - elif 'bt_infohash' in claim['source']: - claim['source']['bt_infohash'] = self.source.bt_infohash - if 'media_type' in claim['source']: - claim['stream_type'] = guess_stream_type(claim['source']['media_type']) - fee = claim.get('fee', {}) - if 'address' in fee: - fee['address'] = self.fee.address - if 'amount' in fee: - fee['amount'] = str(self.fee.amount) - return claim - - def update(self, file_path=None, height=None, width=None, duration=None, **kwargs): - - if kwargs.pop('clear_fee', False): - self.message.ClearField('fee') - else: - self.fee.update( - kwargs.pop('fee_address', None), - kwargs.pop('fee_currency', None), - kwargs.pop('fee_amount', None) - ) - - self.none_check(kwargs) - - if 'sd_hash' in kwargs: - self.source.sd_hash = kwargs.pop('sd_hash') - elif 'bt_infohash' in kwargs: - self.source.bt_infohash = kwargs.pop('bt_infohash') - if 'file_name' in kwargs: - self.source.name = kwargs.pop('file_name') - if 'file_hash' in kwargs: - self.source.file_hash = kwargs.pop('file_hash') - - stream_type = None - if file_path is not None: - stream_type = self.source.update(file_path=file_path) - elif self.source.name: - self.source.media_type, stream_type = guess_media_type(self.source.name) - elif self.source.media_type: - stream_type = guess_stream_type(self.source.media_type) - - if 'file_size' in kwargs: - self.source.size = kwargs.pop('file_size') - - if self.stream_type is not None and self.stream_type != stream_type: - self.message.ClearField(self.stream_type) - - if stream_type in ('image', 'video', 'audio'): - media = getattr(self, stream_type) - media_args = {'file_metadata': None} - if file_path is not None and not all((duration, width, height)): - try: - media_args['file_metadata'] = binary_file_metadata(binary_file_parser(file_path)) - except: - log.exception('Could not read file metadata.') - if isinstance(media, Playable): - media_args['duration'] = duration - if isinstance(media, Dimmensional): - media_args['height'] = height - media_args['width'] = width - media.update(**media_args) - - super().update(**kwargs) - - @property - def author(self) -> str: - return self.message.author - - @author.setter - def author(self, author: str): - self.message.author = author - - @property - def license(self) -> str: - return self.message.license - - @license.setter - def license(self, license: str): - self.message.license = license - - @property - def license_url(self) -> str: - return self.message.license_url - - @license_url.setter - def license_url(self, license_url: str): - self.message.license_url = license_url - - @property - def release_time(self) -> int: - return self.message.release_time - - @release_time.setter - def release_time(self, release_time: int): - self.message.release_time = release_time - - @property - def fee(self) -> Fee: - return Fee(self.message.fee) - - @property - def has_fee(self) -> bool: - return self.message.HasField('fee') - - @property - def has_source(self) -> bool: - return self.message.HasField('source') - - @property - def source(self) -> Source: - return Source(self.message.source) - - @property - def stream_type(self) -> str: - return self.message.WhichOneof('type') - - @property - def image(self) -> Image: - return Image(self.message.image) - - @property - def video(self) -> Video: - return Video(self.message.video) - - @property - def audio(self) -> Audio: - return Audio(self.message.audio) - - -class Channel(BaseClaim): - - __slots__ = () - - claim_type = Claim.CHANNEL - - object_fields = BaseClaim.object_fields + ('cover',) - repeat_fields = BaseClaim.repeat_fields + ('featured',) - - def to_dict(self): - claim = super().to_dict() - claim['public_key'] = self.public_key - if 'featured' in claim: - claim['featured'] = self.featured.ids - return claim - - @property - def public_key(self) -> str: - return hexlify(self.public_key_bytes).decode() - - @public_key.setter - def public_key(self, sd_public_key: str): - self.message.public_key = unhexlify(sd_public_key.encode()) - - @property - def public_key_bytes(self) -> bytes: - if len(self.message.public_key) == 33: - return self.message.public_key - public_key_info = PublicKeyInfo.load(self.message.public_key) - public_key = cPublicKey(public_key_info.native['public_key']) - return public_key.format(compressed=True) - - @public_key_bytes.setter - def public_key_bytes(self, public_key: bytes): - self.message.public_key = public_key - - @property - def email(self) -> str: - return self.message.email - - @email.setter - def email(self, email: str): - self.message.email = email - - @property - def website_url(self) -> str: - return self.message.website_url - - @website_url.setter - def website_url(self, website_url: str): - self.message.website_url = website_url - - @property - def cover(self) -> Source: - return Source(self.message.cover) - - @property - def featured(self) -> ClaimList: - return ClaimList(self.message.featured) - - -class Repost(BaseClaim): - - __slots__ = () - - claim_type = Claim.REPOST - - @property - def reference(self) -> ClaimReference: - return ClaimReference(self.message) - - -class Collection(BaseClaim): - - __slots__ = () - - claim_type = Claim.COLLECTION - - repeat_fields = BaseClaim.repeat_fields + ('claims',) - - def to_dict(self): - claim = super().to_dict() - if claim.pop('claim_references', None): - claim['claims'] = self.claims.ids - return claim - - @property - def claims(self) -> ClaimList: - return ClaimList(self.message) diff --git a/lbry/schema/compat.py b/lbry/schema/compat.py deleted file mode 100644 index 7a4143dfa..000000000 --- a/lbry/schema/compat.py +++ /dev/null @@ -1,93 +0,0 @@ -import json -from decimal import Decimal - -from google.protobuf.message import DecodeError - -from lbry.schema.types.v1.legacy_claim_pb2 import Claim as OldClaimMessage -from lbry.schema.types.v1.certificate_pb2 import KeyType -from lbry.schema.types.v1.fee_pb2 import Fee as FeeMessage - - -def from_old_json_schema(claim, payload: bytes): - try: - value = json.loads(payload) - except: - raise DecodeError('Could not parse JSON.') - stream = claim.stream - stream.source.sd_hash = value['sources']['lbry_sd_hash'] - stream.source.media_type = ( - value.get('content_type', value.get('content-type')) or - 'application/octet-stream' - ) - stream.title = value.get('title', '') - stream.description = value.get('description', '') - if value.get('thumbnail', ''): - stream.thumbnail.url = value.get('thumbnail', '') - stream.author = value.get('author', '') - stream.license = value.get('license', '') - stream.license_url = value.get('license_url', '') - language = value.get('language', '') - if language: - if language.lower() == 'english': - language = 'en' - try: - stream.languages.append(language) - except: - pass - if value.get('nsfw', False): - stream.tags.append('mature') - if "fee" in value and isinstance(value['fee'], dict): - fee = value["fee"] - currency = list(fee.keys())[0] - if currency == 'LBC': - stream.fee.lbc = Decimal(fee[currency]['amount']) - elif currency == 'USD': - stream.fee.usd = Decimal(fee[currency]['amount']) - elif currency == 'BTC': - stream.fee.btc = Decimal(fee[currency]['amount']) - else: - raise DecodeError(f'Unknown currency: {currency}') - stream.fee.address = fee[currency]['address'] - return claim - - -def from_types_v1(claim, payload: bytes): - old = OldClaimMessage() - old.ParseFromString(payload) - if old.claimType == 2: - channel = claim.channel - channel.public_key_bytes = old.certificate.publicKey - else: - stream = claim.stream - stream.title = old.stream.metadata.title - stream.description = old.stream.metadata.description - stream.author = old.stream.metadata.author - stream.license = old.stream.metadata.license - stream.license_url = old.stream.metadata.licenseUrl - stream.thumbnail.url = old.stream.metadata.thumbnail - if old.stream.metadata.HasField('language'): - stream.languages.add().message.language = old.stream.metadata.language - stream.source.media_type = old.stream.source.contentType - stream.source.sd_hash_bytes = old.stream.source.source - if old.stream.metadata.nsfw: - stream.tags.append('mature') - if old.stream.metadata.HasField('fee'): - fee = old.stream.metadata.fee - stream.fee.address_bytes = fee.address - currency = FeeMessage.Currency.Name(fee.currency) - if currency == 'LBC': - stream.fee.lbc = Decimal(fee.amount) - elif currency == 'USD': - stream.fee.usd = Decimal(fee.amount) - elif currency == 'BTC': - stream.fee.btc = Decimal(fee.amount) - else: - raise DecodeError(f'Unsupported currency: {currency}') - if old.HasField('publisherSignature'): - sig = old.publisherSignature - claim.signature = sig.signature - claim.signature_type = KeyType.Name(sig.signatureType) - claim.signing_channel_hash = sig.certificateId[::-1] - old.ClearField("publisherSignature") - claim.unsigned_payload = old.SerializeToString() - return claim diff --git a/lbry/schema/mime_types.py b/lbry/schema/mime_types.py deleted file mode 100644 index 62505be04..000000000 --- a/lbry/schema/mime_types.py +++ /dev/null @@ -1,214 +0,0 @@ -import os -import filetype -import logging - -types_map = { - # http://www.iana.org/assignments/media-types - # Type mapping for automated metadata extraction (video, audio, image, document, binary, model) - '.a': ('application/octet-stream', 'binary'), - '.ai': ('application/postscript', 'image'), - '.aif': ('audio/x-aiff', 'audio'), - '.aifc': ('audio/x-aiff', 'audio'), - '.aiff': ('audio/x-aiff', 'audio'), - '.au': ('audio/basic', 'audio'), - '.avi': ('video/x-msvideo', 'video'), - '.bat': ('text/plain', 'document'), - '.bcpio': ('application/x-bcpio', 'binary'), - '.bin': ('application/octet-stream', 'binary'), - '.bmp': ('image/bmp', 'image'), - '.c': ('text/plain', 'document'), - '.cdf': ('application/x-netcdf', 'binary'), - '.cpio': ('application/x-cpio', 'binary'), - '.csh': ('application/x-csh', 'binary'), - '.css': ('text/css', 'document'), - '.csv': ('text/csv', 'document'), - '.dll': ('application/octet-stream', 'binary'), - '.doc': ('application/msword', 'document'), - '.dot': ('application/msword', 'document'), - '.dvi': ('application/x-dvi', 'binary'), - '.eml': ('message/rfc822', 'document'), - '.eps': ('application/postscript', 'document'), - '.epub': ('application/epub+zip', 'document'), - '.etx': ('text/x-setext', 'document'), - '.exe': ('application/octet-stream', 'binary'), - '.gif': ('image/gif', 'image'), - '.gtar': ('application/x-gtar', 'binary'), - '.h': ('text/plain', 'document'), - '.hdf': ('application/x-hdf', 'binary'), - '.htm': ('text/html', 'document'), - '.html': ('text/html', 'document'), - '.ico': ('image/vnd.microsoft.icon', 'image'), - '.ief': ('image/ief', 'image'), - '.iges': ('model/iges', 'model'), - '.jpe': ('image/jpeg', 'image'), - '.jpeg': ('image/jpeg', 'image'), - '.jpg': ('image/jpeg', 'image'), - '.js': ('application/javascript', 'document'), - '.json': ('application/json', 'document'), - '.ksh': ('text/plain', 'document'), - '.latex': ('application/x-latex', 'binary'), - '.m1v': ('video/mpeg', 'video'), - '.m3u': ('application/x-mpegurl', 'audio'), - '.m3u8': ('application/x-mpegurl', 'video'), - '.man': ('application/x-troff-man', 'document'), - '.markdown': ('text/markdown', 'document'), - '.md': ('text/markdown', 'document'), - '.me': ('application/x-troff-me', 'binary'), - '.mht': ('message/rfc822', 'document'), - '.mhtml': ('message/rfc822', 'document'), - '.mif': ('application/x-mif', 'binary'), - '.mov': ('video/quicktime', 'video'), - '.movie': ('video/x-sgi-movie', 'video'), - '.mp2': ('audio/mpeg', 'audio'), - '.mp3': ('audio/mpeg', 'audio'), - '.mp4': ('video/mp4', 'video'), - '.mpa': ('video/mpeg', 'video'), - '.mpd': ('application/dash+xml', 'video'), - '.mpe': ('video/mpeg', 'video'), - '.mpeg': ('video/mpeg', 'video'), - '.mpg': ('video/mpeg', 'video'), - '.ms': ('application/x-troff-ms', 'binary'), - '.m4s': ('video/iso.segment', 'binary'), - '.nc': ('application/x-netcdf', 'binary'), - '.nws': ('message/rfc822', 'document'), - '.o': ('application/octet-stream', 'binary'), - '.obj': ('application/octet-stream', 'model'), - '.oda': ('application/oda', 'binary'), - '.p12': ('application/x-pkcs12', 'binary'), - '.p7c': ('application/pkcs7-mime', 'binary'), - '.pbm': ('image/x-portable-bitmap', 'image'), - '.pdf': ('application/pdf', 'document'), - '.pfx': ('application/x-pkcs12', 'binary'), - '.pgm': ('image/x-portable-graymap', 'image'), - '.pl': ('text/plain', 'document'), - '.png': ('image/png', 'image'), - '.pnm': ('image/x-portable-anymap', 'image'), - '.pot': ('application/vnd.ms-powerpoint', 'document'), - '.ppa': ('application/vnd.ms-powerpoint', 'document'), - '.ppm': ('image/x-portable-pixmap', 'image'), - '.pps': ('application/vnd.ms-powerpoint', 'document'), - '.ppt': ('application/vnd.ms-powerpoint', 'document'), - '.ps': ('application/postscript', 'document'), - '.pwz': ('application/vnd.ms-powerpoint', 'document'), - '.py': ('text/x-python', 'document'), - '.pyc': ('application/x-python-code', 'binary'), - '.pyo': ('application/x-python-code', 'binary'), - '.qt': ('video/quicktime', 'video'), - '.ra': ('audio/x-pn-realaudio', 'audio'), - '.ram': ('application/x-pn-realaudio', 'audio'), - '.ras': ('image/x-cmu-raster', 'image'), - '.rdf': ('application/xml', 'binary'), - '.rgb': ('image/x-rgb', 'image'), - '.roff': ('application/x-troff', 'binary'), - '.rtx': ('text/richtext', 'document'), - '.sgm': ('text/x-sgml', 'document'), - '.sgml': ('text/x-sgml', 'document'), - '.sh': ('application/x-sh', 'document'), - '.shar': ('application/x-shar', 'binary'), - '.snd': ('audio/basic', 'audio'), - '.so': ('application/octet-stream', 'binary'), - '.src': ('application/x-wais-source', 'binary'), - '.stl': ('model/stl', 'model'), - '.sv4cpio': ('application/x-sv4cpio', 'binary'), - '.sv4crc': ('application/x-sv4crc', 'binary'), - '.svg': ('image/svg+xml', 'image'), - '.swf': ('application/x-shockwave-flash', 'binary'), - '.t': ('application/x-troff', 'binary'), - '.tar': ('application/x-tar', 'binary'), - '.tcl': ('application/x-tcl', 'binary'), - '.tex': ('application/x-tex', 'binary'), - '.texi': ('application/x-texinfo', 'binary'), - '.texinfo': ('application/x-texinfo', 'binary'), - '.tif': ('image/tiff', 'image'), - '.tiff': ('image/tiff', 'image'), - '.tr': ('application/x-troff', 'binary'), - '.ts': ('video/mp2t', 'video'), - '.tsv': ('text/tab-separated-values', 'document'), - '.txt': ('text/plain', 'document'), - '.ustar': ('application/x-ustar', 'binary'), - '.vcf': ('text/x-vcard', 'document'), - '.vtt': ('text/vtt', 'document'), - '.wav': ('audio/x-wav', 'audio'), - '.webm': ('video/webm', 'video'), - '.wiz': ('application/msword', 'document'), - '.wsdl': ('application/xml', 'document'), - '.xbm': ('image/x-xbitmap', 'image'), - '.xlb': ('application/vnd.ms-excel', 'document'), - '.xls': ('application/vnd.ms-excel', 'document'), - '.xml': ('text/xml', 'document'), - '.xpdl': ('application/xml', 'document'), - '.xpm': ('image/x-xpixmap', 'image'), - '.xsl': ('application/xml', 'document'), - '.xwd': ('image/x-xwindowdump', 'image'), - '.zip': ('application/zip', 'binary'), - - # These are non-standard types, commonly found in the wild. - '.cbr': ('application/vnd.comicbook-rar', 'document'), - '.cbz': ('application/vnd.comicbook+zip', 'document'), - '.flac': ('audio/flac', 'audio'), - '.lbry': ('application/x-ext-lbry', 'document'), - '.m4a': ('audio/mp4', 'audio'), - '.m4v': ('video/m4v', 'video'), - '.mid': ('audio/midi', 'audio'), - '.midi': ('audio/midi', 'audio'), - '.mkv': ('video/x-matroska', 'video'), - '.mobi': ('application/x-mobipocket-ebook', 'document'), - '.oga': ('audio/ogg', 'audio'), - '.ogv': ('video/ogg', 'video'), - '.ogg': ('video/ogg', 'video'), - '.pct': ('image/pict', 'image'), - '.pic': ('image/pict', 'image'), - '.pict': ('image/pict', 'image'), - '.prc': ('application/x-mobipocket-ebook', 'document'), - '.rtf': ('application/rtf', 'document'), - '.xul': ('text/xul', 'document'), - - # microsoft is special and has its own 'standard' - # https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions - '.wmv': ('video/x-ms-wmv', 'video') -} - -# maps detected extensions to the possible analogs -# i.e. .cbz file is actually a .zip -synonyms_map = { - '.zip': ['.cbz'], - '.rar': ['.cbr'], - '.ar': ['.a'] -} - -log = logging.getLogger(__name__) - - -def guess_media_type(path): - _, ext = os.path.splitext(path) - extension = ext.strip().lower() - - try: - kind = filetype.guess(path) - if kind: - real_extension = f".{kind.extension}" - - if extension != real_extension: - if extension: - log.warning(f"file extension does not match it's contents: {path}, identified as {real_extension}") - else: - log.debug(f"file {path} does not have extension, identified by it's contents as {real_extension}") - - if extension not in synonyms_map.get(real_extension, []): - extension = real_extension - - except OSError as error: - pass - - if extension[1:]: - if extension in types_map: - return types_map[extension] - return f'application/x-ext-{extension[1:]}', 'binary' - return 'application/octet-stream', 'binary' - - -def guess_stream_type(media_type): - for media, stream in types_map.values(): - if media == media_type: - return stream - return 'binary' diff --git a/lbry/schema/purchase.py b/lbry/schema/purchase.py deleted file mode 100644 index 880fa76bc..000000000 --- a/lbry/schema/purchase.py +++ /dev/null @@ -1,47 +0,0 @@ -from google.protobuf.message import DecodeError -from google.protobuf.json_format import MessageToDict -from lbry.schema.types.v2.purchase_pb2 import Purchase as PurchaseMessage -from .attrs import ClaimReference - - -class Purchase(ClaimReference): - - START_BYTE = ord('P') - - __slots__ = () - - def __init__(self, claim_id=None): - super().__init__(PurchaseMessage()) - if claim_id is not None: - self.claim_id = claim_id - - def to_dict(self): - return MessageToDict(self.message) - - def to_message_bytes(self) -> bytes: - return self.message.SerializeToString() - - def to_bytes(self) -> bytes: - pieces = bytearray() - pieces.append(self.START_BYTE) - pieces.extend(self.to_message_bytes()) - return bytes(pieces) - - @classmethod - def has_start_byte(cls, data: bytes): - return data and data[0] == cls.START_BYTE - - @classmethod - def from_bytes(cls, data: bytes): - purchase = cls() - if purchase.has_start_byte(data): - purchase.message.ParseFromString(data[1:]) - else: - raise DecodeError('Message does not start with correct byte.') - return purchase - - def __len__(self): - return len(self.to_bytes()) - - def __bytes__(self): - return self.to_bytes() diff --git a/lbry/schema/result.py b/lbry/schema/result.py deleted file mode 100644 index 399aeabdf..000000000 --- a/lbry/schema/result.py +++ /dev/null @@ -1,256 +0,0 @@ -import base64 -from typing import List, Union, Optional, NamedTuple -from binascii import hexlify -from itertools import chain - -from lbry.error import ResolveCensoredError -from lbry.schema.types.v2.result_pb2 import Outputs as OutputsMessage -from lbry.schema.types.v2.result_pb2 import Error as ErrorMessage - -INVALID = ErrorMessage.Code.Name(ErrorMessage.INVALID) -NOT_FOUND = ErrorMessage.Code.Name(ErrorMessage.NOT_FOUND) -BLOCKED = ErrorMessage.Code.Name(ErrorMessage.BLOCKED) - - -def set_reference(reference, claim_hash, rows): - if claim_hash: - for txo in rows: - if claim_hash == txo.claim_hash: - reference.tx_hash = txo.tx_hash - reference.nout = txo.position - reference.height = txo.height - return - - -class Censor: - - NOT_CENSORED = 0 - SEARCH = 1 - RESOLVE = 2 - - __slots__ = 'censor_type', 'censored' - - def __init__(self, censor_type): - self.censor_type = censor_type - self.censored = {} - - def is_censored(self, row): - return (row.get('censor_type') or self.NOT_CENSORED) >= self.censor_type - - def apply(self, rows): - return [row for row in rows if not self.censor(row)] - - def censor(self, row) -> Optional[bytes]: - if self.is_censored(row): - censoring_channel_hash = bytes.fromhex(row['censoring_channel_id'])[::-1] - self.censored.setdefault(censoring_channel_hash, set()) - self.censored[censoring_channel_hash].add(row['tx_hash']) - return censoring_channel_hash - return None - - def to_message(self, outputs: OutputsMessage, extra_txo_rows: dict): - for censoring_channel_hash, count in self.censored.items(): - blocked = outputs.blocked.add() - blocked.count = len(count) - set_reference(blocked.channel, censoring_channel_hash, extra_txo_rows) - outputs.blocked_total += len(count) - - -class ResolveResult(NamedTuple): # a named tuple returned by the database containing all the resolve fields - name: str - normalized_name: str - claim_hash: bytes - tx_num: int - position: int - tx_hash: bytes - height: int - amount: int - short_url: str - is_controlling: bool - canonical_url: str - creation_height: int - activation_height: int - expiration_height: int - effective_amount: int - support_amount: int - reposted: int - last_takeover_height: Optional[int] - claims_in_channel: Optional[int] - channel_hash: Optional[bytes] - reposted_claim_hash: Optional[bytes] - signature_valid: Optional[bool] - - - -class Outputs: - - __slots__ = 'txos', 'extra_txos', 'txs', 'offset', 'total', 'blocked', 'blocked_total' - - def __init__(self, txos: List, extra_txos: List, txs: set, - offset: int, total: int, blocked: List, blocked_total: int): - self.txos = txos - self.txs = txs - self.extra_txos = extra_txos - self.offset = offset - self.total = total - self.blocked = blocked - self.blocked_total = blocked_total - - def inflate(self, txs): - tx_map = {tx.hash: tx for tx in txs} - for txo_message in self.extra_txos: - self.message_to_txo(txo_message, tx_map) - txos = [self.message_to_txo(txo_message, tx_map) for txo_message in self.txos] - return txos, self.inflate_blocked(tx_map) - - def inflate_blocked(self, tx_map): - return { - "total": self.blocked_total, - "channels": [{ - 'channel': self.message_to_txo(blocked.channel, tx_map), - 'blocked': blocked.count - } for blocked in self.blocked] - } - - def message_to_txo(self, txo_message, tx_map): - if txo_message.WhichOneof('meta') == 'error': - error = { - 'error': { - 'name': txo_message.error.Code.Name(txo_message.error.code), - 'text': txo_message.error.text, - } - } - if error['error']['name'] == BLOCKED: - error['error']['censor'] = self.message_to_txo( - txo_message.error.blocked.channel, tx_map - ) - return error - - tx = tx_map.get(txo_message.tx_hash) - if not tx: - return - txo = tx.outputs[txo_message.nout] - if txo_message.WhichOneof('meta') == 'claim': - claim = txo_message.claim - txo.meta = { - 'short_url': f'lbry://{claim.short_url}', - 'canonical_url': f'lbry://{claim.canonical_url or claim.short_url}', - 'reposted': claim.reposted, - 'is_controlling': claim.is_controlling, - 'take_over_height': claim.take_over_height, - 'creation_height': claim.creation_height, - 'activation_height': claim.activation_height, - 'expiration_height': claim.expiration_height, - 'effective_amount': claim.effective_amount, - 'support_amount': claim.support_amount, - # 'trending_group': claim.trending_group, - # 'trending_mixed': claim.trending_mixed, - # 'trending_local': claim.trending_local, - # 'trending_global': claim.trending_global, - } - if claim.HasField('channel'): - txo.channel = tx_map[claim.channel.tx_hash].outputs[claim.channel.nout] - if claim.HasField('repost'): - txo.reposted_claim = tx_map[claim.repost.tx_hash].outputs[claim.repost.nout] - try: - if txo.claim.is_channel: - txo.meta['claims_in_channel'] = claim.claims_in_channel - except: - pass - return txo - - @classmethod - def from_base64(cls, data: str) -> 'Outputs': - return cls.from_bytes(base64.b64decode(data)) - - @classmethod - def from_bytes(cls, data: bytes) -> 'Outputs': - outputs = OutputsMessage() - outputs.ParseFromString(data) - txs = set() - for txo_message in chain(outputs.txos, outputs.extra_txos): - if txo_message.WhichOneof('meta') == 'error': - continue - txs.add((hexlify(txo_message.tx_hash[::-1]).decode(), txo_message.height)) - return cls( - outputs.txos, outputs.extra_txos, txs, - outputs.offset, outputs.total, - outputs.blocked, outputs.blocked_total - ) - - @classmethod - def from_grpc(cls, outputs: OutputsMessage) -> 'Outputs': - txs = set() - for txo_message in chain(outputs.txos, outputs.extra_txos): - if txo_message.WhichOneof('meta') == 'error': - continue - txs.add((hexlify(txo_message.tx_hash[::-1]).decode(), txo_message.height)) - return cls( - outputs.txos, outputs.extra_txos, txs, - outputs.offset, outputs.total, - outputs.blocked, outputs.blocked_total - ) - - @classmethod - def to_base64(cls, txo_rows, extra_txo_rows, offset=0, total=None, blocked=None) -> str: - return base64.b64encode(cls.to_bytes(txo_rows, extra_txo_rows, offset, total, blocked)).decode() - - @classmethod - def to_bytes(cls, txo_rows, extra_txo_rows, offset=0, total=None, blocked: Censor = None) -> bytes: - page = OutputsMessage() - page.offset = offset - if total is not None: - page.total = total - if blocked is not None: - blocked.to_message(page, extra_txo_rows) - for row in extra_txo_rows: - txo_message: 'OutputsMessage' = page.extra_txos.add() - if not isinstance(row, Exception): - if row.channel_hash: - set_reference(txo_message.claim.channel, row.channel_hash, extra_txo_rows) - if row.reposted_claim_hash: - set_reference(txo_message.claim.repost, row.reposted_claim_hash, extra_txo_rows) - cls.encode_txo(txo_message, row) - - for row in txo_rows: - # cls.row_to_message(row, page.txos.add(), extra_txo_rows) - txo_message: 'OutputsMessage' = page.txos.add() - cls.encode_txo(txo_message, row) - if not isinstance(row, Exception): - if row.channel_hash: - set_reference(txo_message.claim.channel, row.channel_hash, extra_txo_rows) - if row.reposted_claim_hash: - set_reference(txo_message.claim.repost, row.reposted_claim_hash, extra_txo_rows) - elif isinstance(row, ResolveCensoredError): - set_reference(txo_message.error.blocked.channel, row.censor_id, extra_txo_rows) - return page.SerializeToString() - - @classmethod - def encode_txo(cls, txo_message, resolve_result: Union[ResolveResult, Exception]): - if isinstance(resolve_result, Exception): - txo_message.error.text = resolve_result.args[0] - if isinstance(resolve_result, ValueError): - txo_message.error.code = ErrorMessage.INVALID - elif isinstance(resolve_result, LookupError): - txo_message.error.code = ErrorMessage.NOT_FOUND - elif isinstance(resolve_result, ResolveCensoredError): - txo_message.error.code = ErrorMessage.BLOCKED - return - txo_message.tx_hash = resolve_result.tx_hash - txo_message.nout = resolve_result.position - txo_message.height = resolve_result.height - txo_message.claim.short_url = resolve_result.short_url - txo_message.claim.reposted = resolve_result.reposted - txo_message.claim.is_controlling = resolve_result.is_controlling - txo_message.claim.creation_height = resolve_result.creation_height - txo_message.claim.activation_height = resolve_result.activation_height - txo_message.claim.expiration_height = resolve_result.expiration_height - txo_message.claim.effective_amount = resolve_result.effective_amount - txo_message.claim.support_amount = resolve_result.support_amount - - if resolve_result.canonical_url is not None: - txo_message.claim.canonical_url = resolve_result.canonical_url - if resolve_result.last_takeover_height is not None: - txo_message.claim.take_over_height = resolve_result.last_takeover_height - if resolve_result.claims_in_channel is not None: - txo_message.claim.claims_in_channel = resolve_result.claims_in_channel diff --git a/lbry/schema/support.py b/lbry/schema/support.py deleted file mode 100644 index 9a9bb599d..000000000 --- a/lbry/schema/support.py +++ /dev/null @@ -1,23 +0,0 @@ -from lbry.schema.base import Signable -from lbry.schema.types.v2.support_pb2 import Support as SupportMessage - - -class Support(Signable): - __slots__ = () - message_class = SupportMessage - - @property - def emoji(self) -> str: - return self.message.emoji - - @emoji.setter - def emoji(self, emoji: str): - self.message.emoji = emoji - - @property - def comment(self) -> str: - return self.message.comment - - @comment.setter - def comment(self, comment: str): - self.message.comment = comment diff --git a/lbry/schema/tags.py b/lbry/schema/tags.py deleted file mode 100644 index 2d3cf79bc..000000000 --- a/lbry/schema/tags.py +++ /dev/null @@ -1,13 +0,0 @@ -from typing import List -import re - -MULTI_SPACE_RE = re.compile(r"\s{2,}") -WEIRD_CHARS_RE = re.compile(r"[#!~]") - - -def normalize_tag(tag: str): - return MULTI_SPACE_RE.sub(' ', WEIRD_CHARS_RE.sub(' ', tag.lower().replace("'", ""))).strip() - - -def clean_tags(tags: List[str]): - return [tag for tag in {normalize_tag(tag) for tag in tags} if tag] diff --git a/lbry/schema/types/__init__.py b/lbry/schema/types/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/lbry/schema/types/v1/__init__.py b/lbry/schema/types/v1/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/lbry/schema/types/v1/certificate_pb2.py b/lbry/schema/types/v1/certificate_pb2.py deleted file mode 100644 index d42df9042..000000000 --- a/lbry/schema/types/v1/certificate_pb2.py +++ /dev/null @@ -1,146 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: certificate.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf.internal import enum_type_wrapper -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='certificate.proto', - package='legacy_pb', - syntax='proto2', - serialized_options=None, - serialized_pb=_b('\n\x11\x63\x65rtificate.proto\x12\tlegacy_pb\"\xa2\x01\n\x0b\x43\x65rtificate\x12/\n\x07version\x18\x01 \x02(\x0e\x32\x1e.legacy_pb.Certificate.Version\x12#\n\x07keyType\x18\x02 \x02(\x0e\x32\x12.legacy_pb.KeyType\x12\x11\n\tpublicKey\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01*Q\n\x07KeyType\x12\x1b\n\x17UNKNOWN_PUBLIC_KEY_TYPE\x10\x00\x12\x0c\n\x08NIST256p\x10\x01\x12\x0c\n\x08NIST384p\x10\x02\x12\r\n\tSECP256k1\x10\x03') -) - -_KEYTYPE = _descriptor.EnumDescriptor( - name='KeyType', - full_name='legacy_pb.KeyType', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='UNKNOWN_PUBLIC_KEY_TYPE', index=0, number=0, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='NIST256p', index=1, number=1, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='NIST384p', index=2, number=2, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SECP256k1', index=3, number=3, - serialized_options=None, - type=None), - ], - containing_type=None, - serialized_options=None, - serialized_start=197, - serialized_end=278, -) -_sym_db.RegisterEnumDescriptor(_KEYTYPE) - -KeyType = enum_type_wrapper.EnumTypeWrapper(_KEYTYPE) -UNKNOWN_PUBLIC_KEY_TYPE = 0 -NIST256p = 1 -NIST384p = 2 -SECP256k1 = 3 - - -_CERTIFICATE_VERSION = _descriptor.EnumDescriptor( - name='Version', - full_name='legacy_pb.Certificate.Version', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='UNKNOWN_VERSION', index=0, number=0, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='_0_0_1', index=1, number=1, - serialized_options=None, - type=None), - ], - containing_type=None, - serialized_options=None, - serialized_start=153, - serialized_end=195, -) -_sym_db.RegisterEnumDescriptor(_CERTIFICATE_VERSION) - - -_CERTIFICATE = _descriptor.Descriptor( - name='Certificate', - full_name='legacy_pb.Certificate', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='version', full_name='legacy_pb.Certificate.version', index=0, - number=1, type=14, cpp_type=8, label=2, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='keyType', full_name='legacy_pb.Certificate.keyType', index=1, - number=2, type=14, cpp_type=8, label=2, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='publicKey', full_name='legacy_pb.Certificate.publicKey', index=2, - number=4, type=12, cpp_type=9, label=2, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _CERTIFICATE_VERSION, - ], - serialized_options=None, - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=33, - serialized_end=195, -) - -_CERTIFICATE.fields_by_name['version'].enum_type = _CERTIFICATE_VERSION -_CERTIFICATE.fields_by_name['keyType'].enum_type = _KEYTYPE -_CERTIFICATE_VERSION.containing_type = _CERTIFICATE -DESCRIPTOR.message_types_by_name['Certificate'] = _CERTIFICATE -DESCRIPTOR.enum_types_by_name['KeyType'] = _KEYTYPE -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -Certificate = _reflection.GeneratedProtocolMessageType('Certificate', (_message.Message,), dict( - DESCRIPTOR = _CERTIFICATE, - __module__ = 'certificate_pb2' - # @@protoc_insertion_point(class_scope:legacy_pb.Certificate) - )) -_sym_db.RegisterMessage(Certificate) - - -# @@protoc_insertion_point(module_scope) diff --git a/lbry/schema/types/v1/fee_pb2.py b/lbry/schema/types/v1/fee_pb2.py deleted file mode 100644 index aa430110e..000000000 --- a/lbry/schema/types/v1/fee_pb2.py +++ /dev/null @@ -1,148 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: fee.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='fee.proto', - package='legacy_pb', - syntax='proto2', - serialized_options=None, - serialized_pb=_b('\n\tfee.proto\x12\tlegacy_pb\"\xe3\x01\n\x03\x46\x65\x65\x12\'\n\x07version\x18\x01 \x02(\x0e\x32\x16.legacy_pb.Fee.Version\x12)\n\x08\x63urrency\x18\x02 \x02(\x0e\x32\x17.legacy_pb.Fee.Currency\x12\x0f\n\x07\x61\x64\x64ress\x18\x03 \x02(\x0c\x12\x0e\n\x06\x61mount\x18\x04 \x02(\x02\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\";\n\x08\x43urrency\x12\x14\n\x10UNKNOWN_CURRENCY\x10\x00\x12\x07\n\x03LBC\x10\x01\x12\x07\n\x03\x42TC\x10\x02\x12\x07\n\x03USD\x10\x03') -) - - - -_FEE_VERSION = _descriptor.EnumDescriptor( - name='Version', - full_name='legacy_pb.Fee.Version', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='UNKNOWN_VERSION', index=0, number=0, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='_0_0_1', index=1, number=1, - serialized_options=None, - type=None), - ], - containing_type=None, - serialized_options=None, - serialized_start=149, - serialized_end=191, -) -_sym_db.RegisterEnumDescriptor(_FEE_VERSION) - -_FEE_CURRENCY = _descriptor.EnumDescriptor( - name='Currency', - full_name='legacy_pb.Fee.Currency', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='UNKNOWN_CURRENCY', index=0, number=0, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='LBC', index=1, number=1, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BTC', index=2, number=2, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='USD', index=3, number=3, - serialized_options=None, - type=None), - ], - containing_type=None, - serialized_options=None, - serialized_start=193, - serialized_end=252, -) -_sym_db.RegisterEnumDescriptor(_FEE_CURRENCY) - - -_FEE = _descriptor.Descriptor( - name='Fee', - full_name='legacy_pb.Fee', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='version', full_name='legacy_pb.Fee.version', index=0, - number=1, type=14, cpp_type=8, label=2, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='currency', full_name='legacy_pb.Fee.currency', index=1, - number=2, type=14, cpp_type=8, label=2, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='address', full_name='legacy_pb.Fee.address', index=2, - number=3, type=12, cpp_type=9, label=2, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='amount', full_name='legacy_pb.Fee.amount', index=3, - number=4, type=2, cpp_type=6, label=2, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _FEE_VERSION, - _FEE_CURRENCY, - ], - serialized_options=None, - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=25, - serialized_end=252, -) - -_FEE.fields_by_name['version'].enum_type = _FEE_VERSION -_FEE.fields_by_name['currency'].enum_type = _FEE_CURRENCY -_FEE_VERSION.containing_type = _FEE -_FEE_CURRENCY.containing_type = _FEE -DESCRIPTOR.message_types_by_name['Fee'] = _FEE -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -Fee = _reflection.GeneratedProtocolMessageType('Fee', (_message.Message,), dict( - DESCRIPTOR = _FEE, - __module__ = 'fee_pb2' - # @@protoc_insertion_point(class_scope:legacy_pb.Fee) - )) -_sym_db.RegisterMessage(Fee) - - -# @@protoc_insertion_point(module_scope) diff --git a/lbry/schema/types/v1/legacy_claim_pb2.py b/lbry/schema/types/v1/legacy_claim_pb2.py deleted file mode 100644 index 7e37f2580..000000000 --- a/lbry/schema/types/v1/legacy_claim_pb2.py +++ /dev/null @@ -1,158 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: legacy_claim.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from . import stream_pb2 as stream__pb2 -from . import certificate_pb2 as certificate__pb2 -from . import signature_pb2 as signature__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='legacy_claim.proto', - package='legacy_pb', - syntax='proto2', - serialized_options=None, - serialized_pb=_b('\n\x12legacy_claim.proto\x12\tlegacy_pb\x1a\x0cstream.proto\x1a\x11\x63\x65rtificate.proto\x1a\x0fsignature.proto\"\xd9\x02\n\x05\x43laim\x12)\n\x07version\x18\x01 \x02(\x0e\x32\x18.legacy_pb.Claim.Version\x12-\n\tclaimType\x18\x02 \x02(\x0e\x32\x1a.legacy_pb.Claim.ClaimType\x12!\n\x06stream\x18\x03 \x01(\x0b\x32\x11.legacy_pb.Stream\x12+\n\x0b\x63\x65rtificate\x18\x04 \x01(\x0b\x32\x16.legacy_pb.Certificate\x12\x30\n\x12publisherSignature\x18\x05 \x01(\x0b\x32\x14.legacy_pb.Signature\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"H\n\tClaimType\x12\x16\n\x12UNKNOWN_CLAIM_TYPE\x10\x00\x12\x0e\n\nstreamType\x10\x01\x12\x13\n\x0f\x63\x65rtificateType\x10\x02') - , - dependencies=[stream__pb2.DESCRIPTOR,certificate__pb2.DESCRIPTOR,signature__pb2.DESCRIPTOR,]) - - - -_CLAIM_VERSION = _descriptor.EnumDescriptor( - name='Version', - full_name='legacy_pb.Claim.Version', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='UNKNOWN_VERSION', index=0, number=0, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='_0_0_1', index=1, number=1, - serialized_options=None, - type=None), - ], - containing_type=None, - serialized_options=None, - serialized_start=313, - serialized_end=355, -) -_sym_db.RegisterEnumDescriptor(_CLAIM_VERSION) - -_CLAIM_CLAIMTYPE = _descriptor.EnumDescriptor( - name='ClaimType', - full_name='legacy_pb.Claim.ClaimType', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='UNKNOWN_CLAIM_TYPE', index=0, number=0, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='streamType', index=1, number=1, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='certificateType', index=2, number=2, - serialized_options=None, - type=None), - ], - containing_type=None, - serialized_options=None, - serialized_start=357, - serialized_end=429, -) -_sym_db.RegisterEnumDescriptor(_CLAIM_CLAIMTYPE) - - -_CLAIM = _descriptor.Descriptor( - name='Claim', - full_name='legacy_pb.Claim', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='version', full_name='legacy_pb.Claim.version', index=0, - number=1, type=14, cpp_type=8, label=2, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='claimType', full_name='legacy_pb.Claim.claimType', index=1, - number=2, type=14, cpp_type=8, label=2, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='stream', full_name='legacy_pb.Claim.stream', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='certificate', full_name='legacy_pb.Claim.certificate', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='publisherSignature', full_name='legacy_pb.Claim.publisherSignature', index=4, - number=5, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _CLAIM_VERSION, - _CLAIM_CLAIMTYPE, - ], - serialized_options=None, - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=84, - serialized_end=429, -) - -_CLAIM.fields_by_name['version'].enum_type = _CLAIM_VERSION -_CLAIM.fields_by_name['claimType'].enum_type = _CLAIM_CLAIMTYPE -_CLAIM.fields_by_name['stream'].message_type = stream__pb2._STREAM -_CLAIM.fields_by_name['certificate'].message_type = certificate__pb2._CERTIFICATE -_CLAIM.fields_by_name['publisherSignature'].message_type = signature__pb2._SIGNATURE -_CLAIM_VERSION.containing_type = _CLAIM -_CLAIM_CLAIMTYPE.containing_type = _CLAIM -DESCRIPTOR.message_types_by_name['Claim'] = _CLAIM -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -Claim = _reflection.GeneratedProtocolMessageType('Claim', (_message.Message,), dict( - DESCRIPTOR = _CLAIM, - __module__ = 'legacy_claim_pb2' - # @@protoc_insertion_point(class_scope:legacy_pb.Claim) - )) -_sym_db.RegisterMessage(Claim) - - -# @@protoc_insertion_point(module_scope) diff --git a/lbry/schema/types/v1/metadata_pb2.py b/lbry/schema/types/v1/metadata_pb2.py deleted file mode 100644 index 631f20651..000000000 --- a/lbry/schema/types/v1/metadata_pb2.py +++ /dev/null @@ -1,936 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: metadata.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from . import fee_pb2 as fee__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='metadata.proto', - package='legacy_pb', - syntax='proto2', - serialized_options=None, - serialized_pb=_b('\n\x0emetadata.proto\x12\tlegacy_pb\x1a\tfee.proto\"\xfc\x0e\n\x08Metadata\x12,\n\x07version\x18\x01 \x02(\x0e\x32\x1b.legacy_pb.Metadata.Version\x12.\n\x08language\x18\x02 \x02(\x0e\x32\x1c.legacy_pb.Metadata.Language\x12\r\n\x05title\x18\x03 \x02(\t\x12\x13\n\x0b\x64\x65scription\x18\x04 \x02(\t\x12\x0e\n\x06\x61uthor\x18\x05 \x02(\t\x12\x0f\n\x07license\x18\x06 \x02(\t\x12\x0c\n\x04nsfw\x18\x07 \x02(\x08\x12\x1b\n\x03\x66\x65\x65\x18\x08 \x01(\x0b\x32\x0e.legacy_pb.Fee\x12\x11\n\tthumbnail\x18\t \x01(\t\x12\x0f\n\x07preview\x18\n \x01(\t\x12\x12\n\nlicenseUrl\x18\x0b \x01(\t\"N\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\x12\n\n\x06_0_0_2\x10\x02\x12\n\n\x06_0_0_3\x10\x03\x12\n\n\x06_0_1_0\x10\x04\"\x99\x0c\n\x08Language\x12\x14\n\x10UNKNOWN_LANGUAGE\x10\x00\x12\x06\n\x02\x65n\x10\x01\x12\x06\n\x02\x61\x61\x10\x02\x12\x06\n\x02\x61\x62\x10\x03\x12\x06\n\x02\x61\x65\x10\x04\x12\x06\n\x02\x61\x66\x10\x05\x12\x06\n\x02\x61k\x10\x06\x12\x06\n\x02\x61m\x10\x07\x12\x06\n\x02\x61n\x10\x08\x12\x06\n\x02\x61r\x10\t\x12\x06\n\x02\x61s\x10\n\x12\x06\n\x02\x61v\x10\x0b\x12\x06\n\x02\x61y\x10\x0c\x12\x06\n\x02\x61z\x10\r\x12\x06\n\x02\x62\x61\x10\x0e\x12\x06\n\x02\x62\x65\x10\x0f\x12\x06\n\x02\x62g\x10\x10\x12\x06\n\x02\x62h\x10\x11\x12\x06\n\x02\x62i\x10\x12\x12\x06\n\x02\x62m\x10\x13\x12\x06\n\x02\x62n\x10\x14\x12\x06\n\x02\x62o\x10\x15\x12\x06\n\x02\x62r\x10\x16\x12\x06\n\x02\x62s\x10\x17\x12\x06\n\x02\x63\x61\x10\x18\x12\x06\n\x02\x63\x65\x10\x19\x12\x06\n\x02\x63h\x10\x1a\x12\x06\n\x02\x63o\x10\x1b\x12\x06\n\x02\x63r\x10\x1c\x12\x06\n\x02\x63s\x10\x1d\x12\x06\n\x02\x63u\x10\x1e\x12\x06\n\x02\x63v\x10\x1f\x12\x06\n\x02\x63y\x10 \x12\x06\n\x02\x64\x61\x10!\x12\x06\n\x02\x64\x65\x10\"\x12\x06\n\x02\x64v\x10#\x12\x06\n\x02\x64z\x10$\x12\x06\n\x02\x65\x65\x10%\x12\x06\n\x02\x65l\x10&\x12\x06\n\x02\x65o\x10\'\x12\x06\n\x02\x65s\x10(\x12\x06\n\x02\x65t\x10)\x12\x06\n\x02\x65u\x10*\x12\x06\n\x02\x66\x61\x10+\x12\x06\n\x02\x66\x66\x10,\x12\x06\n\x02\x66i\x10-\x12\x06\n\x02\x66j\x10.\x12\x06\n\x02\x66o\x10/\x12\x06\n\x02\x66r\x10\x30\x12\x06\n\x02\x66y\x10\x31\x12\x06\n\x02ga\x10\x32\x12\x06\n\x02gd\x10\x33\x12\x06\n\x02gl\x10\x34\x12\x06\n\x02gn\x10\x35\x12\x06\n\x02gu\x10\x36\x12\x06\n\x02gv\x10\x37\x12\x06\n\x02ha\x10\x38\x12\x06\n\x02he\x10\x39\x12\x06\n\x02hi\x10:\x12\x06\n\x02ho\x10;\x12\x06\n\x02hr\x10<\x12\x06\n\x02ht\x10=\x12\x06\n\x02hu\x10>\x12\x06\n\x02hy\x10?\x12\x06\n\x02hz\x10@\x12\x06\n\x02ia\x10\x41\x12\x06\n\x02id\x10\x42\x12\x06\n\x02ie\x10\x43\x12\x06\n\x02ig\x10\x44\x12\x06\n\x02ii\x10\x45\x12\x06\n\x02ik\x10\x46\x12\x06\n\x02io\x10G\x12\x06\n\x02is\x10H\x12\x06\n\x02it\x10I\x12\x06\n\x02iu\x10J\x12\x06\n\x02ja\x10K\x12\x06\n\x02jv\x10L\x12\x06\n\x02ka\x10M\x12\x06\n\x02kg\x10N\x12\x06\n\x02ki\x10O\x12\x06\n\x02kj\x10P\x12\x06\n\x02kk\x10Q\x12\x06\n\x02kl\x10R\x12\x06\n\x02km\x10S\x12\x06\n\x02kn\x10T\x12\x06\n\x02ko\x10U\x12\x06\n\x02kr\x10V\x12\x06\n\x02ks\x10W\x12\x06\n\x02ku\x10X\x12\x06\n\x02kv\x10Y\x12\x06\n\x02kw\x10Z\x12\x06\n\x02ky\x10[\x12\x06\n\x02la\x10\\\x12\x06\n\x02lb\x10]\x12\x06\n\x02lg\x10^\x12\x06\n\x02li\x10_\x12\x06\n\x02ln\x10`\x12\x06\n\x02lo\x10\x61\x12\x06\n\x02lt\x10\x62\x12\x06\n\x02lu\x10\x63\x12\x06\n\x02lv\x10\x64\x12\x06\n\x02mg\x10\x65\x12\x06\n\x02mh\x10\x66\x12\x06\n\x02mi\x10g\x12\x06\n\x02mk\x10h\x12\x06\n\x02ml\x10i\x12\x06\n\x02mn\x10j\x12\x06\n\x02mr\x10k\x12\x06\n\x02ms\x10l\x12\x06\n\x02mt\x10m\x12\x06\n\x02my\x10n\x12\x06\n\x02na\x10o\x12\x06\n\x02nb\x10p\x12\x06\n\x02nd\x10q\x12\x06\n\x02ne\x10r\x12\x06\n\x02ng\x10s\x12\x06\n\x02nl\x10t\x12\x06\n\x02nn\x10u\x12\x06\n\x02no\x10v\x12\x06\n\x02nr\x10w\x12\x06\n\x02nv\x10x\x12\x06\n\x02ny\x10y\x12\x06\n\x02oc\x10z\x12\x06\n\x02oj\x10{\x12\x06\n\x02om\x10|\x12\x06\n\x02or\x10}\x12\x06\n\x02os\x10~\x12\x06\n\x02pa\x10\x7f\x12\x07\n\x02pi\x10\x80\x01\x12\x07\n\x02pl\x10\x81\x01\x12\x07\n\x02ps\x10\x82\x01\x12\x07\n\x02pt\x10\x83\x01\x12\x07\n\x02qu\x10\x84\x01\x12\x07\n\x02rm\x10\x85\x01\x12\x07\n\x02rn\x10\x86\x01\x12\x07\n\x02ro\x10\x87\x01\x12\x07\n\x02ru\x10\x88\x01\x12\x07\n\x02rw\x10\x89\x01\x12\x07\n\x02sa\x10\x8a\x01\x12\x07\n\x02sc\x10\x8b\x01\x12\x07\n\x02sd\x10\x8c\x01\x12\x07\n\x02se\x10\x8d\x01\x12\x07\n\x02sg\x10\x8e\x01\x12\x07\n\x02si\x10\x8f\x01\x12\x07\n\x02sk\x10\x90\x01\x12\x07\n\x02sl\x10\x91\x01\x12\x07\n\x02sm\x10\x92\x01\x12\x07\n\x02sn\x10\x93\x01\x12\x07\n\x02so\x10\x94\x01\x12\x07\n\x02sq\x10\x95\x01\x12\x07\n\x02sr\x10\x96\x01\x12\x07\n\x02ss\x10\x97\x01\x12\x07\n\x02st\x10\x98\x01\x12\x07\n\x02su\x10\x99\x01\x12\x07\n\x02sv\x10\x9a\x01\x12\x07\n\x02sw\x10\x9b\x01\x12\x07\n\x02ta\x10\x9c\x01\x12\x07\n\x02te\x10\x9d\x01\x12\x07\n\x02tg\x10\x9e\x01\x12\x07\n\x02th\x10\x9f\x01\x12\x07\n\x02ti\x10\xa0\x01\x12\x07\n\x02tk\x10\xa1\x01\x12\x07\n\x02tl\x10\xa2\x01\x12\x07\n\x02tn\x10\xa3\x01\x12\x07\n\x02to\x10\xa4\x01\x12\x07\n\x02tr\x10\xa5\x01\x12\x07\n\x02ts\x10\xa6\x01\x12\x07\n\x02tt\x10\xa7\x01\x12\x07\n\x02tw\x10\xa8\x01\x12\x07\n\x02ty\x10\xa9\x01\x12\x07\n\x02ug\x10\xaa\x01\x12\x07\n\x02uk\x10\xab\x01\x12\x07\n\x02ur\x10\xac\x01\x12\x07\n\x02uz\x10\xad\x01\x12\x07\n\x02ve\x10\xae\x01\x12\x07\n\x02vi\x10\xaf\x01\x12\x07\n\x02vo\x10\xb0\x01\x12\x07\n\x02wa\x10\xb1\x01\x12\x07\n\x02wo\x10\xb2\x01\x12\x07\n\x02xh\x10\xb3\x01\x12\x07\n\x02yi\x10\xb4\x01\x12\x07\n\x02yo\x10\xb5\x01\x12\x07\n\x02za\x10\xb6\x01\x12\x07\n\x02zh\x10\xb7\x01\x12\x07\n\x02zu\x10\xb8\x01') - , - dependencies=[fee__pb2.DESCRIPTOR,]) - - - -_METADATA_VERSION = _descriptor.EnumDescriptor( - name='Version', - full_name='legacy_pb.Metadata.Version', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='UNKNOWN_VERSION', index=0, number=0, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='_0_0_1', index=1, number=1, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='_0_0_2', index=2, number=2, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='_0_0_3', index=3, number=3, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='_0_1_0', index=4, number=4, - serialized_options=None, - type=None), - ], - containing_type=None, - serialized_options=None, - serialized_start=315, - serialized_end=393, -) -_sym_db.RegisterEnumDescriptor(_METADATA_VERSION) - -_METADATA_LANGUAGE = _descriptor.EnumDescriptor( - name='Language', - full_name='legacy_pb.Metadata.Language', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='UNKNOWN_LANGUAGE', index=0, number=0, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='en', index=1, number=1, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='aa', index=2, number=2, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ab', index=3, number=3, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ae', index=4, number=4, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='af', index=5, number=5, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ak', index=6, number=6, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='am', index=7, number=7, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='an', index=8, number=8, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ar', index=9, number=9, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='as', index=10, number=10, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='av', index=11, number=11, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ay', index=12, number=12, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='az', index=13, number=13, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ba', index=14, number=14, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='be', index=15, number=15, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='bg', index=16, number=16, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='bh', index=17, number=17, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='bi', index=18, number=18, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='bm', index=19, number=19, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='bn', index=20, number=20, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='bo', index=21, number=21, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='br', index=22, number=22, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='bs', index=23, number=23, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ca', index=24, number=24, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ce', index=25, number=25, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ch', index=26, number=26, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='co', index=27, number=27, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='cr', index=28, number=28, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='cs', index=29, number=29, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='cu', index=30, number=30, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='cv', index=31, number=31, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='cy', index=32, number=32, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='da', index=33, number=33, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='de', index=34, number=34, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='dv', index=35, number=35, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='dz', index=36, number=36, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ee', index=37, number=37, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='el', index=38, number=38, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='eo', index=39, number=39, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='es', index=40, number=40, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='et', index=41, number=41, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='eu', index=42, number=42, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='fa', index=43, number=43, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ff', index=44, number=44, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='fi', index=45, number=45, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='fj', index=46, number=46, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='fo', index=47, number=47, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='fr', index=48, number=48, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='fy', index=49, number=49, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ga', index=50, number=50, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='gd', index=51, number=51, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='gl', index=52, number=52, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='gn', index=53, number=53, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='gu', index=54, number=54, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='gv', index=55, number=55, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ha', index=56, number=56, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='he', index=57, number=57, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='hi', index=58, number=58, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ho', index=59, number=59, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='hr', index=60, number=60, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ht', index=61, number=61, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='hu', index=62, number=62, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='hy', index=63, number=63, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='hz', index=64, number=64, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ia', index=65, number=65, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='id', index=66, number=66, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ie', index=67, number=67, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ig', index=68, number=68, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ii', index=69, number=69, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ik', index=70, number=70, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='io', index=71, number=71, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='is', index=72, number=72, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='it', index=73, number=73, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='iu', index=74, number=74, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ja', index=75, number=75, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='jv', index=76, number=76, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ka', index=77, number=77, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='kg', index=78, number=78, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ki', index=79, number=79, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='kj', index=80, number=80, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='kk', index=81, number=81, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='kl', index=82, number=82, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='km', index=83, number=83, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='kn', index=84, number=84, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ko', index=85, number=85, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='kr', index=86, number=86, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ks', index=87, number=87, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ku', index=88, number=88, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='kv', index=89, number=89, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='kw', index=90, number=90, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ky', index=91, number=91, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='la', index=92, number=92, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='lb', index=93, number=93, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='lg', index=94, number=94, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='li', index=95, number=95, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ln', index=96, number=96, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='lo', index=97, number=97, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='lt', index=98, number=98, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='lu', index=99, number=99, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='lv', index=100, number=100, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='mg', index=101, number=101, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='mh', index=102, number=102, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='mi', index=103, number=103, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='mk', index=104, number=104, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ml', index=105, number=105, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='mn', index=106, number=106, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='mr', index=107, number=107, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ms', index=108, number=108, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='mt', index=109, number=109, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='my', index=110, number=110, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='na', index=111, number=111, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='nb', index=112, number=112, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='nd', index=113, number=113, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ne', index=114, number=114, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ng', index=115, number=115, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='nl', index=116, number=116, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='nn', index=117, number=117, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='no', index=118, number=118, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='nr', index=119, number=119, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='nv', index=120, number=120, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ny', index=121, number=121, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='oc', index=122, number=122, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='oj', index=123, number=123, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='om', index=124, number=124, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='or', index=125, number=125, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='os', index=126, number=126, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='pa', index=127, number=127, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='pi', index=128, number=128, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='pl', index=129, number=129, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ps', index=130, number=130, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='pt', index=131, number=131, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='qu', index=132, number=132, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='rm', index=133, number=133, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='rn', index=134, number=134, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ro', index=135, number=135, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ru', index=136, number=136, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='rw', index=137, number=137, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='sa', index=138, number=138, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='sc', index=139, number=139, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='sd', index=140, number=140, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='se', index=141, number=141, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='sg', index=142, number=142, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='si', index=143, number=143, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='sk', index=144, number=144, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='sl', index=145, number=145, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='sm', index=146, number=146, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='sn', index=147, number=147, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='so', index=148, number=148, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='sq', index=149, number=149, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='sr', index=150, number=150, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ss', index=151, number=151, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='st', index=152, number=152, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='su', index=153, number=153, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='sv', index=154, number=154, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='sw', index=155, number=155, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ta', index=156, number=156, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='te', index=157, number=157, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='tg', index=158, number=158, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='th', index=159, number=159, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ti', index=160, number=160, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='tk', index=161, number=161, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='tl', index=162, number=162, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='tn', index=163, number=163, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='to', index=164, number=164, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='tr', index=165, number=165, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ts', index=166, number=166, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='tt', index=167, number=167, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='tw', index=168, number=168, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ty', index=169, number=169, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ug', index=170, number=170, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='uk', index=171, number=171, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ur', index=172, number=172, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='uz', index=173, number=173, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ve', index=174, number=174, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='vi', index=175, number=175, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='vo', index=176, number=176, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='wa', index=177, number=177, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='wo', index=178, number=178, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='xh', index=179, number=179, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='yi', index=180, number=180, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='yo', index=181, number=181, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='za', index=182, number=182, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='zh', index=183, number=183, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='zu', index=184, number=184, - serialized_options=None, - type=None), - ], - containing_type=None, - serialized_options=None, - serialized_start=396, - serialized_end=1957, -) -_sym_db.RegisterEnumDescriptor(_METADATA_LANGUAGE) - - -_METADATA = _descriptor.Descriptor( - name='Metadata', - full_name='legacy_pb.Metadata', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='version', full_name='legacy_pb.Metadata.version', index=0, - number=1, type=14, cpp_type=8, label=2, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='language', full_name='legacy_pb.Metadata.language', index=1, - number=2, type=14, cpp_type=8, label=2, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='title', full_name='legacy_pb.Metadata.title', index=2, - number=3, type=9, cpp_type=9, label=2, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='description', full_name='legacy_pb.Metadata.description', index=3, - number=4, type=9, cpp_type=9, label=2, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='author', full_name='legacy_pb.Metadata.author', index=4, - number=5, type=9, cpp_type=9, label=2, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='license', full_name='legacy_pb.Metadata.license', index=5, - number=6, type=9, cpp_type=9, label=2, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='nsfw', full_name='legacy_pb.Metadata.nsfw', index=6, - number=7, type=8, cpp_type=7, label=2, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='fee', full_name='legacy_pb.Metadata.fee', index=7, - number=8, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='thumbnail', full_name='legacy_pb.Metadata.thumbnail', index=8, - number=9, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='preview', full_name='legacy_pb.Metadata.preview', index=9, - number=10, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='licenseUrl', full_name='legacy_pb.Metadata.licenseUrl', index=10, - number=11, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _METADATA_VERSION, - _METADATA_LANGUAGE, - ], - serialized_options=None, - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=41, - serialized_end=1957, -) - -_METADATA.fields_by_name['version'].enum_type = _METADATA_VERSION -_METADATA.fields_by_name['language'].enum_type = _METADATA_LANGUAGE -_METADATA.fields_by_name['fee'].message_type = fee__pb2._FEE -_METADATA_VERSION.containing_type = _METADATA -_METADATA_LANGUAGE.containing_type = _METADATA -DESCRIPTOR.message_types_by_name['Metadata'] = _METADATA -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -Metadata = _reflection.GeneratedProtocolMessageType('Metadata', (_message.Message,), dict( - DESCRIPTOR = _METADATA, - __module__ = 'metadata_pb2' - # @@protoc_insertion_point(class_scope:legacy_pb.Metadata) - )) -_sym_db.RegisterMessage(Metadata) - - -# @@protoc_insertion_point(module_scope) diff --git a/lbry/schema/types/v1/signature_pb2.py b/lbry/schema/types/v1/signature_pb2.py deleted file mode 100644 index 8e1663fe5..000000000 --- a/lbry/schema/types/v1/signature_pb2.py +++ /dev/null @@ -1,118 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: signature.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from . import certificate_pb2 as certificate__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='signature.proto', - package='legacy_pb', - syntax='proto2', - serialized_options=None, - serialized_pb=_b('\n\x0fsignature.proto\x12\tlegacy_pb\x1a\x11\x63\x65rtificate.proto\"\xbb\x01\n\tSignature\x12-\n\x07version\x18\x01 \x02(\x0e\x32\x1c.legacy_pb.Signature.Version\x12)\n\rsignatureType\x18\x02 \x02(\x0e\x32\x12.legacy_pb.KeyType\x12\x11\n\tsignature\x18\x03 \x02(\x0c\x12\x15\n\rcertificateId\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01') - , - dependencies=[certificate__pb2.DESCRIPTOR,]) - - - -_SIGNATURE_VERSION = _descriptor.EnumDescriptor( - name='Version', - full_name='legacy_pb.Signature.Version', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='UNKNOWN_VERSION', index=0, number=0, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='_0_0_1', index=1, number=1, - serialized_options=None, - type=None), - ], - containing_type=None, - serialized_options=None, - serialized_start=195, - serialized_end=237, -) -_sym_db.RegisterEnumDescriptor(_SIGNATURE_VERSION) - - -_SIGNATURE = _descriptor.Descriptor( - name='Signature', - full_name='legacy_pb.Signature', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='version', full_name='legacy_pb.Signature.version', index=0, - number=1, type=14, cpp_type=8, label=2, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='signatureType', full_name='legacy_pb.Signature.signatureType', index=1, - number=2, type=14, cpp_type=8, label=2, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='signature', full_name='legacy_pb.Signature.signature', index=2, - number=3, type=12, cpp_type=9, label=2, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='certificateId', full_name='legacy_pb.Signature.certificateId', index=3, - number=4, type=12, cpp_type=9, label=2, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _SIGNATURE_VERSION, - ], - serialized_options=None, - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=50, - serialized_end=237, -) - -_SIGNATURE.fields_by_name['version'].enum_type = _SIGNATURE_VERSION -_SIGNATURE.fields_by_name['signatureType'].enum_type = certificate__pb2._KEYTYPE -_SIGNATURE_VERSION.containing_type = _SIGNATURE -DESCRIPTOR.message_types_by_name['Signature'] = _SIGNATURE -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -Signature = _reflection.GeneratedProtocolMessageType('Signature', (_message.Message,), dict( - DESCRIPTOR = _SIGNATURE, - __module__ = 'signature_pb2' - # @@protoc_insertion_point(class_scope:legacy_pb.Signature) - )) -_sym_db.RegisterMessage(Signature) - - -# @@protoc_insertion_point(module_scope) diff --git a/lbry/schema/types/v1/source_pb2.py b/lbry/schema/types/v1/source_pb2.py deleted file mode 100644 index 7c3bf696b..000000000 --- a/lbry/schema/types/v1/source_pb2.py +++ /dev/null @@ -1,140 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: source.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='source.proto', - package='legacy_pb', - syntax='proto2', - serialized_options=None, - serialized_pb=_b('\n\x0csource.proto\x12\tlegacy_pb\"\xf2\x01\n\x06Source\x12*\n\x07version\x18\x01 \x02(\x0e\x32\x19.legacy_pb.Source.Version\x12\x31\n\nsourceType\x18\x02 \x02(\x0e\x32\x1d.legacy_pb.Source.SourceTypes\x12\x0e\n\x06source\x18\x03 \x02(\x0c\x12\x13\n\x0b\x63ontentType\x18\x04 \x02(\t\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"8\n\x0bSourceTypes\x12\x17\n\x13UNKNOWN_SOURCE_TYPE\x10\x00\x12\x10\n\x0clbry_sd_hash\x10\x01') -) - - - -_SOURCE_VERSION = _descriptor.EnumDescriptor( - name='Version', - full_name='legacy_pb.Source.Version', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='UNKNOWN_VERSION', index=0, number=0, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='_0_0_1', index=1, number=1, - serialized_options=None, - type=None), - ], - containing_type=None, - serialized_options=None, - serialized_start=170, - serialized_end=212, -) -_sym_db.RegisterEnumDescriptor(_SOURCE_VERSION) - -_SOURCE_SOURCETYPES = _descriptor.EnumDescriptor( - name='SourceTypes', - full_name='legacy_pb.Source.SourceTypes', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='UNKNOWN_SOURCE_TYPE', index=0, number=0, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='lbry_sd_hash', index=1, number=1, - serialized_options=None, - type=None), - ], - containing_type=None, - serialized_options=None, - serialized_start=214, - serialized_end=270, -) -_sym_db.RegisterEnumDescriptor(_SOURCE_SOURCETYPES) - - -_SOURCE = _descriptor.Descriptor( - name='Source', - full_name='legacy_pb.Source', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='version', full_name='legacy_pb.Source.version', index=0, - number=1, type=14, cpp_type=8, label=2, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='sourceType', full_name='legacy_pb.Source.sourceType', index=1, - number=2, type=14, cpp_type=8, label=2, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='source', full_name='legacy_pb.Source.source', index=2, - number=3, type=12, cpp_type=9, label=2, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='contentType', full_name='legacy_pb.Source.contentType', index=3, - number=4, type=9, cpp_type=9, label=2, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _SOURCE_VERSION, - _SOURCE_SOURCETYPES, - ], - serialized_options=None, - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=28, - serialized_end=270, -) - -_SOURCE.fields_by_name['version'].enum_type = _SOURCE_VERSION -_SOURCE.fields_by_name['sourceType'].enum_type = _SOURCE_SOURCETYPES -_SOURCE_VERSION.containing_type = _SOURCE -_SOURCE_SOURCETYPES.containing_type = _SOURCE -DESCRIPTOR.message_types_by_name['Source'] = _SOURCE -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -Source = _reflection.GeneratedProtocolMessageType('Source', (_message.Message,), dict( - DESCRIPTOR = _SOURCE, - __module__ = 'source_pb2' - # @@protoc_insertion_point(class_scope:legacy_pb.Source) - )) -_sym_db.RegisterMessage(Source) - - -# @@protoc_insertion_point(module_scope) diff --git a/lbry/schema/types/v1/stream_pb2.py b/lbry/schema/types/v1/stream_pb2.py deleted file mode 100644 index ec96d0b01..000000000 --- a/lbry/schema/types/v1/stream_pb2.py +++ /dev/null @@ -1,113 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: stream.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from . import metadata_pb2 as metadata__pb2 -from . import source_pb2 as source__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='stream.proto', - package='legacy_pb', - syntax='proto2', - serialized_options=None, - serialized_pb=_b('\n\x0cstream.proto\x12\tlegacy_pb\x1a\x0emetadata.proto\x1a\x0csource.proto\"\xaa\x01\n\x06Stream\x12*\n\x07version\x18\x01 \x02(\x0e\x32\x19.legacy_pb.Stream.Version\x12%\n\x08metadata\x18\x02 \x02(\x0b\x32\x13.legacy_pb.Metadata\x12!\n\x06source\x18\x03 \x02(\x0b\x32\x11.legacy_pb.Source\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01') - , - dependencies=[metadata__pb2.DESCRIPTOR,source__pb2.DESCRIPTOR,]) - - - -_STREAM_VERSION = _descriptor.EnumDescriptor( - name='Version', - full_name='legacy_pb.Stream.Version', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='UNKNOWN_VERSION', index=0, number=0, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='_0_0_1', index=1, number=1, - serialized_options=None, - type=None), - ], - containing_type=None, - serialized_options=None, - serialized_start=186, - serialized_end=228, -) -_sym_db.RegisterEnumDescriptor(_STREAM_VERSION) - - -_STREAM = _descriptor.Descriptor( - name='Stream', - full_name='legacy_pb.Stream', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='version', full_name='legacy_pb.Stream.version', index=0, - number=1, type=14, cpp_type=8, label=2, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='metadata', full_name='legacy_pb.Stream.metadata', index=1, - number=2, type=11, cpp_type=10, label=2, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='source', full_name='legacy_pb.Stream.source', index=2, - number=3, type=11, cpp_type=10, label=2, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _STREAM_VERSION, - ], - serialized_options=None, - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=58, - serialized_end=228, -) - -_STREAM.fields_by_name['version'].enum_type = _STREAM_VERSION -_STREAM.fields_by_name['metadata'].message_type = metadata__pb2._METADATA -_STREAM.fields_by_name['source'].message_type = source__pb2._SOURCE -_STREAM_VERSION.containing_type = _STREAM -DESCRIPTOR.message_types_by_name['Stream'] = _STREAM -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -Stream = _reflection.GeneratedProtocolMessageType('Stream', (_message.Message,), dict( - DESCRIPTOR = _STREAM, - __module__ = 'stream_pb2' - # @@protoc_insertion_point(class_scope:legacy_pb.Stream) - )) -_sym_db.RegisterMessage(Stream) - - -# @@protoc_insertion_point(module_scope) diff --git a/lbry/schema/types/v2/__init__.py b/lbry/schema/types/v2/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/lbry/schema/types/v2/claim_pb2.py b/lbry/schema/types/v2/claim_pb2.py deleted file mode 100644 index d63f73964..000000000 --- a/lbry/schema/types/v2/claim_pb2.py +++ /dev/null @@ -1,4692 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: claim.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='claim.proto', - package='pb', - syntax='proto3', - serialized_pb=_b('\n\x0b\x63laim.proto\x12\x02pb\"\xab\x02\n\x05\x43laim\x12\x1c\n\x06stream\x18\x01 \x01(\x0b\x32\n.pb.StreamH\x00\x12\x1e\n\x07\x63hannel\x18\x02 \x01(\x0b\x32\x0b.pb.ChannelH\x00\x12#\n\ncollection\x18\x03 \x01(\x0b\x32\r.pb.ClaimListH\x00\x12$\n\x06repost\x18\x04 \x01(\x0b\x32\x12.pb.ClaimReferenceH\x00\x12\r\n\x05title\x18\x08 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\t \x01(\t\x12\x1d\n\tthumbnail\x18\n \x01(\x0b\x32\n.pb.Source\x12\x0c\n\x04tags\x18\x0b \x03(\t\x12\x1f\n\tlanguages\x18\x0c \x03(\x0b\x32\x0c.pb.Language\x12\x1f\n\tlocations\x18\r \x03(\x0b\x32\x0c.pb.LocationB\x06\n\x04type\"\x84\x02\n\x06Stream\x12\x1a\n\x06source\x18\x01 \x01(\x0b\x32\n.pb.Source\x12\x0e\n\x06\x61uthor\x18\x02 \x01(\t\x12\x0f\n\x07license\x18\x03 \x01(\t\x12\x13\n\x0blicense_url\x18\x04 \x01(\t\x12\x14\n\x0crelease_time\x18\x05 \x01(\x03\x12\x14\n\x03\x66\x65\x65\x18\x06 \x01(\x0b\x32\x07.pb.Fee\x12\x1a\n\x05image\x18\n \x01(\x0b\x32\t.pb.ImageH\x00\x12\x1a\n\x05video\x18\x0b \x01(\x0b\x32\t.pb.VideoH\x00\x12\x1a\n\x05\x61udio\x18\x0c \x01(\x0b\x32\t.pb.AudioH\x00\x12 \n\x08software\x18\r \x01(\x0b\x32\x0c.pb.SoftwareH\x00\x42\x06\n\x04type\"}\n\x07\x43hannel\x12\x12\n\npublic_key\x18\x01 \x01(\x0c\x12\r\n\x05\x65mail\x18\x02 \x01(\t\x12\x13\n\x0bwebsite_url\x18\x03 \x01(\t\x12\x19\n\x05\x63over\x18\x04 \x01(\x0b\x32\n.pb.Source\x12\x1f\n\x08\x66\x65\x61tured\x18\x05 \x01(\x0b\x32\r.pb.ClaimList\"$\n\x0e\x43laimReference\x12\x12\n\nclaim_hash\x18\x01 \x01(\x0c\"\x90\x01\n\tClaimList\x12)\n\tlist_type\x18\x01 \x01(\x0e\x32\x16.pb.ClaimList.ListType\x12,\n\x10\x63laim_references\x18\x02 \x03(\x0b\x32\x12.pb.ClaimReference\"*\n\x08ListType\x12\x0e\n\nCOLLECTION\x10\x00\x12\x0e\n\nDERIVATION\x10\x02\"y\n\x06Source\x12\x0c\n\x04hash\x18\x01 \x01(\x0c\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0c\n\x04size\x18\x03 \x01(\x04\x12\x12\n\nmedia_type\x18\x04 \x01(\t\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0f\n\x07sd_hash\x18\x06 \x01(\x0c\x12\x13\n\x0b\x62t_infohash\x18\x07 \x01(\x0c\"\x87\x01\n\x03\x46\x65\x65\x12\"\n\x08\x63urrency\x18\x01 \x01(\x0e\x32\x10.pb.Fee.Currency\x12\x0f\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0c\x12\x0e\n\x06\x61mount\x18\x03 \x01(\x04\";\n\x08\x43urrency\x12\x14\n\x10UNKNOWN_CURRENCY\x10\x00\x12\x07\n\x03LBC\x10\x01\x12\x07\n\x03\x42TC\x10\x02\x12\x07\n\x03USD\x10\x03\"&\n\x05Image\x12\r\n\x05width\x18\x01 \x01(\r\x12\x0e\n\x06height\x18\x02 \x01(\r\"R\n\x05Video\x12\r\n\x05width\x18\x01 \x01(\r\x12\x0e\n\x06height\x18\x02 \x01(\r\x12\x10\n\x08\x64uration\x18\x03 \x01(\r\x12\x18\n\x05\x61udio\x18\x0f \x01(\x0b\x32\t.pb.Audio\"\x19\n\x05\x41udio\x12\x10\n\x08\x64uration\x18\x01 \x01(\r\"l\n\x08Software\x12\n\n\x02os\x18\x01 \x01(\t\"T\n\x02OS\x12\x0e\n\nUNKNOWN_OS\x10\x00\x12\x07\n\x03\x41NY\x10\x01\x12\t\n\x05LINUX\x10\x02\x12\x0b\n\x07WINDOWS\x10\x03\x12\x07\n\x03MAC\x10\x04\x12\x0b\n\x07\x41NDROID\x10\x05\x12\x07\n\x03IOS\x10\x06\"\xc7\x1d\n\x08Language\x12\'\n\x08language\x18\x01 \x01(\x0e\x32\x15.pb.Language.Language\x12#\n\x06script\x18\x02 \x01(\x0e\x32\x13.pb.Language.Script\x12$\n\x06region\x18\x03 \x01(\x0e\x32\x14.pb.Location.Country\"\x99\x0c\n\x08Language\x12\x14\n\x10UNKNOWN_LANGUAGE\x10\x00\x12\x06\n\x02\x65n\x10\x01\x12\x06\n\x02\x61\x61\x10\x02\x12\x06\n\x02\x61\x62\x10\x03\x12\x06\n\x02\x61\x65\x10\x04\x12\x06\n\x02\x61\x66\x10\x05\x12\x06\n\x02\x61k\x10\x06\x12\x06\n\x02\x61m\x10\x07\x12\x06\n\x02\x61n\x10\x08\x12\x06\n\x02\x61r\x10\t\x12\x06\n\x02\x61s\x10\n\x12\x06\n\x02\x61v\x10\x0b\x12\x06\n\x02\x61y\x10\x0c\x12\x06\n\x02\x61z\x10\r\x12\x06\n\x02\x62\x61\x10\x0e\x12\x06\n\x02\x62\x65\x10\x0f\x12\x06\n\x02\x62g\x10\x10\x12\x06\n\x02\x62h\x10\x11\x12\x06\n\x02\x62i\x10\x12\x12\x06\n\x02\x62m\x10\x13\x12\x06\n\x02\x62n\x10\x14\x12\x06\n\x02\x62o\x10\x15\x12\x06\n\x02\x62r\x10\x16\x12\x06\n\x02\x62s\x10\x17\x12\x06\n\x02\x63\x61\x10\x18\x12\x06\n\x02\x63\x65\x10\x19\x12\x06\n\x02\x63h\x10\x1a\x12\x06\n\x02\x63o\x10\x1b\x12\x06\n\x02\x63r\x10\x1c\x12\x06\n\x02\x63s\x10\x1d\x12\x06\n\x02\x63u\x10\x1e\x12\x06\n\x02\x63v\x10\x1f\x12\x06\n\x02\x63y\x10 \x12\x06\n\x02\x64\x61\x10!\x12\x06\n\x02\x64\x65\x10\"\x12\x06\n\x02\x64v\x10#\x12\x06\n\x02\x64z\x10$\x12\x06\n\x02\x65\x65\x10%\x12\x06\n\x02\x65l\x10&\x12\x06\n\x02\x65o\x10\'\x12\x06\n\x02\x65s\x10(\x12\x06\n\x02\x65t\x10)\x12\x06\n\x02\x65u\x10*\x12\x06\n\x02\x66\x61\x10+\x12\x06\n\x02\x66\x66\x10,\x12\x06\n\x02\x66i\x10-\x12\x06\n\x02\x66j\x10.\x12\x06\n\x02\x66o\x10/\x12\x06\n\x02\x66r\x10\x30\x12\x06\n\x02\x66y\x10\x31\x12\x06\n\x02ga\x10\x32\x12\x06\n\x02gd\x10\x33\x12\x06\n\x02gl\x10\x34\x12\x06\n\x02gn\x10\x35\x12\x06\n\x02gu\x10\x36\x12\x06\n\x02gv\x10\x37\x12\x06\n\x02ha\x10\x38\x12\x06\n\x02he\x10\x39\x12\x06\n\x02hi\x10:\x12\x06\n\x02ho\x10;\x12\x06\n\x02hr\x10<\x12\x06\n\x02ht\x10=\x12\x06\n\x02hu\x10>\x12\x06\n\x02hy\x10?\x12\x06\n\x02hz\x10@\x12\x06\n\x02ia\x10\x41\x12\x06\n\x02id\x10\x42\x12\x06\n\x02ie\x10\x43\x12\x06\n\x02ig\x10\x44\x12\x06\n\x02ii\x10\x45\x12\x06\n\x02ik\x10\x46\x12\x06\n\x02io\x10G\x12\x06\n\x02is\x10H\x12\x06\n\x02it\x10I\x12\x06\n\x02iu\x10J\x12\x06\n\x02ja\x10K\x12\x06\n\x02jv\x10L\x12\x06\n\x02ka\x10M\x12\x06\n\x02kg\x10N\x12\x06\n\x02ki\x10O\x12\x06\n\x02kj\x10P\x12\x06\n\x02kk\x10Q\x12\x06\n\x02kl\x10R\x12\x06\n\x02km\x10S\x12\x06\n\x02kn\x10T\x12\x06\n\x02ko\x10U\x12\x06\n\x02kr\x10V\x12\x06\n\x02ks\x10W\x12\x06\n\x02ku\x10X\x12\x06\n\x02kv\x10Y\x12\x06\n\x02kw\x10Z\x12\x06\n\x02ky\x10[\x12\x06\n\x02la\x10\\\x12\x06\n\x02lb\x10]\x12\x06\n\x02lg\x10^\x12\x06\n\x02li\x10_\x12\x06\n\x02ln\x10`\x12\x06\n\x02lo\x10\x61\x12\x06\n\x02lt\x10\x62\x12\x06\n\x02lu\x10\x63\x12\x06\n\x02lv\x10\x64\x12\x06\n\x02mg\x10\x65\x12\x06\n\x02mh\x10\x66\x12\x06\n\x02mi\x10g\x12\x06\n\x02mk\x10h\x12\x06\n\x02ml\x10i\x12\x06\n\x02mn\x10j\x12\x06\n\x02mr\x10k\x12\x06\n\x02ms\x10l\x12\x06\n\x02mt\x10m\x12\x06\n\x02my\x10n\x12\x06\n\x02na\x10o\x12\x06\n\x02nb\x10p\x12\x06\n\x02nd\x10q\x12\x06\n\x02ne\x10r\x12\x06\n\x02ng\x10s\x12\x06\n\x02nl\x10t\x12\x06\n\x02nn\x10u\x12\x06\n\x02no\x10v\x12\x06\n\x02nr\x10w\x12\x06\n\x02nv\x10x\x12\x06\n\x02ny\x10y\x12\x06\n\x02oc\x10z\x12\x06\n\x02oj\x10{\x12\x06\n\x02om\x10|\x12\x06\n\x02or\x10}\x12\x06\n\x02os\x10~\x12\x06\n\x02pa\x10\x7f\x12\x07\n\x02pi\x10\x80\x01\x12\x07\n\x02pl\x10\x81\x01\x12\x07\n\x02ps\x10\x82\x01\x12\x07\n\x02pt\x10\x83\x01\x12\x07\n\x02qu\x10\x84\x01\x12\x07\n\x02rm\x10\x85\x01\x12\x07\n\x02rn\x10\x86\x01\x12\x07\n\x02ro\x10\x87\x01\x12\x07\n\x02ru\x10\x88\x01\x12\x07\n\x02rw\x10\x89\x01\x12\x07\n\x02sa\x10\x8a\x01\x12\x07\n\x02sc\x10\x8b\x01\x12\x07\n\x02sd\x10\x8c\x01\x12\x07\n\x02se\x10\x8d\x01\x12\x07\n\x02sg\x10\x8e\x01\x12\x07\n\x02si\x10\x8f\x01\x12\x07\n\x02sk\x10\x90\x01\x12\x07\n\x02sl\x10\x91\x01\x12\x07\n\x02sm\x10\x92\x01\x12\x07\n\x02sn\x10\x93\x01\x12\x07\n\x02so\x10\x94\x01\x12\x07\n\x02sq\x10\x95\x01\x12\x07\n\x02sr\x10\x96\x01\x12\x07\n\x02ss\x10\x97\x01\x12\x07\n\x02st\x10\x98\x01\x12\x07\n\x02su\x10\x99\x01\x12\x07\n\x02sv\x10\x9a\x01\x12\x07\n\x02sw\x10\x9b\x01\x12\x07\n\x02ta\x10\x9c\x01\x12\x07\n\x02te\x10\x9d\x01\x12\x07\n\x02tg\x10\x9e\x01\x12\x07\n\x02th\x10\x9f\x01\x12\x07\n\x02ti\x10\xa0\x01\x12\x07\n\x02tk\x10\xa1\x01\x12\x07\n\x02tl\x10\xa2\x01\x12\x07\n\x02tn\x10\xa3\x01\x12\x07\n\x02to\x10\xa4\x01\x12\x07\n\x02tr\x10\xa5\x01\x12\x07\n\x02ts\x10\xa6\x01\x12\x07\n\x02tt\x10\xa7\x01\x12\x07\n\x02tw\x10\xa8\x01\x12\x07\n\x02ty\x10\xa9\x01\x12\x07\n\x02ug\x10\xaa\x01\x12\x07\n\x02uk\x10\xab\x01\x12\x07\n\x02ur\x10\xac\x01\x12\x07\n\x02uz\x10\xad\x01\x12\x07\n\x02ve\x10\xae\x01\x12\x07\n\x02vi\x10\xaf\x01\x12\x07\n\x02vo\x10\xb0\x01\x12\x07\n\x02wa\x10\xb1\x01\x12\x07\n\x02wo\x10\xb2\x01\x12\x07\n\x02xh\x10\xb3\x01\x12\x07\n\x02yi\x10\xb4\x01\x12\x07\n\x02yo\x10\xb5\x01\x12\x07\n\x02za\x10\xb6\x01\x12\x07\n\x02zh\x10\xb7\x01\x12\x07\n\x02zu\x10\xb8\x01\"\xaa\x10\n\x06Script\x12\x12\n\x0eUNKNOWN_SCRIPT\x10\x00\x12\x08\n\x04\x41\x64lm\x10\x01\x12\x08\n\x04\x41\x66\x61k\x10\x02\x12\x08\n\x04\x41ghb\x10\x03\x12\x08\n\x04\x41hom\x10\x04\x12\x08\n\x04\x41rab\x10\x05\x12\x08\n\x04\x41ran\x10\x06\x12\x08\n\x04\x41rmi\x10\x07\x12\x08\n\x04\x41rmn\x10\x08\x12\x08\n\x04\x41vst\x10\t\x12\x08\n\x04\x42\x61li\x10\n\x12\x08\n\x04\x42\x61mu\x10\x0b\x12\x08\n\x04\x42\x61ss\x10\x0c\x12\x08\n\x04\x42\x61tk\x10\r\x12\x08\n\x04\x42\x65ng\x10\x0e\x12\x08\n\x04\x42hks\x10\x0f\x12\x08\n\x04\x42lis\x10\x10\x12\x08\n\x04\x42opo\x10\x11\x12\x08\n\x04\x42rah\x10\x12\x12\x08\n\x04\x42rai\x10\x13\x12\x08\n\x04\x42ugi\x10\x14\x12\x08\n\x04\x42uhd\x10\x15\x12\x08\n\x04\x43\x61km\x10\x16\x12\x08\n\x04\x43\x61ns\x10\x17\x12\x08\n\x04\x43\x61ri\x10\x18\x12\x08\n\x04\x43ham\x10\x19\x12\x08\n\x04\x43her\x10\x1a\x12\x08\n\x04\x43irt\x10\x1b\x12\x08\n\x04\x43opt\x10\x1c\x12\x08\n\x04\x43pmn\x10\x1d\x12\x08\n\x04\x43prt\x10\x1e\x12\x08\n\x04\x43yrl\x10\x1f\x12\x08\n\x04\x43yrs\x10 \x12\x08\n\x04\x44\x65va\x10!\x12\x08\n\x04\x44ogr\x10\"\x12\x08\n\x04\x44srt\x10#\x12\x08\n\x04\x44upl\x10$\x12\x08\n\x04\x45gyd\x10%\x12\x08\n\x04\x45gyh\x10&\x12\x08\n\x04\x45gyp\x10\'\x12\x08\n\x04\x45lba\x10(\x12\x08\n\x04\x45lym\x10)\x12\x08\n\x04\x45thi\x10*\x12\x08\n\x04Geok\x10+\x12\x08\n\x04Geor\x10,\x12\x08\n\x04Glag\x10-\x12\x08\n\x04Gong\x10.\x12\x08\n\x04Gonm\x10/\x12\x08\n\x04Goth\x10\x30\x12\x08\n\x04Gran\x10\x31\x12\x08\n\x04Grek\x10\x32\x12\x08\n\x04Gujr\x10\x33\x12\x08\n\x04Guru\x10\x34\x12\x08\n\x04Hanb\x10\x35\x12\x08\n\x04Hang\x10\x36\x12\x08\n\x04Hani\x10\x37\x12\x08\n\x04Hano\x10\x38\x12\x08\n\x04Hans\x10\x39\x12\x08\n\x04Hant\x10:\x12\x08\n\x04Hatr\x10;\x12\x08\n\x04Hebr\x10<\x12\x08\n\x04Hira\x10=\x12\x08\n\x04Hluw\x10>\x12\x08\n\x04Hmng\x10?\x12\x08\n\x04Hmnp\x10@\x12\x08\n\x04Hrkt\x10\x41\x12\x08\n\x04Hung\x10\x42\x12\x08\n\x04Inds\x10\x43\x12\x08\n\x04Ital\x10\x44\x12\x08\n\x04Jamo\x10\x45\x12\x08\n\x04Java\x10\x46\x12\x08\n\x04Jpan\x10G\x12\x08\n\x04Jurc\x10H\x12\x08\n\x04Kali\x10I\x12\x08\n\x04Kana\x10J\x12\x08\n\x04Khar\x10K\x12\x08\n\x04Khmr\x10L\x12\x08\n\x04Khoj\x10M\x12\x08\n\x04Kitl\x10N\x12\x08\n\x04Kits\x10O\x12\x08\n\x04Knda\x10P\x12\x08\n\x04Kore\x10Q\x12\x08\n\x04Kpel\x10R\x12\x08\n\x04Kthi\x10S\x12\x08\n\x04Lana\x10T\x12\x08\n\x04Laoo\x10U\x12\x08\n\x04Latf\x10V\x12\x08\n\x04Latg\x10W\x12\x08\n\x04Latn\x10X\x12\x08\n\x04Leke\x10Y\x12\x08\n\x04Lepc\x10Z\x12\x08\n\x04Limb\x10[\x12\x08\n\x04Lina\x10\\\x12\x08\n\x04Linb\x10]\x12\x08\n\x04Lisu\x10^\x12\x08\n\x04Loma\x10_\x12\x08\n\x04Lyci\x10`\x12\x08\n\x04Lydi\x10\x61\x12\x08\n\x04Mahj\x10\x62\x12\x08\n\x04Maka\x10\x63\x12\x08\n\x04Mand\x10\x64\x12\x08\n\x04Mani\x10\x65\x12\x08\n\x04Marc\x10\x66\x12\x08\n\x04Maya\x10g\x12\x08\n\x04Medf\x10h\x12\x08\n\x04Mend\x10i\x12\x08\n\x04Merc\x10j\x12\x08\n\x04Mero\x10k\x12\x08\n\x04Mlym\x10l\x12\x08\n\x04Modi\x10m\x12\x08\n\x04Mong\x10n\x12\x08\n\x04Moon\x10o\x12\x08\n\x04Mroo\x10p\x12\x08\n\x04Mtei\x10q\x12\x08\n\x04Mult\x10r\x12\x08\n\x04Mymr\x10s\x12\x08\n\x04Nand\x10t\x12\x08\n\x04Narb\x10u\x12\x08\n\x04Nbat\x10v\x12\x08\n\x04Newa\x10w\x12\x08\n\x04Nkdb\x10x\x12\x08\n\x04Nkgb\x10y\x12\x08\n\x04Nkoo\x10z\x12\x08\n\x04Nshu\x10{\x12\x08\n\x04Ogam\x10|\x12\x08\n\x04Olck\x10}\x12\x08\n\x04Orkh\x10~\x12\x08\n\x04Orya\x10\x7f\x12\t\n\x04Osge\x10\x80\x01\x12\t\n\x04Osma\x10\x81\x01\x12\t\n\x04Palm\x10\x82\x01\x12\t\n\x04Pauc\x10\x83\x01\x12\t\n\x04Perm\x10\x84\x01\x12\t\n\x04Phag\x10\x85\x01\x12\t\n\x04Phli\x10\x86\x01\x12\t\n\x04Phlp\x10\x87\x01\x12\t\n\x04Phlv\x10\x88\x01\x12\t\n\x04Phnx\x10\x89\x01\x12\t\n\x04Plrd\x10\x8a\x01\x12\t\n\x04Piqd\x10\x8b\x01\x12\t\n\x04Prti\x10\x8c\x01\x12\t\n\x04Qaaa\x10\x8d\x01\x12\t\n\x04Qabx\x10\x8e\x01\x12\t\n\x04Rjng\x10\x8f\x01\x12\t\n\x04Rohg\x10\x90\x01\x12\t\n\x04Roro\x10\x91\x01\x12\t\n\x04Runr\x10\x92\x01\x12\t\n\x04Samr\x10\x93\x01\x12\t\n\x04Sara\x10\x94\x01\x12\t\n\x04Sarb\x10\x95\x01\x12\t\n\x04Saur\x10\x96\x01\x12\t\n\x04Sgnw\x10\x97\x01\x12\t\n\x04Shaw\x10\x98\x01\x12\t\n\x04Shrd\x10\x99\x01\x12\t\n\x04Shui\x10\x9a\x01\x12\t\n\x04Sidd\x10\x9b\x01\x12\t\n\x04Sind\x10\x9c\x01\x12\t\n\x04Sinh\x10\x9d\x01\x12\t\n\x04Sogd\x10\x9e\x01\x12\t\n\x04Sogo\x10\x9f\x01\x12\t\n\x04Sora\x10\xa0\x01\x12\t\n\x04Soyo\x10\xa1\x01\x12\t\n\x04Sund\x10\xa2\x01\x12\t\n\x04Sylo\x10\xa3\x01\x12\t\n\x04Syrc\x10\xa4\x01\x12\t\n\x04Syre\x10\xa5\x01\x12\t\n\x04Syrj\x10\xa6\x01\x12\t\n\x04Syrn\x10\xa7\x01\x12\t\n\x04Tagb\x10\xa8\x01\x12\t\n\x04Takr\x10\xa9\x01\x12\t\n\x04Tale\x10\xaa\x01\x12\t\n\x04Talu\x10\xab\x01\x12\t\n\x04Taml\x10\xac\x01\x12\t\n\x04Tang\x10\xad\x01\x12\t\n\x04Tavt\x10\xae\x01\x12\t\n\x04Telu\x10\xaf\x01\x12\t\n\x04Teng\x10\xb0\x01\x12\t\n\x04Tfng\x10\xb1\x01\x12\t\n\x04Tglg\x10\xb2\x01\x12\t\n\x04Thaa\x10\xb3\x01\x12\t\n\x04Thai\x10\xb4\x01\x12\t\n\x04Tibt\x10\xb5\x01\x12\t\n\x04Tirh\x10\xb6\x01\x12\t\n\x04Ugar\x10\xb7\x01\x12\t\n\x04Vaii\x10\xb8\x01\x12\t\n\x04Visp\x10\xb9\x01\x12\t\n\x04Wara\x10\xba\x01\x12\t\n\x04Wcho\x10\xbb\x01\x12\t\n\x04Wole\x10\xbc\x01\x12\t\n\x04Xpeo\x10\xbd\x01\x12\t\n\x04Xsux\x10\xbe\x01\x12\t\n\x04Yiii\x10\xbf\x01\x12\t\n\x04Zanb\x10\xc0\x01\x12\t\n\x04Zinh\x10\xc1\x01\x12\t\n\x04Zmth\x10\xc2\x01\x12\t\n\x04Zsye\x10\xc3\x01\x12\t\n\x04Zsym\x10\xc4\x01\x12\t\n\x04Zxxx\x10\xc5\x01\x12\t\n\x04Zyyy\x10\xc6\x01\x12\t\n\x04Zzzz\x10\xc7\x01\"\xec)\n\x08Location\x12%\n\x07\x63ountry\x18\x01 \x01(\x0e\x32\x14.pb.Location.Country\x12\r\n\x05state\x18\x02 \x01(\t\x12\x0c\n\x04\x63ity\x18\x03 \x01(\t\x12\x0c\n\x04\x63ode\x18\x04 \x01(\t\x12\x10\n\x08latitude\x18\x05 \x01(\x11\x12\x11\n\tlongitude\x18\x06 \x01(\x11\"\xe8(\n\x07\x43ountry\x12\x13\n\x0fUNKNOWN_COUNTRY\x10\x00\x12\x06\n\x02\x41\x46\x10\x01\x12\x06\n\x02\x41X\x10\x02\x12\x06\n\x02\x41L\x10\x03\x12\x06\n\x02\x44Z\x10\x04\x12\x06\n\x02\x41S\x10\x05\x12\x06\n\x02\x41\x44\x10\x06\x12\x06\n\x02\x41O\x10\x07\x12\x06\n\x02\x41I\x10\x08\x12\x06\n\x02\x41Q\x10\t\x12\x06\n\x02\x41G\x10\n\x12\x06\n\x02\x41R\x10\x0b\x12\x06\n\x02\x41M\x10\x0c\x12\x06\n\x02\x41W\x10\r\x12\x06\n\x02\x41U\x10\x0e\x12\x06\n\x02\x41T\x10\x0f\x12\x06\n\x02\x41Z\x10\x10\x12\x06\n\x02\x42S\x10\x11\x12\x06\n\x02\x42H\x10\x12\x12\x06\n\x02\x42\x44\x10\x13\x12\x06\n\x02\x42\x42\x10\x14\x12\x06\n\x02\x42Y\x10\x15\x12\x06\n\x02\x42\x45\x10\x16\x12\x06\n\x02\x42Z\x10\x17\x12\x06\n\x02\x42J\x10\x18\x12\x06\n\x02\x42M\x10\x19\x12\x06\n\x02\x42T\x10\x1a\x12\x06\n\x02\x42O\x10\x1b\x12\x06\n\x02\x42Q\x10\x1c\x12\x06\n\x02\x42\x41\x10\x1d\x12\x06\n\x02\x42W\x10\x1e\x12\x06\n\x02\x42V\x10\x1f\x12\x06\n\x02\x42R\x10 \x12\x06\n\x02IO\x10!\x12\x06\n\x02\x42N\x10\"\x12\x06\n\x02\x42G\x10#\x12\x06\n\x02\x42\x46\x10$\x12\x06\n\x02\x42I\x10%\x12\x06\n\x02KH\x10&\x12\x06\n\x02\x43M\x10\'\x12\x06\n\x02\x43\x41\x10(\x12\x06\n\x02\x43V\x10)\x12\x06\n\x02KY\x10*\x12\x06\n\x02\x43\x46\x10+\x12\x06\n\x02TD\x10,\x12\x06\n\x02\x43L\x10-\x12\x06\n\x02\x43N\x10.\x12\x06\n\x02\x43X\x10/\x12\x06\n\x02\x43\x43\x10\x30\x12\x06\n\x02\x43O\x10\x31\x12\x06\n\x02KM\x10\x32\x12\x06\n\x02\x43G\x10\x33\x12\x06\n\x02\x43\x44\x10\x34\x12\x06\n\x02\x43K\x10\x35\x12\x06\n\x02\x43R\x10\x36\x12\x06\n\x02\x43I\x10\x37\x12\x06\n\x02HR\x10\x38\x12\x06\n\x02\x43U\x10\x39\x12\x06\n\x02\x43W\x10:\x12\x06\n\x02\x43Y\x10;\x12\x06\n\x02\x43Z\x10<\x12\x06\n\x02\x44K\x10=\x12\x06\n\x02\x44J\x10>\x12\x06\n\x02\x44M\x10?\x12\x06\n\x02\x44O\x10@\x12\x06\n\x02\x45\x43\x10\x41\x12\x06\n\x02\x45G\x10\x42\x12\x06\n\x02SV\x10\x43\x12\x06\n\x02GQ\x10\x44\x12\x06\n\x02\x45R\x10\x45\x12\x06\n\x02\x45\x45\x10\x46\x12\x06\n\x02\x45T\x10G\x12\x06\n\x02\x46K\x10H\x12\x06\n\x02\x46O\x10I\x12\x06\n\x02\x46J\x10J\x12\x06\n\x02\x46I\x10K\x12\x06\n\x02\x46R\x10L\x12\x06\n\x02GF\x10M\x12\x06\n\x02PF\x10N\x12\x06\n\x02TF\x10O\x12\x06\n\x02GA\x10P\x12\x06\n\x02GM\x10Q\x12\x06\n\x02GE\x10R\x12\x06\n\x02\x44\x45\x10S\x12\x06\n\x02GH\x10T\x12\x06\n\x02GI\x10U\x12\x06\n\x02GR\x10V\x12\x06\n\x02GL\x10W\x12\x06\n\x02GD\x10X\x12\x06\n\x02GP\x10Y\x12\x06\n\x02GU\x10Z\x12\x06\n\x02GT\x10[\x12\x06\n\x02GG\x10\\\x12\x06\n\x02GN\x10]\x12\x06\n\x02GW\x10^\x12\x06\n\x02GY\x10_\x12\x06\n\x02HT\x10`\x12\x06\n\x02HM\x10\x61\x12\x06\n\x02VA\x10\x62\x12\x06\n\x02HN\x10\x63\x12\x06\n\x02HK\x10\x64\x12\x06\n\x02HU\x10\x65\x12\x06\n\x02IS\x10\x66\x12\x06\n\x02IN\x10g\x12\x06\n\x02ID\x10h\x12\x06\n\x02IR\x10i\x12\x06\n\x02IQ\x10j\x12\x06\n\x02IE\x10k\x12\x06\n\x02IM\x10l\x12\x06\n\x02IL\x10m\x12\x06\n\x02IT\x10n\x12\x06\n\x02JM\x10o\x12\x06\n\x02JP\x10p\x12\x06\n\x02JE\x10q\x12\x06\n\x02JO\x10r\x12\x06\n\x02KZ\x10s\x12\x06\n\x02KE\x10t\x12\x06\n\x02KI\x10u\x12\x06\n\x02KP\x10v\x12\x06\n\x02KR\x10w\x12\x06\n\x02KW\x10x\x12\x06\n\x02KG\x10y\x12\x06\n\x02LA\x10z\x12\x06\n\x02LV\x10{\x12\x06\n\x02LB\x10|\x12\x06\n\x02LS\x10}\x12\x06\n\x02LR\x10~\x12\x06\n\x02LY\x10\x7f\x12\x07\n\x02LI\x10\x80\x01\x12\x07\n\x02LT\x10\x81\x01\x12\x07\n\x02LU\x10\x82\x01\x12\x07\n\x02MO\x10\x83\x01\x12\x07\n\x02MK\x10\x84\x01\x12\x07\n\x02MG\x10\x85\x01\x12\x07\n\x02MW\x10\x86\x01\x12\x07\n\x02MY\x10\x87\x01\x12\x07\n\x02MV\x10\x88\x01\x12\x07\n\x02ML\x10\x89\x01\x12\x07\n\x02MT\x10\x8a\x01\x12\x07\n\x02MH\x10\x8b\x01\x12\x07\n\x02MQ\x10\x8c\x01\x12\x07\n\x02MR\x10\x8d\x01\x12\x07\n\x02MU\x10\x8e\x01\x12\x07\n\x02YT\x10\x8f\x01\x12\x07\n\x02MX\x10\x90\x01\x12\x07\n\x02\x46M\x10\x91\x01\x12\x07\n\x02MD\x10\x92\x01\x12\x07\n\x02MC\x10\x93\x01\x12\x07\n\x02MN\x10\x94\x01\x12\x07\n\x02ME\x10\x95\x01\x12\x07\n\x02MS\x10\x96\x01\x12\x07\n\x02MA\x10\x97\x01\x12\x07\n\x02MZ\x10\x98\x01\x12\x07\n\x02MM\x10\x99\x01\x12\x07\n\x02NA\x10\x9a\x01\x12\x07\n\x02NR\x10\x9b\x01\x12\x07\n\x02NP\x10\x9c\x01\x12\x07\n\x02NL\x10\x9d\x01\x12\x07\n\x02NC\x10\x9e\x01\x12\x07\n\x02NZ\x10\x9f\x01\x12\x07\n\x02NI\x10\xa0\x01\x12\x07\n\x02NE\x10\xa1\x01\x12\x07\n\x02NG\x10\xa2\x01\x12\x07\n\x02NU\x10\xa3\x01\x12\x07\n\x02NF\x10\xa4\x01\x12\x07\n\x02MP\x10\xa5\x01\x12\x07\n\x02NO\x10\xa6\x01\x12\x07\n\x02OM\x10\xa7\x01\x12\x07\n\x02PK\x10\xa8\x01\x12\x07\n\x02PW\x10\xa9\x01\x12\x07\n\x02PS\x10\xaa\x01\x12\x07\n\x02PA\x10\xab\x01\x12\x07\n\x02PG\x10\xac\x01\x12\x07\n\x02PY\x10\xad\x01\x12\x07\n\x02PE\x10\xae\x01\x12\x07\n\x02PH\x10\xaf\x01\x12\x07\n\x02PN\x10\xb0\x01\x12\x07\n\x02PL\x10\xb1\x01\x12\x07\n\x02PT\x10\xb2\x01\x12\x07\n\x02PR\x10\xb3\x01\x12\x07\n\x02QA\x10\xb4\x01\x12\x07\n\x02RE\x10\xb5\x01\x12\x07\n\x02RO\x10\xb6\x01\x12\x07\n\x02RU\x10\xb7\x01\x12\x07\n\x02RW\x10\xb8\x01\x12\x07\n\x02\x42L\x10\xb9\x01\x12\x07\n\x02SH\x10\xba\x01\x12\x07\n\x02KN\x10\xbb\x01\x12\x07\n\x02LC\x10\xbc\x01\x12\x07\n\x02MF\x10\xbd\x01\x12\x07\n\x02PM\x10\xbe\x01\x12\x07\n\x02VC\x10\xbf\x01\x12\x07\n\x02WS\x10\xc0\x01\x12\x07\n\x02SM\x10\xc1\x01\x12\x07\n\x02ST\x10\xc2\x01\x12\x07\n\x02SA\x10\xc3\x01\x12\x07\n\x02SN\x10\xc4\x01\x12\x07\n\x02RS\x10\xc5\x01\x12\x07\n\x02SC\x10\xc6\x01\x12\x07\n\x02SL\x10\xc7\x01\x12\x07\n\x02SG\x10\xc8\x01\x12\x07\n\x02SX\x10\xc9\x01\x12\x07\n\x02SK\x10\xca\x01\x12\x07\n\x02SI\x10\xcb\x01\x12\x07\n\x02SB\x10\xcc\x01\x12\x07\n\x02SO\x10\xcd\x01\x12\x07\n\x02ZA\x10\xce\x01\x12\x07\n\x02GS\x10\xcf\x01\x12\x07\n\x02SS\x10\xd0\x01\x12\x07\n\x02\x45S\x10\xd1\x01\x12\x07\n\x02LK\x10\xd2\x01\x12\x07\n\x02SD\x10\xd3\x01\x12\x07\n\x02SR\x10\xd4\x01\x12\x07\n\x02SJ\x10\xd5\x01\x12\x07\n\x02SZ\x10\xd6\x01\x12\x07\n\x02SE\x10\xd7\x01\x12\x07\n\x02\x43H\x10\xd8\x01\x12\x07\n\x02SY\x10\xd9\x01\x12\x07\n\x02TW\x10\xda\x01\x12\x07\n\x02TJ\x10\xdb\x01\x12\x07\n\x02TZ\x10\xdc\x01\x12\x07\n\x02TH\x10\xdd\x01\x12\x07\n\x02TL\x10\xde\x01\x12\x07\n\x02TG\x10\xdf\x01\x12\x07\n\x02TK\x10\xe0\x01\x12\x07\n\x02TO\x10\xe1\x01\x12\x07\n\x02TT\x10\xe2\x01\x12\x07\n\x02TN\x10\xe3\x01\x12\x07\n\x02TR\x10\xe4\x01\x12\x07\n\x02TM\x10\xe5\x01\x12\x07\n\x02TC\x10\xe6\x01\x12\x07\n\x02TV\x10\xe7\x01\x12\x07\n\x02UG\x10\xe8\x01\x12\x07\n\x02UA\x10\xe9\x01\x12\x07\n\x02\x41\x45\x10\xea\x01\x12\x07\n\x02GB\x10\xeb\x01\x12\x07\n\x02US\x10\xec\x01\x12\x07\n\x02UM\x10\xed\x01\x12\x07\n\x02UY\x10\xee\x01\x12\x07\n\x02UZ\x10\xef\x01\x12\x07\n\x02VU\x10\xf0\x01\x12\x07\n\x02VE\x10\xf1\x01\x12\x07\n\x02VN\x10\xf2\x01\x12\x07\n\x02VG\x10\xf3\x01\x12\x07\n\x02VI\x10\xf4\x01\x12\x07\n\x02WF\x10\xf5\x01\x12\x07\n\x02\x45H\x10\xf6\x01\x12\x07\n\x02YE\x10\xf7\x01\x12\x07\n\x02ZM\x10\xf8\x01\x12\x07\n\x02ZW\x10\xf9\x01\x12\t\n\x04R001\x10\xfa\x01\x12\t\n\x04R002\x10\xfb\x01\x12\t\n\x04R015\x10\xfc\x01\x12\t\n\x04R012\x10\xfd\x01\x12\t\n\x04R818\x10\xfe\x01\x12\t\n\x04R434\x10\xff\x01\x12\t\n\x04R504\x10\x80\x02\x12\t\n\x04R729\x10\x81\x02\x12\t\n\x04R788\x10\x82\x02\x12\t\n\x04R732\x10\x83\x02\x12\t\n\x04R202\x10\x84\x02\x12\t\n\x04R014\x10\x85\x02\x12\t\n\x04R086\x10\x86\x02\x12\t\n\x04R108\x10\x87\x02\x12\t\n\x04R174\x10\x88\x02\x12\t\n\x04R262\x10\x89\x02\x12\t\n\x04R232\x10\x8a\x02\x12\t\n\x04R231\x10\x8b\x02\x12\t\n\x04R260\x10\x8c\x02\x12\t\n\x04R404\x10\x8d\x02\x12\t\n\x04R450\x10\x8e\x02\x12\t\n\x04R454\x10\x8f\x02\x12\t\n\x04R480\x10\x90\x02\x12\t\n\x04R175\x10\x91\x02\x12\t\n\x04R508\x10\x92\x02\x12\t\n\x04R638\x10\x93\x02\x12\t\n\x04R646\x10\x94\x02\x12\t\n\x04R690\x10\x95\x02\x12\t\n\x04R706\x10\x96\x02\x12\t\n\x04R728\x10\x97\x02\x12\t\n\x04R800\x10\x98\x02\x12\t\n\x04R834\x10\x99\x02\x12\t\n\x04R894\x10\x9a\x02\x12\t\n\x04R716\x10\x9b\x02\x12\t\n\x04R017\x10\x9c\x02\x12\t\n\x04R024\x10\x9d\x02\x12\t\n\x04R120\x10\x9e\x02\x12\t\n\x04R140\x10\x9f\x02\x12\t\n\x04R148\x10\xa0\x02\x12\t\n\x04R178\x10\xa1\x02\x12\t\n\x04R180\x10\xa2\x02\x12\t\n\x04R226\x10\xa3\x02\x12\t\n\x04R266\x10\xa4\x02\x12\t\n\x04R678\x10\xa5\x02\x12\t\n\x04R018\x10\xa6\x02\x12\t\n\x04R072\x10\xa7\x02\x12\t\n\x04R748\x10\xa8\x02\x12\t\n\x04R426\x10\xa9\x02\x12\t\n\x04R516\x10\xaa\x02\x12\t\n\x04R710\x10\xab\x02\x12\t\n\x04R011\x10\xac\x02\x12\t\n\x04R204\x10\xad\x02\x12\t\n\x04R854\x10\xae\x02\x12\t\n\x04R132\x10\xaf\x02\x12\t\n\x04R384\x10\xb0\x02\x12\t\n\x04R270\x10\xb1\x02\x12\t\n\x04R288\x10\xb2\x02\x12\t\n\x04R324\x10\xb3\x02\x12\t\n\x04R624\x10\xb4\x02\x12\t\n\x04R430\x10\xb5\x02\x12\t\n\x04R466\x10\xb6\x02\x12\t\n\x04R478\x10\xb7\x02\x12\t\n\x04R562\x10\xb8\x02\x12\t\n\x04R566\x10\xb9\x02\x12\t\n\x04R654\x10\xba\x02\x12\t\n\x04R686\x10\xbb\x02\x12\t\n\x04R694\x10\xbc\x02\x12\t\n\x04R768\x10\xbd\x02\x12\t\n\x04R019\x10\xbe\x02\x12\t\n\x04R419\x10\xbf\x02\x12\t\n\x04R029\x10\xc0\x02\x12\t\n\x04R660\x10\xc1\x02\x12\t\n\x04R028\x10\xc2\x02\x12\t\n\x04R533\x10\xc3\x02\x12\t\n\x04R044\x10\xc4\x02\x12\t\n\x04R052\x10\xc5\x02\x12\t\n\x04R535\x10\xc6\x02\x12\t\n\x04R092\x10\xc7\x02\x12\t\n\x04R136\x10\xc8\x02\x12\t\n\x04R192\x10\xc9\x02\x12\t\n\x04R531\x10\xca\x02\x12\t\n\x04R212\x10\xcb\x02\x12\t\n\x04R214\x10\xcc\x02\x12\t\n\x04R308\x10\xcd\x02\x12\t\n\x04R312\x10\xce\x02\x12\t\n\x04R332\x10\xcf\x02\x12\t\n\x04R388\x10\xd0\x02\x12\t\n\x04R474\x10\xd1\x02\x12\t\n\x04R500\x10\xd2\x02\x12\t\n\x04R630\x10\xd3\x02\x12\t\n\x04R652\x10\xd4\x02\x12\t\n\x04R659\x10\xd5\x02\x12\t\n\x04R662\x10\xd6\x02\x12\t\n\x04R663\x10\xd7\x02\x12\t\n\x04R670\x10\xd8\x02\x12\t\n\x04R534\x10\xd9\x02\x12\t\n\x04R780\x10\xda\x02\x12\t\n\x04R796\x10\xdb\x02\x12\t\n\x04R850\x10\xdc\x02\x12\t\n\x04R013\x10\xdd\x02\x12\t\n\x04R084\x10\xde\x02\x12\t\n\x04R188\x10\xdf\x02\x12\t\n\x04R222\x10\xe0\x02\x12\t\n\x04R320\x10\xe1\x02\x12\t\n\x04R340\x10\xe2\x02\x12\t\n\x04R484\x10\xe3\x02\x12\t\n\x04R558\x10\xe4\x02\x12\t\n\x04R591\x10\xe5\x02\x12\t\n\x04R005\x10\xe6\x02\x12\t\n\x04R032\x10\xe7\x02\x12\t\n\x04R068\x10\xe8\x02\x12\t\n\x04R074\x10\xe9\x02\x12\t\n\x04R076\x10\xea\x02\x12\t\n\x04R152\x10\xeb\x02\x12\t\n\x04R170\x10\xec\x02\x12\t\n\x04R218\x10\xed\x02\x12\t\n\x04R238\x10\xee\x02\x12\t\n\x04R254\x10\xef\x02\x12\t\n\x04R328\x10\xf0\x02\x12\t\n\x04R600\x10\xf1\x02\x12\t\n\x04R604\x10\xf2\x02\x12\t\n\x04R239\x10\xf3\x02\x12\t\n\x04R740\x10\xf4\x02\x12\t\n\x04R858\x10\xf5\x02\x12\t\n\x04R862\x10\xf6\x02\x12\t\n\x04R021\x10\xf7\x02\x12\t\n\x04R060\x10\xf8\x02\x12\t\n\x04R124\x10\xf9\x02\x12\t\n\x04R304\x10\xfa\x02\x12\t\n\x04R666\x10\xfb\x02\x12\t\n\x04R840\x10\xfc\x02\x12\t\n\x04R010\x10\xfd\x02\x12\t\n\x04R142\x10\xfe\x02\x12\t\n\x04R143\x10\xff\x02\x12\t\n\x04R398\x10\x80\x03\x12\t\n\x04R417\x10\x81\x03\x12\t\n\x04R762\x10\x82\x03\x12\t\n\x04R795\x10\x83\x03\x12\t\n\x04R860\x10\x84\x03\x12\t\n\x04R030\x10\x85\x03\x12\t\n\x04R156\x10\x86\x03\x12\t\n\x04R344\x10\x87\x03\x12\t\n\x04R446\x10\x88\x03\x12\t\n\x04R408\x10\x89\x03\x12\t\n\x04R392\x10\x8a\x03\x12\t\n\x04R496\x10\x8b\x03\x12\t\n\x04R410\x10\x8c\x03\x12\t\n\x04R035\x10\x8d\x03\x12\t\n\x04R096\x10\x8e\x03\x12\t\n\x04R116\x10\x8f\x03\x12\t\n\x04R360\x10\x90\x03\x12\t\n\x04R418\x10\x91\x03\x12\t\n\x04R458\x10\x92\x03\x12\t\n\x04R104\x10\x93\x03\x12\t\n\x04R608\x10\x94\x03\x12\t\n\x04R702\x10\x95\x03\x12\t\n\x04R764\x10\x96\x03\x12\t\n\x04R626\x10\x97\x03\x12\t\n\x04R704\x10\x98\x03\x12\t\n\x04R034\x10\x99\x03\x12\t\n\x04R004\x10\x9a\x03\x12\t\n\x04R050\x10\x9b\x03\x12\t\n\x04R064\x10\x9c\x03\x12\t\n\x04R356\x10\x9d\x03\x12\t\n\x04R364\x10\x9e\x03\x12\t\n\x04R462\x10\x9f\x03\x12\t\n\x04R524\x10\xa0\x03\x12\t\n\x04R586\x10\xa1\x03\x12\t\n\x04R144\x10\xa2\x03\x12\t\n\x04R145\x10\xa3\x03\x12\t\n\x04R051\x10\xa4\x03\x12\t\n\x04R031\x10\xa5\x03\x12\t\n\x04R048\x10\xa6\x03\x12\t\n\x04R196\x10\xa7\x03\x12\t\n\x04R268\x10\xa8\x03\x12\t\n\x04R368\x10\xa9\x03\x12\t\n\x04R376\x10\xaa\x03\x12\t\n\x04R400\x10\xab\x03\x12\t\n\x04R414\x10\xac\x03\x12\t\n\x04R422\x10\xad\x03\x12\t\n\x04R512\x10\xae\x03\x12\t\n\x04R634\x10\xaf\x03\x12\t\n\x04R682\x10\xb0\x03\x12\t\n\x04R275\x10\xb1\x03\x12\t\n\x04R760\x10\xb2\x03\x12\t\n\x04R792\x10\xb3\x03\x12\t\n\x04R784\x10\xb4\x03\x12\t\n\x04R887\x10\xb5\x03\x12\t\n\x04R150\x10\xb6\x03\x12\t\n\x04R151\x10\xb7\x03\x12\t\n\x04R112\x10\xb8\x03\x12\t\n\x04R100\x10\xb9\x03\x12\t\n\x04R203\x10\xba\x03\x12\t\n\x04R348\x10\xbb\x03\x12\t\n\x04R616\x10\xbc\x03\x12\t\n\x04R498\x10\xbd\x03\x12\t\n\x04R642\x10\xbe\x03\x12\t\n\x04R643\x10\xbf\x03\x12\t\n\x04R703\x10\xc0\x03\x12\t\n\x04R804\x10\xc1\x03\x12\t\n\x04R154\x10\xc2\x03\x12\t\n\x04R248\x10\xc3\x03\x12\t\n\x04R830\x10\xc4\x03\x12\t\n\x04R831\x10\xc5\x03\x12\t\n\x04R832\x10\xc6\x03\x12\t\n\x04R680\x10\xc7\x03\x12\t\n\x04R208\x10\xc8\x03\x12\t\n\x04R233\x10\xc9\x03\x12\t\n\x04R234\x10\xca\x03\x12\t\n\x04R246\x10\xcb\x03\x12\t\n\x04R352\x10\xcc\x03\x12\t\n\x04R372\x10\xcd\x03\x12\t\n\x04R833\x10\xce\x03\x12\t\n\x04R428\x10\xcf\x03\x12\t\n\x04R440\x10\xd0\x03\x12\t\n\x04R578\x10\xd1\x03\x12\t\n\x04R744\x10\xd2\x03\x12\t\n\x04R752\x10\xd3\x03\x12\t\n\x04R826\x10\xd4\x03\x12\t\n\x04R039\x10\xd5\x03\x12\t\n\x04R008\x10\xd6\x03\x12\t\n\x04R020\x10\xd7\x03\x12\t\n\x04R070\x10\xd8\x03\x12\t\n\x04R191\x10\xd9\x03\x12\t\n\x04R292\x10\xda\x03\x12\t\n\x04R300\x10\xdb\x03\x12\t\n\x04R336\x10\xdc\x03\x12\t\n\x04R380\x10\xdd\x03\x12\t\n\x04R470\x10\xde\x03\x12\t\n\x04R499\x10\xdf\x03\x12\t\n\x04R807\x10\xe0\x03\x12\t\n\x04R620\x10\xe1\x03\x12\t\n\x04R674\x10\xe2\x03\x12\t\n\x04R688\x10\xe3\x03\x12\t\n\x04R705\x10\xe4\x03\x12\t\n\x04R724\x10\xe5\x03\x12\t\n\x04R155\x10\xe6\x03\x12\t\n\x04R040\x10\xe7\x03\x12\t\n\x04R056\x10\xe8\x03\x12\t\n\x04R250\x10\xe9\x03\x12\t\n\x04R276\x10\xea\x03\x12\t\n\x04R438\x10\xeb\x03\x12\t\n\x04R442\x10\xec\x03\x12\t\n\x04R492\x10\xed\x03\x12\t\n\x04R528\x10\xee\x03\x12\t\n\x04R756\x10\xef\x03\x12\t\n\x04R009\x10\xf0\x03\x12\t\n\x04R053\x10\xf1\x03\x12\t\n\x04R036\x10\xf2\x03\x12\t\n\x04R162\x10\xf3\x03\x12\t\n\x04R166\x10\xf4\x03\x12\t\n\x04R334\x10\xf5\x03\x12\t\n\x04R554\x10\xf6\x03\x12\t\n\x04R574\x10\xf7\x03\x12\t\n\x04R054\x10\xf8\x03\x12\t\n\x04R242\x10\xf9\x03\x12\t\n\x04R540\x10\xfa\x03\x12\t\n\x04R598\x10\xfb\x03\x12\t\n\x04R090\x10\xfc\x03\x12\t\n\x04R548\x10\xfd\x03\x12\t\n\x04R057\x10\xfe\x03\x12\t\n\x04R316\x10\xff\x03\x12\t\n\x04R296\x10\x80\x04\x12\t\n\x04R584\x10\x81\x04\x12\t\n\x04R583\x10\x82\x04\x12\t\n\x04R520\x10\x83\x04\x12\t\n\x04R580\x10\x84\x04\x12\t\n\x04R585\x10\x85\x04\x12\t\n\x04R581\x10\x86\x04\x12\t\n\x04R061\x10\x87\x04\x12\t\n\x04R016\x10\x88\x04\x12\t\n\x04R184\x10\x89\x04\x12\t\n\x04R258\x10\x8a\x04\x12\t\n\x04R570\x10\x8b\x04\x12\t\n\x04R612\x10\x8c\x04\x12\t\n\x04R882\x10\x8d\x04\x12\t\n\x04R772\x10\x8e\x04\x12\t\n\x04R776\x10\x8f\x04\x12\t\n\x04R798\x10\x90\x04\x12\t\n\x04R876\x10\x91\x04\x62\x06proto3') -) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - -_CLAIMLIST_LISTTYPE = _descriptor.EnumDescriptor( - name='ListType', - full_name='pb.ClaimList.ListType', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='COLLECTION', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='DERIVATION', index=1, number=2, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=852, - serialized_end=894, -) -_sym_db.RegisterEnumDescriptor(_CLAIMLIST_LISTTYPE) - -_FEE_CURRENCY = _descriptor.EnumDescriptor( - name='Currency', - full_name='pb.Fee.Currency', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='UNKNOWN_CURRENCY', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='LBC', index=1, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BTC', index=2, number=2, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='USD', index=3, number=3, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=1096, - serialized_end=1155, -) -_sym_db.RegisterEnumDescriptor(_FEE_CURRENCY) - -_SOFTWARE_OS = _descriptor.EnumDescriptor( - name='OS', - full_name='pb.Software.OS', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='UNKNOWN_OS', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ANY', index=1, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='LINUX', index=2, number=2, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='WINDOWS', index=3, number=3, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MAC', index=4, number=4, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ANDROID', index=5, number=5, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='IOS', index=6, number=6, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=1332, - serialized_end=1416, -) -_sym_db.RegisterEnumDescriptor(_SOFTWARE_OS) - -_LANGUAGE_LANGUAGE = _descriptor.EnumDescriptor( - name='Language', - full_name='pb.Language.Language', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='UNKNOWN_LANGUAGE', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='en', index=1, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='aa', index=2, number=2, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ab', index=3, number=3, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ae', index=4, number=4, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='af', index=5, number=5, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ak', index=6, number=6, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='am', index=7, number=7, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='an', index=8, number=8, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ar', index=9, number=9, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='as', index=10, number=10, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='av', index=11, number=11, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ay', index=12, number=12, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='az', index=13, number=13, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ba', index=14, number=14, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='be', index=15, number=15, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='bg', index=16, number=16, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='bh', index=17, number=17, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='bi', index=18, number=18, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='bm', index=19, number=19, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='bn', index=20, number=20, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='bo', index=21, number=21, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='br', index=22, number=22, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='bs', index=23, number=23, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ca', index=24, number=24, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ce', index=25, number=25, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ch', index=26, number=26, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='co', index=27, number=27, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='cr', index=28, number=28, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='cs', index=29, number=29, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='cu', index=30, number=30, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='cv', index=31, number=31, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='cy', index=32, number=32, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='da', index=33, number=33, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='de', index=34, number=34, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='dv', index=35, number=35, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='dz', index=36, number=36, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ee', index=37, number=37, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='el', index=38, number=38, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='eo', index=39, number=39, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='es', index=40, number=40, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='et', index=41, number=41, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='eu', index=42, number=42, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='fa', index=43, number=43, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ff', index=44, number=44, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='fi', index=45, number=45, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='fj', index=46, number=46, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='fo', index=47, number=47, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='fr', index=48, number=48, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='fy', index=49, number=49, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ga', index=50, number=50, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='gd', index=51, number=51, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='gl', index=52, number=52, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='gn', index=53, number=53, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='gu', index=54, number=54, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='gv', index=55, number=55, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ha', index=56, number=56, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='he', index=57, number=57, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='hi', index=58, number=58, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ho', index=59, number=59, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='hr', index=60, number=60, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ht', index=61, number=61, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='hu', index=62, number=62, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='hy', index=63, number=63, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='hz', index=64, number=64, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ia', index=65, number=65, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='id', index=66, number=66, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ie', index=67, number=67, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ig', index=68, number=68, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ii', index=69, number=69, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ik', index=70, number=70, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='io', index=71, number=71, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='is', index=72, number=72, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='it', index=73, number=73, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='iu', index=74, number=74, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ja', index=75, number=75, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='jv', index=76, number=76, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ka', index=77, number=77, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='kg', index=78, number=78, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ki', index=79, number=79, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='kj', index=80, number=80, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='kk', index=81, number=81, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='kl', index=82, number=82, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='km', index=83, number=83, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='kn', index=84, number=84, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ko', index=85, number=85, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='kr', index=86, number=86, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ks', index=87, number=87, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ku', index=88, number=88, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='kv', index=89, number=89, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='kw', index=90, number=90, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ky', index=91, number=91, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='la', index=92, number=92, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='lb', index=93, number=93, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='lg', index=94, number=94, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='li', index=95, number=95, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ln', index=96, number=96, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='lo', index=97, number=97, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='lt', index=98, number=98, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='lu', index=99, number=99, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='lv', index=100, number=100, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='mg', index=101, number=101, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='mh', index=102, number=102, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='mi', index=103, number=103, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='mk', index=104, number=104, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ml', index=105, number=105, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='mn', index=106, number=106, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='mr', index=107, number=107, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ms', index=108, number=108, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='mt', index=109, number=109, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='my', index=110, number=110, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='na', index=111, number=111, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='nb', index=112, number=112, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='nd', index=113, number=113, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ne', index=114, number=114, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ng', index=115, number=115, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='nl', index=116, number=116, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='nn', index=117, number=117, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='no', index=118, number=118, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='nr', index=119, number=119, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='nv', index=120, number=120, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ny', index=121, number=121, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='oc', index=122, number=122, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='oj', index=123, number=123, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='om', index=124, number=124, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='or', index=125, number=125, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='os', index=126, number=126, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='pa', index=127, number=127, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='pi', index=128, number=128, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='pl', index=129, number=129, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ps', index=130, number=130, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='pt', index=131, number=131, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='qu', index=132, number=132, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='rm', index=133, number=133, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='rn', index=134, number=134, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ro', index=135, number=135, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ru', index=136, number=136, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='rw', index=137, number=137, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='sa', index=138, number=138, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='sc', index=139, number=139, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='sd', index=140, number=140, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='se', index=141, number=141, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='sg', index=142, number=142, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='si', index=143, number=143, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='sk', index=144, number=144, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='sl', index=145, number=145, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='sm', index=146, number=146, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='sn', index=147, number=147, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='so', index=148, number=148, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='sq', index=149, number=149, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='sr', index=150, number=150, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ss', index=151, number=151, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='st', index=152, number=152, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='su', index=153, number=153, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='sv', index=154, number=154, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='sw', index=155, number=155, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ta', index=156, number=156, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='te', index=157, number=157, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='tg', index=158, number=158, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='th', index=159, number=159, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ti', index=160, number=160, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='tk', index=161, number=161, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='tl', index=162, number=162, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='tn', index=163, number=163, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='to', index=164, number=164, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='tr', index=165, number=165, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ts', index=166, number=166, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='tt', index=167, number=167, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='tw', index=168, number=168, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ty', index=169, number=169, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ug', index=170, number=170, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='uk', index=171, number=171, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ur', index=172, number=172, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='uz', index=173, number=173, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ve', index=174, number=174, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='vi', index=175, number=175, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='vo', index=176, number=176, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='wa', index=177, number=177, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='wo', index=178, number=178, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='xh', index=179, number=179, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='yi', index=180, number=180, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='yo', index=181, number=181, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='za', index=182, number=182, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='zh', index=183, number=183, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='zu', index=184, number=184, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=1548, - serialized_end=3109, -) -_sym_db.RegisterEnumDescriptor(_LANGUAGE_LANGUAGE) - -_LANGUAGE_SCRIPT = _descriptor.EnumDescriptor( - name='Script', - full_name='pb.Language.Script', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='UNKNOWN_SCRIPT', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Adlm', index=1, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Afak', index=2, number=2, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Aghb', index=3, number=3, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Ahom', index=4, number=4, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Arab', index=5, number=5, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Aran', index=6, number=6, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Armi', index=7, number=7, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Armn', index=8, number=8, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Avst', index=9, number=9, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Bali', index=10, number=10, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Bamu', index=11, number=11, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Bass', index=12, number=12, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Batk', index=13, number=13, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Beng', index=14, number=14, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Bhks', index=15, number=15, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Blis', index=16, number=16, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Bopo', index=17, number=17, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Brah', index=18, number=18, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Brai', index=19, number=19, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Bugi', index=20, number=20, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Buhd', index=21, number=21, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Cakm', index=22, number=22, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Cans', index=23, number=23, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Cari', index=24, number=24, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Cham', index=25, number=25, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Cher', index=26, number=26, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Cirt', index=27, number=27, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Copt', index=28, number=28, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Cpmn', index=29, number=29, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Cprt', index=30, number=30, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Cyrl', index=31, number=31, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Cyrs', index=32, number=32, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Deva', index=33, number=33, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Dogr', index=34, number=34, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Dsrt', index=35, number=35, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Dupl', index=36, number=36, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Egyd', index=37, number=37, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Egyh', index=38, number=38, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Egyp', index=39, number=39, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Elba', index=40, number=40, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Elym', index=41, number=41, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Ethi', index=42, number=42, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Geok', index=43, number=43, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Geor', index=44, number=44, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Glag', index=45, number=45, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Gong', index=46, number=46, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Gonm', index=47, number=47, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Goth', index=48, number=48, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Gran', index=49, number=49, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Grek', index=50, number=50, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Gujr', index=51, number=51, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Guru', index=52, number=52, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Hanb', index=53, number=53, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Hang', index=54, number=54, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Hani', index=55, number=55, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Hano', index=56, number=56, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Hans', index=57, number=57, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Hant', index=58, number=58, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Hatr', index=59, number=59, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Hebr', index=60, number=60, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Hira', index=61, number=61, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Hluw', index=62, number=62, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Hmng', index=63, number=63, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Hmnp', index=64, number=64, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Hrkt', index=65, number=65, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Hung', index=66, number=66, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Inds', index=67, number=67, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Ital', index=68, number=68, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Jamo', index=69, number=69, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Java', index=70, number=70, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Jpan', index=71, number=71, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Jurc', index=72, number=72, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Kali', index=73, number=73, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Kana', index=74, number=74, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Khar', index=75, number=75, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Khmr', index=76, number=76, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Khoj', index=77, number=77, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Kitl', index=78, number=78, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Kits', index=79, number=79, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Knda', index=80, number=80, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Kore', index=81, number=81, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Kpel', index=82, number=82, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Kthi', index=83, number=83, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Lana', index=84, number=84, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Laoo', index=85, number=85, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Latf', index=86, number=86, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Latg', index=87, number=87, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Latn', index=88, number=88, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Leke', index=89, number=89, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Lepc', index=90, number=90, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Limb', index=91, number=91, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Lina', index=92, number=92, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Linb', index=93, number=93, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Lisu', index=94, number=94, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Loma', index=95, number=95, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Lyci', index=96, number=96, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Lydi', index=97, number=97, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Mahj', index=98, number=98, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Maka', index=99, number=99, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Mand', index=100, number=100, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Mani', index=101, number=101, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Marc', index=102, number=102, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Maya', index=103, number=103, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Medf', index=104, number=104, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Mend', index=105, number=105, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Merc', index=106, number=106, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Mero', index=107, number=107, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Mlym', index=108, number=108, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Modi', index=109, number=109, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Mong', index=110, number=110, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Moon', index=111, number=111, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Mroo', index=112, number=112, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Mtei', index=113, number=113, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Mult', index=114, number=114, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Mymr', index=115, number=115, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Nand', index=116, number=116, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Narb', index=117, number=117, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Nbat', index=118, number=118, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Newa', index=119, number=119, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Nkdb', index=120, number=120, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Nkgb', index=121, number=121, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Nkoo', index=122, number=122, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Nshu', index=123, number=123, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Ogam', index=124, number=124, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Olck', index=125, number=125, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Orkh', index=126, number=126, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Orya', index=127, number=127, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Osge', index=128, number=128, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Osma', index=129, number=129, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Palm', index=130, number=130, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Pauc', index=131, number=131, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Perm', index=132, number=132, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Phag', index=133, number=133, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Phli', index=134, number=134, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Phlp', index=135, number=135, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Phlv', index=136, number=136, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Phnx', index=137, number=137, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Plrd', index=138, number=138, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Piqd', index=139, number=139, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Prti', index=140, number=140, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Qaaa', index=141, number=141, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Qabx', index=142, number=142, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Rjng', index=143, number=143, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Rohg', index=144, number=144, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Roro', index=145, number=145, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Runr', index=146, number=146, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Samr', index=147, number=147, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Sara', index=148, number=148, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Sarb', index=149, number=149, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Saur', index=150, number=150, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Sgnw', index=151, number=151, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Shaw', index=152, number=152, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Shrd', index=153, number=153, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Shui', index=154, number=154, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Sidd', index=155, number=155, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Sind', index=156, number=156, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Sinh', index=157, number=157, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Sogd', index=158, number=158, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Sogo', index=159, number=159, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Sora', index=160, number=160, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Soyo', index=161, number=161, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Sund', index=162, number=162, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Sylo', index=163, number=163, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Syrc', index=164, number=164, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Syre', index=165, number=165, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Syrj', index=166, number=166, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Syrn', index=167, number=167, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Tagb', index=168, number=168, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Takr', index=169, number=169, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Tale', index=170, number=170, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Talu', index=171, number=171, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Taml', index=172, number=172, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Tang', index=173, number=173, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Tavt', index=174, number=174, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Telu', index=175, number=175, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Teng', index=176, number=176, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Tfng', index=177, number=177, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Tglg', index=178, number=178, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Thaa', index=179, number=179, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Thai', index=180, number=180, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Tibt', index=181, number=181, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Tirh', index=182, number=182, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Ugar', index=183, number=183, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Vaii', index=184, number=184, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Visp', index=185, number=185, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Wara', index=186, number=186, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Wcho', index=187, number=187, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Wole', index=188, number=188, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Xpeo', index=189, number=189, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Xsux', index=190, number=190, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Yiii', index=191, number=191, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Zanb', index=192, number=192, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Zinh', index=193, number=193, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Zmth', index=194, number=194, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Zsye', index=195, number=195, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Zsym', index=196, number=196, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Zxxx', index=197, number=197, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Zyyy', index=198, number=198, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='Zzzz', index=199, number=199, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=3112, - serialized_end=5202, -) -_sym_db.RegisterEnumDescriptor(_LANGUAGE_SCRIPT) - -_LOCATION_COUNTRY = _descriptor.EnumDescriptor( - name='Country', - full_name='pb.Location.Country', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='UNKNOWN_COUNTRY', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='AF', index=1, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='AX', index=2, number=2, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='AL', index=3, number=3, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='DZ', index=4, number=4, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='AS', index=5, number=5, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='AD', index=6, number=6, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='AO', index=7, number=7, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='AI', index=8, number=8, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='AQ', index=9, number=9, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='AG', index=10, number=10, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='AR', index=11, number=11, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='AM', index=12, number=12, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='AW', index=13, number=13, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='AU', index=14, number=14, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='AT', index=15, number=15, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='AZ', index=16, number=16, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BS', index=17, number=17, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BH', index=18, number=18, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BD', index=19, number=19, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BB', index=20, number=20, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BY', index=21, number=21, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BE', index=22, number=22, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BZ', index=23, number=23, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BJ', index=24, number=24, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BM', index=25, number=25, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BT', index=26, number=26, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BO', index=27, number=27, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BQ', index=28, number=28, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BA', index=29, number=29, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BW', index=30, number=30, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BV', index=31, number=31, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BR', index=32, number=32, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='IO', index=33, number=33, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BN', index=34, number=34, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BG', index=35, number=35, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BF', index=36, number=36, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BI', index=37, number=37, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='KH', index=38, number=38, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CM', index=39, number=39, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CA', index=40, number=40, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CV', index=41, number=41, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='KY', index=42, number=42, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CF', index=43, number=43, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TD', index=44, number=44, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CL', index=45, number=45, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CN', index=46, number=46, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CX', index=47, number=47, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CC', index=48, number=48, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CO', index=49, number=49, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='KM', index=50, number=50, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CG', index=51, number=51, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CD', index=52, number=52, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CK', index=53, number=53, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CR', index=54, number=54, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CI', index=55, number=55, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='HR', index=56, number=56, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CU', index=57, number=57, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CW', index=58, number=58, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CY', index=59, number=59, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CZ', index=60, number=60, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='DK', index=61, number=61, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='DJ', index=62, number=62, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='DM', index=63, number=63, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='DO', index=64, number=64, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='EC', index=65, number=65, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='EG', index=66, number=66, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SV', index=67, number=67, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='GQ', index=68, number=68, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ER', index=69, number=69, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='EE', index=70, number=70, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ET', index=71, number=71, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='FK', index=72, number=72, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='FO', index=73, number=73, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='FJ', index=74, number=74, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='FI', index=75, number=75, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='FR', index=76, number=76, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='GF', index=77, number=77, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='PF', index=78, number=78, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TF', index=79, number=79, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='GA', index=80, number=80, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='GM', index=81, number=81, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='GE', index=82, number=82, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='DE', index=83, number=83, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='GH', index=84, number=84, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='GI', index=85, number=85, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='GR', index=86, number=86, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='GL', index=87, number=87, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='GD', index=88, number=88, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='GP', index=89, number=89, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='GU', index=90, number=90, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='GT', index=91, number=91, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='GG', index=92, number=92, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='GN', index=93, number=93, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='GW', index=94, number=94, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='GY', index=95, number=95, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='HT', index=96, number=96, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='HM', index=97, number=97, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='VA', index=98, number=98, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='HN', index=99, number=99, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='HK', index=100, number=100, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='HU', index=101, number=101, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='IS', index=102, number=102, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='IN', index=103, number=103, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ID', index=104, number=104, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='IR', index=105, number=105, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='IQ', index=106, number=106, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='IE', index=107, number=107, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='IM', index=108, number=108, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='IL', index=109, number=109, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='IT', index=110, number=110, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='JM', index=111, number=111, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='JP', index=112, number=112, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='JE', index=113, number=113, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='JO', index=114, number=114, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='KZ', index=115, number=115, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='KE', index=116, number=116, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='KI', index=117, number=117, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='KP', index=118, number=118, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='KR', index=119, number=119, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='KW', index=120, number=120, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='KG', index=121, number=121, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='LA', index=122, number=122, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='LV', index=123, number=123, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='LB', index=124, number=124, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='LS', index=125, number=125, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='LR', index=126, number=126, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='LY', index=127, number=127, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='LI', index=128, number=128, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='LT', index=129, number=129, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='LU', index=130, number=130, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MO', index=131, number=131, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MK', index=132, number=132, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MG', index=133, number=133, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MW', index=134, number=134, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MY', index=135, number=135, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MV', index=136, number=136, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ML', index=137, number=137, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MT', index=138, number=138, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MH', index=139, number=139, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MQ', index=140, number=140, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MR', index=141, number=141, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MU', index=142, number=142, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='YT', index=143, number=143, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MX', index=144, number=144, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='FM', index=145, number=145, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MD', index=146, number=146, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MC', index=147, number=147, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MN', index=148, number=148, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ME', index=149, number=149, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MS', index=150, number=150, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MA', index=151, number=151, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MZ', index=152, number=152, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MM', index=153, number=153, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='NA', index=154, number=154, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='NR', index=155, number=155, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='NP', index=156, number=156, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='NL', index=157, number=157, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='NC', index=158, number=158, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='NZ', index=159, number=159, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='NI', index=160, number=160, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='NE', index=161, number=161, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='NG', index=162, number=162, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='NU', index=163, number=163, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='NF', index=164, number=164, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MP', index=165, number=165, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='NO', index=166, number=166, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='OM', index=167, number=167, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='PK', index=168, number=168, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='PW', index=169, number=169, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='PS', index=170, number=170, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='PA', index=171, number=171, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='PG', index=172, number=172, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='PY', index=173, number=173, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='PE', index=174, number=174, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='PH', index=175, number=175, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='PN', index=176, number=176, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='PL', index=177, number=177, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='PT', index=178, number=178, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='PR', index=179, number=179, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='QA', index=180, number=180, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='RE', index=181, number=181, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='RO', index=182, number=182, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='RU', index=183, number=183, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='RW', index=184, number=184, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='BL', index=185, number=185, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SH', index=186, number=186, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='KN', index=187, number=187, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='LC', index=188, number=188, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='MF', index=189, number=189, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='PM', index=190, number=190, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='VC', index=191, number=191, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='WS', index=192, number=192, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SM', index=193, number=193, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ST', index=194, number=194, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SA', index=195, number=195, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SN', index=196, number=196, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='RS', index=197, number=197, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SC', index=198, number=198, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SL', index=199, number=199, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SG', index=200, number=200, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SX', index=201, number=201, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SK', index=202, number=202, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SI', index=203, number=203, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SB', index=204, number=204, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SO', index=205, number=205, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ZA', index=206, number=206, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='GS', index=207, number=207, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SS', index=208, number=208, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ES', index=209, number=209, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='LK', index=210, number=210, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SD', index=211, number=211, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SR', index=212, number=212, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SJ', index=213, number=213, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SZ', index=214, number=214, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SE', index=215, number=215, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CH', index=216, number=216, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='SY', index=217, number=217, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TW', index=218, number=218, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TJ', index=219, number=219, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TZ', index=220, number=220, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TH', index=221, number=221, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TL', index=222, number=222, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TG', index=223, number=223, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TK', index=224, number=224, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TO', index=225, number=225, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TT', index=226, number=226, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TN', index=227, number=227, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TR', index=228, number=228, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TM', index=229, number=229, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TC', index=230, number=230, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TV', index=231, number=231, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='UG', index=232, number=232, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='UA', index=233, number=233, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='AE', index=234, number=234, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='GB', index=235, number=235, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='US', index=236, number=236, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='UM', index=237, number=237, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='UY', index=238, number=238, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='UZ', index=239, number=239, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='VU', index=240, number=240, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='VE', index=241, number=241, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='VN', index=242, number=242, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='VG', index=243, number=243, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='VI', index=244, number=244, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='WF', index=245, number=245, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='EH', index=246, number=246, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='YE', index=247, number=247, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ZM', index=248, number=248, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='ZW', index=249, number=249, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R001', index=250, number=250, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R002', index=251, number=251, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R015', index=252, number=252, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R012', index=253, number=253, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R818', index=254, number=254, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R434', index=255, number=255, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R504', index=256, number=256, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R729', index=257, number=257, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R788', index=258, number=258, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R732', index=259, number=259, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R202', index=260, number=260, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R014', index=261, number=261, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R086', index=262, number=262, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R108', index=263, number=263, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R174', index=264, number=264, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R262', index=265, number=265, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R232', index=266, number=266, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R231', index=267, number=267, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R260', index=268, number=268, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R404', index=269, number=269, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R450', index=270, number=270, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R454', index=271, number=271, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R480', index=272, number=272, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R175', index=273, number=273, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R508', index=274, number=274, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R638', index=275, number=275, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R646', index=276, number=276, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R690', index=277, number=277, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R706', index=278, number=278, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R728', index=279, number=279, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R800', index=280, number=280, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R834', index=281, number=281, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R894', index=282, number=282, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R716', index=283, number=283, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R017', index=284, number=284, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R024', index=285, number=285, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R120', index=286, number=286, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R140', index=287, number=287, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R148', index=288, number=288, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R178', index=289, number=289, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R180', index=290, number=290, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R226', index=291, number=291, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R266', index=292, number=292, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R678', index=293, number=293, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R018', index=294, number=294, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R072', index=295, number=295, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R748', index=296, number=296, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R426', index=297, number=297, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R516', index=298, number=298, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R710', index=299, number=299, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R011', index=300, number=300, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R204', index=301, number=301, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R854', index=302, number=302, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R132', index=303, number=303, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R384', index=304, number=304, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R270', index=305, number=305, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R288', index=306, number=306, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R324', index=307, number=307, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R624', index=308, number=308, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R430', index=309, number=309, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R466', index=310, number=310, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R478', index=311, number=311, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R562', index=312, number=312, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R566', index=313, number=313, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R654', index=314, number=314, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R686', index=315, number=315, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R694', index=316, number=316, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R768', index=317, number=317, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R019', index=318, number=318, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R419', index=319, number=319, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R029', index=320, number=320, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R660', index=321, number=321, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R028', index=322, number=322, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R533', index=323, number=323, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R044', index=324, number=324, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R052', index=325, number=325, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R535', index=326, number=326, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R092', index=327, number=327, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R136', index=328, number=328, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R192', index=329, number=329, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R531', index=330, number=330, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R212', index=331, number=331, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R214', index=332, number=332, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R308', index=333, number=333, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R312', index=334, number=334, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R332', index=335, number=335, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R388', index=336, number=336, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R474', index=337, number=337, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R500', index=338, number=338, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R630', index=339, number=339, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R652', index=340, number=340, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R659', index=341, number=341, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R662', index=342, number=342, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R663', index=343, number=343, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R670', index=344, number=344, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R534', index=345, number=345, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R780', index=346, number=346, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R796', index=347, number=347, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R850', index=348, number=348, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R013', index=349, number=349, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R084', index=350, number=350, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R188', index=351, number=351, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R222', index=352, number=352, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R320', index=353, number=353, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R340', index=354, number=354, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R484', index=355, number=355, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R558', index=356, number=356, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R591', index=357, number=357, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R005', index=358, number=358, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R032', index=359, number=359, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R068', index=360, number=360, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R074', index=361, number=361, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R076', index=362, number=362, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R152', index=363, number=363, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R170', index=364, number=364, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R218', index=365, number=365, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R238', index=366, number=366, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R254', index=367, number=367, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R328', index=368, number=368, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R600', index=369, number=369, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R604', index=370, number=370, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R239', index=371, number=371, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R740', index=372, number=372, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R858', index=373, number=373, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R862', index=374, number=374, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R021', index=375, number=375, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R060', index=376, number=376, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R124', index=377, number=377, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R304', index=378, number=378, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R666', index=379, number=379, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R840', index=380, number=380, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R010', index=381, number=381, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R142', index=382, number=382, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R143', index=383, number=383, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R398', index=384, number=384, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R417', index=385, number=385, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R762', index=386, number=386, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R795', index=387, number=387, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R860', index=388, number=388, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R030', index=389, number=389, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R156', index=390, number=390, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R344', index=391, number=391, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R446', index=392, number=392, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R408', index=393, number=393, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R392', index=394, number=394, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R496', index=395, number=395, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R410', index=396, number=396, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R035', index=397, number=397, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R096', index=398, number=398, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R116', index=399, number=399, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R360', index=400, number=400, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R418', index=401, number=401, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R458', index=402, number=402, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R104', index=403, number=403, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R608', index=404, number=404, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R702', index=405, number=405, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R764', index=406, number=406, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R626', index=407, number=407, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R704', index=408, number=408, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R034', index=409, number=409, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R004', index=410, number=410, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R050', index=411, number=411, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R064', index=412, number=412, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R356', index=413, number=413, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R364', index=414, number=414, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R462', index=415, number=415, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R524', index=416, number=416, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R586', index=417, number=417, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R144', index=418, number=418, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R145', index=419, number=419, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R051', index=420, number=420, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R031', index=421, number=421, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R048', index=422, number=422, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R196', index=423, number=423, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R268', index=424, number=424, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R368', index=425, number=425, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R376', index=426, number=426, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R400', index=427, number=427, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R414', index=428, number=428, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R422', index=429, number=429, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R512', index=430, number=430, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R634', index=431, number=431, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R682', index=432, number=432, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R275', index=433, number=433, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R760', index=434, number=434, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R792', index=435, number=435, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R784', index=436, number=436, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R887', index=437, number=437, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R150', index=438, number=438, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R151', index=439, number=439, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R112', index=440, number=440, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R100', index=441, number=441, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R203', index=442, number=442, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R348', index=443, number=443, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R616', index=444, number=444, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R498', index=445, number=445, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R642', index=446, number=446, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R643', index=447, number=447, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R703', index=448, number=448, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R804', index=449, number=449, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R154', index=450, number=450, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R248', index=451, number=451, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R830', index=452, number=452, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R831', index=453, number=453, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R832', index=454, number=454, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R680', index=455, number=455, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R208', index=456, number=456, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R233', index=457, number=457, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R234', index=458, number=458, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R246', index=459, number=459, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R352', index=460, number=460, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R372', index=461, number=461, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R833', index=462, number=462, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R428', index=463, number=463, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R440', index=464, number=464, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R578', index=465, number=465, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R744', index=466, number=466, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R752', index=467, number=467, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R826', index=468, number=468, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R039', index=469, number=469, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R008', index=470, number=470, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R020', index=471, number=471, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R070', index=472, number=472, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R191', index=473, number=473, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R292', index=474, number=474, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R300', index=475, number=475, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R336', index=476, number=476, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R380', index=477, number=477, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R470', index=478, number=478, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R499', index=479, number=479, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R807', index=480, number=480, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R620', index=481, number=481, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R674', index=482, number=482, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R688', index=483, number=483, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R705', index=484, number=484, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R724', index=485, number=485, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R155', index=486, number=486, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R040', index=487, number=487, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R056', index=488, number=488, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R250', index=489, number=489, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R276', index=490, number=490, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R438', index=491, number=491, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R442', index=492, number=492, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R492', index=493, number=493, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R528', index=494, number=494, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R756', index=495, number=495, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R009', index=496, number=496, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R053', index=497, number=497, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R036', index=498, number=498, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R162', index=499, number=499, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R166', index=500, number=500, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R334', index=501, number=501, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R554', index=502, number=502, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R574', index=503, number=503, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R054', index=504, number=504, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R242', index=505, number=505, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R540', index=506, number=506, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R598', index=507, number=507, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R090', index=508, number=508, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R548', index=509, number=509, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R057', index=510, number=510, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R316', index=511, number=511, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R296', index=512, number=512, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R584', index=513, number=513, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R583', index=514, number=514, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R520', index=515, number=515, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R580', index=516, number=516, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R585', index=517, number=517, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R581', index=518, number=518, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R061', index=519, number=519, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R016', index=520, number=520, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R184', index=521, number=521, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R258', index=522, number=522, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R570', index=523, number=523, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R612', index=524, number=524, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R882', index=525, number=525, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R772', index=526, number=526, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R776', index=527, number=527, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R798', index=528, number=528, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='R876', index=529, number=529, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=5337, - serialized_end=10561, -) -_sym_db.RegisterEnumDescriptor(_LOCATION_COUNTRY) - - -_CLAIM = _descriptor.Descriptor( - name='Claim', - full_name='pb.Claim', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stream', full_name='pb.Claim.stream', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='channel', full_name='pb.Claim.channel', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='collection', full_name='pb.Claim.collection', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='repost', full_name='pb.Claim.repost', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='title', full_name='pb.Claim.title', index=4, - number=8, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='description', full_name='pb.Claim.description', index=5, - number=9, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='thumbnail', full_name='pb.Claim.thumbnail', index=6, - number=10, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='tags', full_name='pb.Claim.tags', index=7, - number=11, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='languages', full_name='pb.Claim.languages', index=8, - number=12, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='locations', full_name='pb.Claim.locations', index=9, - number=13, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='type', full_name='pb.Claim.type', - index=0, containing_type=None, fields=[]), - ], - serialized_start=20, - serialized_end=319, -) - - -_STREAM = _descriptor.Descriptor( - name='Stream', - full_name='pb.Stream', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='source', full_name='pb.Stream.source', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='author', full_name='pb.Stream.author', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='license', full_name='pb.Stream.license', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='license_url', full_name='pb.Stream.license_url', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='release_time', full_name='pb.Stream.release_time', index=4, - number=5, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='fee', full_name='pb.Stream.fee', index=5, - number=6, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='image', full_name='pb.Stream.image', index=6, - number=10, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='video', full_name='pb.Stream.video', index=7, - number=11, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='audio', full_name='pb.Stream.audio', index=8, - number=12, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='software', full_name='pb.Stream.software', index=9, - number=13, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='type', full_name='pb.Stream.type', - index=0, containing_type=None, fields=[]), - ], - serialized_start=322, - serialized_end=582, -) - - -_CHANNEL = _descriptor.Descriptor( - name='Channel', - full_name='pb.Channel', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='public_key', full_name='pb.Channel.public_key', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='email', full_name='pb.Channel.email', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='website_url', full_name='pb.Channel.website_url', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='cover', full_name='pb.Channel.cover', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='featured', full_name='pb.Channel.featured', index=4, - number=5, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=584, - serialized_end=709, -) - - -_CLAIMREFERENCE = _descriptor.Descriptor( - name='ClaimReference', - full_name='pb.ClaimReference', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='claim_hash', full_name='pb.ClaimReference.claim_hash', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=711, - serialized_end=747, -) - - -_CLAIMLIST = _descriptor.Descriptor( - name='ClaimList', - full_name='pb.ClaimList', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='list_type', full_name='pb.ClaimList.list_type', index=0, - number=1, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='claim_references', full_name='pb.ClaimList.claim_references', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _CLAIMLIST_LISTTYPE, - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=750, - serialized_end=894, -) - - -_SOURCE = _descriptor.Descriptor( - name='Source', - full_name='pb.Source', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='hash', full_name='pb.Source.hash', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='name', full_name='pb.Source.name', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='size', full_name='pb.Source.size', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='media_type', full_name='pb.Source.media_type', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='url', full_name='pb.Source.url', index=4, - number=5, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='sd_hash', full_name='pb.Source.sd_hash', index=5, - number=6, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='bt_infohash', full_name='pb.Source.bt_infohash', index=6, - number=7, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=896, - serialized_end=1017, -) - - -_FEE = _descriptor.Descriptor( - name='Fee', - full_name='pb.Fee', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='currency', full_name='pb.Fee.currency', index=0, - number=1, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='address', full_name='pb.Fee.address', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='amount', full_name='pb.Fee.amount', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _FEE_CURRENCY, - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1020, - serialized_end=1155, -) - - -_IMAGE = _descriptor.Descriptor( - name='Image', - full_name='pb.Image', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='width', full_name='pb.Image.width', index=0, - number=1, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='height', full_name='pb.Image.height', index=1, - number=2, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1157, - serialized_end=1195, -) - - -_VIDEO = _descriptor.Descriptor( - name='Video', - full_name='pb.Video', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='width', full_name='pb.Video.width', index=0, - number=1, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='height', full_name='pb.Video.height', index=1, - number=2, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='duration', full_name='pb.Video.duration', index=2, - number=3, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='audio', full_name='pb.Video.audio', index=3, - number=15, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1197, - serialized_end=1279, -) - - -_AUDIO = _descriptor.Descriptor( - name='Audio', - full_name='pb.Audio', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='duration', full_name='pb.Audio.duration', index=0, - number=1, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1281, - serialized_end=1306, -) - - -_SOFTWARE = _descriptor.Descriptor( - name='Software', - full_name='pb.Software', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='os', full_name='pb.Software.os', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _SOFTWARE_OS, - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1308, - serialized_end=1416, -) - - -_LANGUAGE = _descriptor.Descriptor( - name='Language', - full_name='pb.Language', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='language', full_name='pb.Language.language', index=0, - number=1, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='script', full_name='pb.Language.script', index=1, - number=2, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='region', full_name='pb.Language.region', index=2, - number=3, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _LANGUAGE_LANGUAGE, - _LANGUAGE_SCRIPT, - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1419, - serialized_end=5202, -) - - -_LOCATION = _descriptor.Descriptor( - name='Location', - full_name='pb.Location', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='country', full_name='pb.Location.country', index=0, - number=1, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='state', full_name='pb.Location.state', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='city', full_name='pb.Location.city', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='code', full_name='pb.Location.code', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='latitude', full_name='pb.Location.latitude', index=4, - number=5, type=17, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='longitude', full_name='pb.Location.longitude', index=5, - number=6, type=17, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _LOCATION_COUNTRY, - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=5205, - serialized_end=10561, -) - -_CLAIM.fields_by_name['stream'].message_type = _STREAM -_CLAIM.fields_by_name['channel'].message_type = _CHANNEL -_CLAIM.fields_by_name['collection'].message_type = _CLAIMLIST -_CLAIM.fields_by_name['repost'].message_type = _CLAIMREFERENCE -_CLAIM.fields_by_name['thumbnail'].message_type = _SOURCE -_CLAIM.fields_by_name['languages'].message_type = _LANGUAGE -_CLAIM.fields_by_name['locations'].message_type = _LOCATION -_CLAIM.oneofs_by_name['type'].fields.append( - _CLAIM.fields_by_name['stream']) -_CLAIM.fields_by_name['stream'].containing_oneof = _CLAIM.oneofs_by_name['type'] -_CLAIM.oneofs_by_name['type'].fields.append( - _CLAIM.fields_by_name['channel']) -_CLAIM.fields_by_name['channel'].containing_oneof = _CLAIM.oneofs_by_name['type'] -_CLAIM.oneofs_by_name['type'].fields.append( - _CLAIM.fields_by_name['collection']) -_CLAIM.fields_by_name['collection'].containing_oneof = _CLAIM.oneofs_by_name['type'] -_CLAIM.oneofs_by_name['type'].fields.append( - _CLAIM.fields_by_name['repost']) -_CLAIM.fields_by_name['repost'].containing_oneof = _CLAIM.oneofs_by_name['type'] -_STREAM.fields_by_name['source'].message_type = _SOURCE -_STREAM.fields_by_name['fee'].message_type = _FEE -_STREAM.fields_by_name['image'].message_type = _IMAGE -_STREAM.fields_by_name['video'].message_type = _VIDEO -_STREAM.fields_by_name['audio'].message_type = _AUDIO -_STREAM.fields_by_name['software'].message_type = _SOFTWARE -_STREAM.oneofs_by_name['type'].fields.append( - _STREAM.fields_by_name['image']) -_STREAM.fields_by_name['image'].containing_oneof = _STREAM.oneofs_by_name['type'] -_STREAM.oneofs_by_name['type'].fields.append( - _STREAM.fields_by_name['video']) -_STREAM.fields_by_name['video'].containing_oneof = _STREAM.oneofs_by_name['type'] -_STREAM.oneofs_by_name['type'].fields.append( - _STREAM.fields_by_name['audio']) -_STREAM.fields_by_name['audio'].containing_oneof = _STREAM.oneofs_by_name['type'] -_STREAM.oneofs_by_name['type'].fields.append( - _STREAM.fields_by_name['software']) -_STREAM.fields_by_name['software'].containing_oneof = _STREAM.oneofs_by_name['type'] -_CHANNEL.fields_by_name['cover'].message_type = _SOURCE -_CHANNEL.fields_by_name['featured'].message_type = _CLAIMLIST -_CLAIMLIST.fields_by_name['list_type'].enum_type = _CLAIMLIST_LISTTYPE -_CLAIMLIST.fields_by_name['claim_references'].message_type = _CLAIMREFERENCE -_CLAIMLIST_LISTTYPE.containing_type = _CLAIMLIST -_FEE.fields_by_name['currency'].enum_type = _FEE_CURRENCY -_FEE_CURRENCY.containing_type = _FEE -_VIDEO.fields_by_name['audio'].message_type = _AUDIO -_SOFTWARE_OS.containing_type = _SOFTWARE -_LANGUAGE.fields_by_name['language'].enum_type = _LANGUAGE_LANGUAGE -_LANGUAGE.fields_by_name['script'].enum_type = _LANGUAGE_SCRIPT -_LANGUAGE.fields_by_name['region'].enum_type = _LOCATION_COUNTRY -_LANGUAGE_LANGUAGE.containing_type = _LANGUAGE -_LANGUAGE_SCRIPT.containing_type = _LANGUAGE -_LOCATION.fields_by_name['country'].enum_type = _LOCATION_COUNTRY -_LOCATION_COUNTRY.containing_type = _LOCATION -DESCRIPTOR.message_types_by_name['Claim'] = _CLAIM -DESCRIPTOR.message_types_by_name['Stream'] = _STREAM -DESCRIPTOR.message_types_by_name['Channel'] = _CHANNEL -DESCRIPTOR.message_types_by_name['ClaimReference'] = _CLAIMREFERENCE -DESCRIPTOR.message_types_by_name['ClaimList'] = _CLAIMLIST -DESCRIPTOR.message_types_by_name['Source'] = _SOURCE -DESCRIPTOR.message_types_by_name['Fee'] = _FEE -DESCRIPTOR.message_types_by_name['Image'] = _IMAGE -DESCRIPTOR.message_types_by_name['Video'] = _VIDEO -DESCRIPTOR.message_types_by_name['Audio'] = _AUDIO -DESCRIPTOR.message_types_by_name['Software'] = _SOFTWARE -DESCRIPTOR.message_types_by_name['Language'] = _LANGUAGE -DESCRIPTOR.message_types_by_name['Location'] = _LOCATION - -Claim = _reflection.GeneratedProtocolMessageType('Claim', (_message.Message,), dict( - DESCRIPTOR = _CLAIM, - __module__ = 'claim_pb2' - # @@protoc_insertion_point(class_scope:pb.Claim) - )) -_sym_db.RegisterMessage(Claim) - -Stream = _reflection.GeneratedProtocolMessageType('Stream', (_message.Message,), dict( - DESCRIPTOR = _STREAM, - __module__ = 'claim_pb2' - # @@protoc_insertion_point(class_scope:pb.Stream) - )) -_sym_db.RegisterMessage(Stream) - -Channel = _reflection.GeneratedProtocolMessageType('Channel', (_message.Message,), dict( - DESCRIPTOR = _CHANNEL, - __module__ = 'claim_pb2' - # @@protoc_insertion_point(class_scope:pb.Channel) - )) -_sym_db.RegisterMessage(Channel) - -ClaimReference = _reflection.GeneratedProtocolMessageType('ClaimReference', (_message.Message,), dict( - DESCRIPTOR = _CLAIMREFERENCE, - __module__ = 'claim_pb2' - # @@protoc_insertion_point(class_scope:pb.ClaimReference) - )) -_sym_db.RegisterMessage(ClaimReference) - -ClaimList = _reflection.GeneratedProtocolMessageType('ClaimList', (_message.Message,), dict( - DESCRIPTOR = _CLAIMLIST, - __module__ = 'claim_pb2' - # @@protoc_insertion_point(class_scope:pb.ClaimList) - )) -_sym_db.RegisterMessage(ClaimList) - -Source = _reflection.GeneratedProtocolMessageType('Source', (_message.Message,), dict( - DESCRIPTOR = _SOURCE, - __module__ = 'claim_pb2' - # @@protoc_insertion_point(class_scope:pb.Source) - )) -_sym_db.RegisterMessage(Source) - -Fee = _reflection.GeneratedProtocolMessageType('Fee', (_message.Message,), dict( - DESCRIPTOR = _FEE, - __module__ = 'claim_pb2' - # @@protoc_insertion_point(class_scope:pb.Fee) - )) -_sym_db.RegisterMessage(Fee) - -Image = _reflection.GeneratedProtocolMessageType('Image', (_message.Message,), dict( - DESCRIPTOR = _IMAGE, - __module__ = 'claim_pb2' - # @@protoc_insertion_point(class_scope:pb.Image) - )) -_sym_db.RegisterMessage(Image) - -Video = _reflection.GeneratedProtocolMessageType('Video', (_message.Message,), dict( - DESCRIPTOR = _VIDEO, - __module__ = 'claim_pb2' - # @@protoc_insertion_point(class_scope:pb.Video) - )) -_sym_db.RegisterMessage(Video) - -Audio = _reflection.GeneratedProtocolMessageType('Audio', (_message.Message,), dict( - DESCRIPTOR = _AUDIO, - __module__ = 'claim_pb2' - # @@protoc_insertion_point(class_scope:pb.Audio) - )) -_sym_db.RegisterMessage(Audio) - -Software = _reflection.GeneratedProtocolMessageType('Software', (_message.Message,), dict( - DESCRIPTOR = _SOFTWARE, - __module__ = 'claim_pb2' - # @@protoc_insertion_point(class_scope:pb.Software) - )) -_sym_db.RegisterMessage(Software) - -Language = _reflection.GeneratedProtocolMessageType('Language', (_message.Message,), dict( - DESCRIPTOR = _LANGUAGE, - __module__ = 'claim_pb2' - # @@protoc_insertion_point(class_scope:pb.Language) - )) -_sym_db.RegisterMessage(Language) - -Location = _reflection.GeneratedProtocolMessageType('Location', (_message.Message,), dict( - DESCRIPTOR = _LOCATION, - __module__ = 'claim_pb2' - # @@protoc_insertion_point(class_scope:pb.Location) - )) -_sym_db.RegisterMessage(Location) - - -# @@protoc_insertion_point(module_scope) diff --git a/lbry/schema/types/v2/hub_pb2.py b/lbry/schema/types/v2/hub_pb2.py deleted file mode 100644 index 1cc0f475e..000000000 --- a/lbry/schema/types/v2/hub_pb2.py +++ /dev/null @@ -1,960 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: hub.proto -"""Generated protocol buffer code.""" -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from . import result_pb2 as result__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='hub.proto', - package='pb', - syntax='proto3', - serialized_options=b'Z$github.com/lbryio/hub/protobuf/go/pb', - create_key=_descriptor._internal_create_key, - serialized_pb=b'\n\thub.proto\x12\x02pb\x1a\x0cresult.proto\"\x0e\n\x0c\x45mptyMessage\".\n\rServerMessage\x12\x0f\n\x07\x61\x64\x64ress\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\t\"N\n\x0cHelloMessage\x12\x0c\n\x04port\x18\x01 \x01(\t\x12\x0c\n\x04host\x18\x02 \x01(\t\x12\"\n\x07servers\x18\x03 \x03(\x0b\x32\x11.pb.ServerMessage\"0\n\x0fInvertibleField\x12\x0e\n\x06invert\x18\x01 \x01(\x08\x12\r\n\x05value\x18\x02 \x03(\t\"\x1c\n\x0bStringValue\x12\r\n\x05value\x18\x01 \x01(\t\"\x1a\n\tBoolValue\x12\r\n\x05value\x18\x01 \x01(\x08\"\x1c\n\x0bUInt32Value\x12\r\n\x05value\x18\x01 \x01(\r\"j\n\nRangeField\x12\x1d\n\x02op\x18\x01 \x01(\x0e\x32\x11.pb.RangeField.Op\x12\r\n\x05value\x18\x02 \x03(\x05\".\n\x02Op\x12\x06\n\x02\x45Q\x10\x00\x12\x07\n\x03LTE\x10\x01\x12\x07\n\x03GTE\x10\x02\x12\x06\n\x02LT\x10\x03\x12\x06\n\x02GT\x10\x04\"\x8e\x0c\n\rSearchRequest\x12%\n\x08\x63laim_id\x18\x01 \x01(\x0b\x32\x13.pb.InvertibleField\x12\'\n\nchannel_id\x18\x02 \x01(\x0b\x32\x13.pb.InvertibleField\x12\x0c\n\x04text\x18\x03 \x01(\t\x12\r\n\x05limit\x18\x04 \x01(\x05\x12\x10\n\x08order_by\x18\x05 \x03(\t\x12\x0e\n\x06offset\x18\x06 \x01(\r\x12\x16\n\x0eis_controlling\x18\x07 \x01(\x08\x12\x1d\n\x15last_take_over_height\x18\x08 \x01(\t\x12\x12\n\nclaim_name\x18\t \x01(\t\x12\x17\n\x0fnormalized_name\x18\n \x01(\t\x12#\n\x0btx_position\x18\x0b \x03(\x0b\x32\x0e.pb.RangeField\x12\x1e\n\x06\x61mount\x18\x0c \x03(\x0b\x32\x0e.pb.RangeField\x12!\n\ttimestamp\x18\r \x03(\x0b\x32\x0e.pb.RangeField\x12*\n\x12\x63reation_timestamp\x18\x0e \x03(\x0b\x32\x0e.pb.RangeField\x12\x1e\n\x06height\x18\x0f \x03(\x0b\x32\x0e.pb.RangeField\x12\'\n\x0f\x63reation_height\x18\x10 \x03(\x0b\x32\x0e.pb.RangeField\x12)\n\x11\x61\x63tivation_height\x18\x11 \x03(\x0b\x32\x0e.pb.RangeField\x12)\n\x11\x65xpiration_height\x18\x12 \x03(\x0b\x32\x0e.pb.RangeField\x12$\n\x0crelease_time\x18\x13 \x03(\x0b\x32\x0e.pb.RangeField\x12\x11\n\tshort_url\x18\x14 \x01(\t\x12\x15\n\rcanonical_url\x18\x15 \x01(\t\x12\r\n\x05title\x18\x16 \x01(\t\x12\x0e\n\x06\x61uthor\x18\x17 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x18 \x01(\t\x12\x12\n\nclaim_type\x18\x19 \x03(\t\x12$\n\x0crepost_count\x18\x1a \x03(\x0b\x32\x0e.pb.RangeField\x12\x13\n\x0bstream_type\x18\x1b \x03(\t\x12\x12\n\nmedia_type\x18\x1c \x03(\t\x12\"\n\nfee_amount\x18\x1d \x03(\x0b\x32\x0e.pb.RangeField\x12\x14\n\x0c\x66\x65\x65_currency\x18\x1e \x01(\t\x12 \n\x08\x64uration\x18\x1f \x03(\x0b\x32\x0e.pb.RangeField\x12\x19\n\x11reposted_claim_id\x18 \x01(\t\x12#\n\x0b\x63\x65nsor_type\x18! \x03(\x0b\x32\x0e.pb.RangeField\x12\x19\n\x11\x63laims_in_channel\x18\" \x01(\t\x12)\n\x12is_signature_valid\x18$ \x01(\x0b\x32\r.pb.BoolValue\x12(\n\x10\x65\x66\x66\x65\x63tive_amount\x18% \x03(\x0b\x32\x0e.pb.RangeField\x12&\n\x0esupport_amount\x18& \x03(\x0b\x32\x0e.pb.RangeField\x12&\n\x0etrending_score\x18\' \x03(\x0b\x32\x0e.pb.RangeField\x12\r\n\x05tx_id\x18+ \x01(\t\x12 \n\x07tx_nout\x18, \x01(\x0b\x32\x0f.pb.UInt32Value\x12\x11\n\tsignature\x18- \x01(\t\x12\x18\n\x10signature_digest\x18. \x01(\t\x12\x18\n\x10public_key_bytes\x18/ \x01(\t\x12\x15\n\rpublic_key_id\x18\x30 \x01(\t\x12\x10\n\x08\x61ny_tags\x18\x31 \x03(\t\x12\x10\n\x08\x61ll_tags\x18\x32 \x03(\t\x12\x10\n\x08not_tags\x18\x33 \x03(\t\x12\x1d\n\x15has_channel_signature\x18\x34 \x01(\x08\x12!\n\nhas_source\x18\x35 \x01(\x0b\x32\r.pb.BoolValue\x12 \n\x18limit_claims_per_channel\x18\x36 \x01(\x05\x12\x15\n\rany_languages\x18\x37 \x03(\t\x12\x15\n\rall_languages\x18\x38 \x03(\t\x12\x19\n\x11remove_duplicates\x18\x39 \x01(\x08\x12\x11\n\tno_totals\x18: \x01(\x08\x12\x0f\n\x07sd_hash\x18; \x01(\t2\x88\x03\n\x03Hub\x12*\n\x06Search\x12\x11.pb.SearchRequest\x1a\x0b.pb.Outputs\"\x00\x12+\n\x04Ping\x12\x10.pb.EmptyMessage\x1a\x0f.pb.StringValue\"\x00\x12-\n\x05Hello\x12\x10.pb.HelloMessage\x1a\x10.pb.HelloMessage\"\x00\x12/\n\x07\x41\x64\x64Peer\x12\x11.pb.ServerMessage\x1a\x0f.pb.StringValue\"\x00\x12\x35\n\rPeerSubscribe\x12\x11.pb.ServerMessage\x1a\x0f.pb.StringValue\"\x00\x12.\n\x07Version\x12\x10.pb.EmptyMessage\x1a\x0f.pb.StringValue\"\x00\x12/\n\x08\x46\x65\x61tures\x12\x10.pb.EmptyMessage\x1a\x0f.pb.StringValue\"\x00\x12\x30\n\tBroadcast\x12\x10.pb.EmptyMessage\x1a\x0f.pb.UInt32Value\"\x00\x42&Z$github.com/lbryio/hub/protobuf/go/pbb\x06proto3' - , - dependencies=[result__pb2.DESCRIPTOR,]) - - - -_RANGEFIELD_OP = _descriptor.EnumDescriptor( - name='Op', - full_name='pb.RangeField.Op', - filename=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - values=[ - _descriptor.EnumValueDescriptor( - name='EQ', index=0, number=0, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key), - _descriptor.EnumValueDescriptor( - name='LTE', index=1, number=1, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key), - _descriptor.EnumValueDescriptor( - name='GTE', index=2, number=2, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key), - _descriptor.EnumValueDescriptor( - name='LT', index=3, number=3, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key), - _descriptor.EnumValueDescriptor( - name='GT', index=4, number=4, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key), - ], - containing_type=None, - serialized_options=None, - serialized_start=373, - serialized_end=419, -) -_sym_db.RegisterEnumDescriptor(_RANGEFIELD_OP) - - -_EMPTYMESSAGE = _descriptor.Descriptor( - name='EmptyMessage', - full_name='pb.EmptyMessage', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=31, - serialized_end=45, -) - - -_SERVERMESSAGE = _descriptor.Descriptor( - name='ServerMessage', - full_name='pb.ServerMessage', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='address', full_name='pb.ServerMessage.address', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='port', full_name='pb.ServerMessage.port', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=47, - serialized_end=93, -) - - -_HELLOMESSAGE = _descriptor.Descriptor( - name='HelloMessage', - full_name='pb.HelloMessage', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='port', full_name='pb.HelloMessage.port', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='host', full_name='pb.HelloMessage.host', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='servers', full_name='pb.HelloMessage.servers', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=95, - serialized_end=173, -) - - -_INVERTIBLEFIELD = _descriptor.Descriptor( - name='InvertibleField', - full_name='pb.InvertibleField', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='invert', full_name='pb.InvertibleField.invert', index=0, - number=1, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='value', full_name='pb.InvertibleField.value', index=1, - number=2, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=175, - serialized_end=223, -) - - -_STRINGVALUE = _descriptor.Descriptor( - name='StringValue', - full_name='pb.StringValue', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='value', full_name='pb.StringValue.value', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=225, - serialized_end=253, -) - - -_BOOLVALUE = _descriptor.Descriptor( - name='BoolValue', - full_name='pb.BoolValue', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='value', full_name='pb.BoolValue.value', index=0, - number=1, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=255, - serialized_end=281, -) - - -_UINT32VALUE = _descriptor.Descriptor( - name='UInt32Value', - full_name='pb.UInt32Value', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='value', full_name='pb.UInt32Value.value', index=0, - number=1, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=283, - serialized_end=311, -) - - -_RANGEFIELD = _descriptor.Descriptor( - name='RangeField', - full_name='pb.RangeField', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='op', full_name='pb.RangeField.op', index=0, - number=1, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='value', full_name='pb.RangeField.value', index=1, - number=2, type=5, cpp_type=1, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _RANGEFIELD_OP, - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=313, - serialized_end=419, -) - - -_SEARCHREQUEST = _descriptor.Descriptor( - name='SearchRequest', - full_name='pb.SearchRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='claim_id', full_name='pb.SearchRequest.claim_id', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='channel_id', full_name='pb.SearchRequest.channel_id', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='text', full_name='pb.SearchRequest.text', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='limit', full_name='pb.SearchRequest.limit', index=3, - number=4, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='order_by', full_name='pb.SearchRequest.order_by', index=4, - number=5, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='offset', full_name='pb.SearchRequest.offset', index=5, - number=6, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='is_controlling', full_name='pb.SearchRequest.is_controlling', index=6, - number=7, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='last_take_over_height', full_name='pb.SearchRequest.last_take_over_height', index=7, - number=8, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='claim_name', full_name='pb.SearchRequest.claim_name', index=8, - number=9, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='normalized_name', full_name='pb.SearchRequest.normalized_name', index=9, - number=10, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='tx_position', full_name='pb.SearchRequest.tx_position', index=10, - number=11, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='amount', full_name='pb.SearchRequest.amount', index=11, - number=12, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='timestamp', full_name='pb.SearchRequest.timestamp', index=12, - number=13, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='creation_timestamp', full_name='pb.SearchRequest.creation_timestamp', index=13, - number=14, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='height', full_name='pb.SearchRequest.height', index=14, - number=15, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='creation_height', full_name='pb.SearchRequest.creation_height', index=15, - number=16, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='activation_height', full_name='pb.SearchRequest.activation_height', index=16, - number=17, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='expiration_height', full_name='pb.SearchRequest.expiration_height', index=17, - number=18, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='release_time', full_name='pb.SearchRequest.release_time', index=18, - number=19, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='short_url', full_name='pb.SearchRequest.short_url', index=19, - number=20, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='canonical_url', full_name='pb.SearchRequest.canonical_url', index=20, - number=21, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='title', full_name='pb.SearchRequest.title', index=21, - number=22, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='author', full_name='pb.SearchRequest.author', index=22, - number=23, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='description', full_name='pb.SearchRequest.description', index=23, - number=24, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='claim_type', full_name='pb.SearchRequest.claim_type', index=24, - number=25, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='repost_count', full_name='pb.SearchRequest.repost_count', index=25, - number=26, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='stream_type', full_name='pb.SearchRequest.stream_type', index=26, - number=27, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='media_type', full_name='pb.SearchRequest.media_type', index=27, - number=28, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='fee_amount', full_name='pb.SearchRequest.fee_amount', index=28, - number=29, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='fee_currency', full_name='pb.SearchRequest.fee_currency', index=29, - number=30, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='duration', full_name='pb.SearchRequest.duration', index=30, - number=31, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='reposted_claim_id', full_name='pb.SearchRequest.reposted_claim_id', index=31, - number=32, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='censor_type', full_name='pb.SearchRequest.censor_type', index=32, - number=33, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='claims_in_channel', full_name='pb.SearchRequest.claims_in_channel', index=33, - number=34, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='is_signature_valid', full_name='pb.SearchRequest.is_signature_valid', index=34, - number=36, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='effective_amount', full_name='pb.SearchRequest.effective_amount', index=35, - number=37, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='support_amount', full_name='pb.SearchRequest.support_amount', index=36, - number=38, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='trending_score', full_name='pb.SearchRequest.trending_score', index=37, - number=39, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='tx_id', full_name='pb.SearchRequest.tx_id', index=38, - number=43, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='tx_nout', full_name='pb.SearchRequest.tx_nout', index=39, - number=44, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='signature', full_name='pb.SearchRequest.signature', index=40, - number=45, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='signature_digest', full_name='pb.SearchRequest.signature_digest', index=41, - number=46, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='public_key_bytes', full_name='pb.SearchRequest.public_key_bytes', index=42, - number=47, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='public_key_id', full_name='pb.SearchRequest.public_key_id', index=43, - number=48, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='any_tags', full_name='pb.SearchRequest.any_tags', index=44, - number=49, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='all_tags', full_name='pb.SearchRequest.all_tags', index=45, - number=50, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='not_tags', full_name='pb.SearchRequest.not_tags', index=46, - number=51, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='has_channel_signature', full_name='pb.SearchRequest.has_channel_signature', index=47, - number=52, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='has_source', full_name='pb.SearchRequest.has_source', index=48, - number=53, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='limit_claims_per_channel', full_name='pb.SearchRequest.limit_claims_per_channel', index=49, - number=54, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='any_languages', full_name='pb.SearchRequest.any_languages', index=50, - number=55, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='all_languages', full_name='pb.SearchRequest.all_languages', index=51, - number=56, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='remove_duplicates', full_name='pb.SearchRequest.remove_duplicates', index=52, - number=57, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='no_totals', full_name='pb.SearchRequest.no_totals', index=53, - number=58, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='sd_hash', full_name='pb.SearchRequest.sd_hash', index=54, - number=59, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=422, - serialized_end=1972, -) - -_HELLOMESSAGE.fields_by_name['servers'].message_type = _SERVERMESSAGE -_RANGEFIELD.fields_by_name['op'].enum_type = _RANGEFIELD_OP -_RANGEFIELD_OP.containing_type = _RANGEFIELD -_SEARCHREQUEST.fields_by_name['claim_id'].message_type = _INVERTIBLEFIELD -_SEARCHREQUEST.fields_by_name['channel_id'].message_type = _INVERTIBLEFIELD -_SEARCHREQUEST.fields_by_name['tx_position'].message_type = _RANGEFIELD -_SEARCHREQUEST.fields_by_name['amount'].message_type = _RANGEFIELD -_SEARCHREQUEST.fields_by_name['timestamp'].message_type = _RANGEFIELD -_SEARCHREQUEST.fields_by_name['creation_timestamp'].message_type = _RANGEFIELD -_SEARCHREQUEST.fields_by_name['height'].message_type = _RANGEFIELD -_SEARCHREQUEST.fields_by_name['creation_height'].message_type = _RANGEFIELD -_SEARCHREQUEST.fields_by_name['activation_height'].message_type = _RANGEFIELD -_SEARCHREQUEST.fields_by_name['expiration_height'].message_type = _RANGEFIELD -_SEARCHREQUEST.fields_by_name['release_time'].message_type = _RANGEFIELD -_SEARCHREQUEST.fields_by_name['repost_count'].message_type = _RANGEFIELD -_SEARCHREQUEST.fields_by_name['fee_amount'].message_type = _RANGEFIELD -_SEARCHREQUEST.fields_by_name['duration'].message_type = _RANGEFIELD -_SEARCHREQUEST.fields_by_name['censor_type'].message_type = _RANGEFIELD -_SEARCHREQUEST.fields_by_name['is_signature_valid'].message_type = _BOOLVALUE -_SEARCHREQUEST.fields_by_name['effective_amount'].message_type = _RANGEFIELD -_SEARCHREQUEST.fields_by_name['support_amount'].message_type = _RANGEFIELD -_SEARCHREQUEST.fields_by_name['trending_score'].message_type = _RANGEFIELD -_SEARCHREQUEST.fields_by_name['tx_nout'].message_type = _UINT32VALUE -_SEARCHREQUEST.fields_by_name['has_source'].message_type = _BOOLVALUE -DESCRIPTOR.message_types_by_name['EmptyMessage'] = _EMPTYMESSAGE -DESCRIPTOR.message_types_by_name['ServerMessage'] = _SERVERMESSAGE -DESCRIPTOR.message_types_by_name['HelloMessage'] = _HELLOMESSAGE -DESCRIPTOR.message_types_by_name['InvertibleField'] = _INVERTIBLEFIELD -DESCRIPTOR.message_types_by_name['StringValue'] = _STRINGVALUE -DESCRIPTOR.message_types_by_name['BoolValue'] = _BOOLVALUE -DESCRIPTOR.message_types_by_name['UInt32Value'] = _UINT32VALUE -DESCRIPTOR.message_types_by_name['RangeField'] = _RANGEFIELD -DESCRIPTOR.message_types_by_name['SearchRequest'] = _SEARCHREQUEST -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -EmptyMessage = _reflection.GeneratedProtocolMessageType('EmptyMessage', (_message.Message,), { - 'DESCRIPTOR' : _EMPTYMESSAGE, - '__module__' : 'hub_pb2' - # @@protoc_insertion_point(class_scope:pb.EmptyMessage) - }) -_sym_db.RegisterMessage(EmptyMessage) - -ServerMessage = _reflection.GeneratedProtocolMessageType('ServerMessage', (_message.Message,), { - 'DESCRIPTOR' : _SERVERMESSAGE, - '__module__' : 'hub_pb2' - # @@protoc_insertion_point(class_scope:pb.ServerMessage) - }) -_sym_db.RegisterMessage(ServerMessage) - -HelloMessage = _reflection.GeneratedProtocolMessageType('HelloMessage', (_message.Message,), { - 'DESCRIPTOR' : _HELLOMESSAGE, - '__module__' : 'hub_pb2' - # @@protoc_insertion_point(class_scope:pb.HelloMessage) - }) -_sym_db.RegisterMessage(HelloMessage) - -InvertibleField = _reflection.GeneratedProtocolMessageType('InvertibleField', (_message.Message,), { - 'DESCRIPTOR' : _INVERTIBLEFIELD, - '__module__' : 'hub_pb2' - # @@protoc_insertion_point(class_scope:pb.InvertibleField) - }) -_sym_db.RegisterMessage(InvertibleField) - -StringValue = _reflection.GeneratedProtocolMessageType('StringValue', (_message.Message,), { - 'DESCRIPTOR' : _STRINGVALUE, - '__module__' : 'hub_pb2' - # @@protoc_insertion_point(class_scope:pb.StringValue) - }) -_sym_db.RegisterMessage(StringValue) - -BoolValue = _reflection.GeneratedProtocolMessageType('BoolValue', (_message.Message,), { - 'DESCRIPTOR' : _BOOLVALUE, - '__module__' : 'hub_pb2' - # @@protoc_insertion_point(class_scope:pb.BoolValue) - }) -_sym_db.RegisterMessage(BoolValue) - -UInt32Value = _reflection.GeneratedProtocolMessageType('UInt32Value', (_message.Message,), { - 'DESCRIPTOR' : _UINT32VALUE, - '__module__' : 'hub_pb2' - # @@protoc_insertion_point(class_scope:pb.UInt32Value) - }) -_sym_db.RegisterMessage(UInt32Value) - -RangeField = _reflection.GeneratedProtocolMessageType('RangeField', (_message.Message,), { - 'DESCRIPTOR' : _RANGEFIELD, - '__module__' : 'hub_pb2' - # @@protoc_insertion_point(class_scope:pb.RangeField) - }) -_sym_db.RegisterMessage(RangeField) - -SearchRequest = _reflection.GeneratedProtocolMessageType('SearchRequest', (_message.Message,), { - 'DESCRIPTOR' : _SEARCHREQUEST, - '__module__' : 'hub_pb2' - # @@protoc_insertion_point(class_scope:pb.SearchRequest) - }) -_sym_db.RegisterMessage(SearchRequest) - - -DESCRIPTOR._options = None - -_HUB = _descriptor.ServiceDescriptor( - name='Hub', - full_name='pb.Hub', - file=DESCRIPTOR, - index=0, - serialized_options=None, - create_key=_descriptor._internal_create_key, - serialized_start=1975, - serialized_end=2367, - methods=[ - _descriptor.MethodDescriptor( - name='Search', - full_name='pb.Hub.Search', - index=0, - containing_service=None, - input_type=_SEARCHREQUEST, - output_type=result__pb2._OUTPUTS, - serialized_options=None, - create_key=_descriptor._internal_create_key, - ), - _descriptor.MethodDescriptor( - name='Ping', - full_name='pb.Hub.Ping', - index=1, - containing_service=None, - input_type=_EMPTYMESSAGE, - output_type=_STRINGVALUE, - serialized_options=None, - create_key=_descriptor._internal_create_key, - ), - _descriptor.MethodDescriptor( - name='Hello', - full_name='pb.Hub.Hello', - index=2, - containing_service=None, - input_type=_HELLOMESSAGE, - output_type=_HELLOMESSAGE, - serialized_options=None, - create_key=_descriptor._internal_create_key, - ), - _descriptor.MethodDescriptor( - name='AddPeer', - full_name='pb.Hub.AddPeer', - index=3, - containing_service=None, - input_type=_SERVERMESSAGE, - output_type=_STRINGVALUE, - serialized_options=None, - create_key=_descriptor._internal_create_key, - ), - _descriptor.MethodDescriptor( - name='PeerSubscribe', - full_name='pb.Hub.PeerSubscribe', - index=4, - containing_service=None, - input_type=_SERVERMESSAGE, - output_type=_STRINGVALUE, - serialized_options=None, - create_key=_descriptor._internal_create_key, - ), - _descriptor.MethodDescriptor( - name='Version', - full_name='pb.Hub.Version', - index=5, - containing_service=None, - input_type=_EMPTYMESSAGE, - output_type=_STRINGVALUE, - serialized_options=None, - create_key=_descriptor._internal_create_key, - ), - _descriptor.MethodDescriptor( - name='Features', - full_name='pb.Hub.Features', - index=6, - containing_service=None, - input_type=_EMPTYMESSAGE, - output_type=_STRINGVALUE, - serialized_options=None, - create_key=_descriptor._internal_create_key, - ), - _descriptor.MethodDescriptor( - name='Broadcast', - full_name='pb.Hub.Broadcast', - index=7, - containing_service=None, - input_type=_EMPTYMESSAGE, - output_type=_UINT32VALUE, - serialized_options=None, - create_key=_descriptor._internal_create_key, - ), -]) -_sym_db.RegisterServiceDescriptor(_HUB) - -DESCRIPTOR.services_by_name['Hub'] = _HUB - -# @@protoc_insertion_point(module_scope) diff --git a/lbry/schema/types/v2/hub_pb2_grpc.py b/lbry/schema/types/v2/hub_pb2_grpc.py deleted file mode 100644 index 16fdafe46..000000000 --- a/lbry/schema/types/v2/hub_pb2_grpc.py +++ /dev/null @@ -1,298 +0,0 @@ -# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! -"""Client and server classes corresponding to protobuf-defined services.""" -import grpc - -from . import hub_pb2 as hub__pb2 -from . import result_pb2 as result__pb2 - - -class HubStub(object): - """Missing associated documentation comment in .proto file.""" - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.Search = channel.unary_unary( - '/pb.Hub/Search', - request_serializer=hub__pb2.SearchRequest.SerializeToString, - response_deserializer=result__pb2.Outputs.FromString, - ) - self.Ping = channel.unary_unary( - '/pb.Hub/Ping', - request_serializer=hub__pb2.EmptyMessage.SerializeToString, - response_deserializer=hub__pb2.StringValue.FromString, - ) - self.Hello = channel.unary_unary( - '/pb.Hub/Hello', - request_serializer=hub__pb2.HelloMessage.SerializeToString, - response_deserializer=hub__pb2.HelloMessage.FromString, - ) - self.AddPeer = channel.unary_unary( - '/pb.Hub/AddPeer', - request_serializer=hub__pb2.ServerMessage.SerializeToString, - response_deserializer=hub__pb2.StringValue.FromString, - ) - self.PeerSubscribe = channel.unary_unary( - '/pb.Hub/PeerSubscribe', - request_serializer=hub__pb2.ServerMessage.SerializeToString, - response_deserializer=hub__pb2.StringValue.FromString, - ) - self.Version = channel.unary_unary( - '/pb.Hub/Version', - request_serializer=hub__pb2.EmptyMessage.SerializeToString, - response_deserializer=hub__pb2.StringValue.FromString, - ) - self.Features = channel.unary_unary( - '/pb.Hub/Features', - request_serializer=hub__pb2.EmptyMessage.SerializeToString, - response_deserializer=hub__pb2.StringValue.FromString, - ) - self.Broadcast = channel.unary_unary( - '/pb.Hub/Broadcast', - request_serializer=hub__pb2.EmptyMessage.SerializeToString, - response_deserializer=hub__pb2.UInt32Value.FromString, - ) - - -class HubServicer(object): - """Missing associated documentation comment in .proto file.""" - - def Search(self, request, context): - """Missing associated documentation comment in .proto file.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Ping(self, request, context): - """Missing associated documentation comment in .proto file.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Hello(self, request, context): - """Missing associated documentation comment in .proto file.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def AddPeer(self, request, context): - """Missing associated documentation comment in .proto file.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def PeerSubscribe(self, request, context): - """Missing associated documentation comment in .proto file.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Version(self, request, context): - """Missing associated documentation comment in .proto file.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Features(self, request, context): - """Missing associated documentation comment in .proto file.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Broadcast(self, request, context): - """Missing associated documentation comment in .proto file.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - -def add_HubServicer_to_server(servicer, server): - rpc_method_handlers = { - 'Search': grpc.unary_unary_rpc_method_handler( - servicer.Search, - request_deserializer=hub__pb2.SearchRequest.FromString, - response_serializer=result__pb2.Outputs.SerializeToString, - ), - 'Ping': grpc.unary_unary_rpc_method_handler( - servicer.Ping, - request_deserializer=hub__pb2.EmptyMessage.FromString, - response_serializer=hub__pb2.StringValue.SerializeToString, - ), - 'Hello': grpc.unary_unary_rpc_method_handler( - servicer.Hello, - request_deserializer=hub__pb2.HelloMessage.FromString, - response_serializer=hub__pb2.HelloMessage.SerializeToString, - ), - 'AddPeer': grpc.unary_unary_rpc_method_handler( - servicer.AddPeer, - request_deserializer=hub__pb2.ServerMessage.FromString, - response_serializer=hub__pb2.StringValue.SerializeToString, - ), - 'PeerSubscribe': grpc.unary_unary_rpc_method_handler( - servicer.PeerSubscribe, - request_deserializer=hub__pb2.ServerMessage.FromString, - response_serializer=hub__pb2.StringValue.SerializeToString, - ), - 'Version': grpc.unary_unary_rpc_method_handler( - servicer.Version, - request_deserializer=hub__pb2.EmptyMessage.FromString, - response_serializer=hub__pb2.StringValue.SerializeToString, - ), - 'Features': grpc.unary_unary_rpc_method_handler( - servicer.Features, - request_deserializer=hub__pb2.EmptyMessage.FromString, - response_serializer=hub__pb2.StringValue.SerializeToString, - ), - 'Broadcast': grpc.unary_unary_rpc_method_handler( - servicer.Broadcast, - request_deserializer=hub__pb2.EmptyMessage.FromString, - response_serializer=hub__pb2.UInt32Value.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'pb.Hub', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) - - - # This class is part of an EXPERIMENTAL API. -class Hub(object): - """Missing associated documentation comment in .proto file.""" - - @staticmethod - def Search(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/pb.Hub/Search', - hub__pb2.SearchRequest.SerializeToString, - result__pb2.Outputs.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def Ping(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/pb.Hub/Ping', - hub__pb2.EmptyMessage.SerializeToString, - hub__pb2.StringValue.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def Hello(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/pb.Hub/Hello', - hub__pb2.HelloMessage.SerializeToString, - hub__pb2.HelloMessage.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def AddPeer(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/pb.Hub/AddPeer', - hub__pb2.ServerMessage.SerializeToString, - hub__pb2.StringValue.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def PeerSubscribe(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/pb.Hub/PeerSubscribe', - hub__pb2.ServerMessage.SerializeToString, - hub__pb2.StringValue.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def Version(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/pb.Hub/Version', - hub__pb2.EmptyMessage.SerializeToString, - hub__pb2.StringValue.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def Features(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/pb.Hub/Features', - hub__pb2.EmptyMessage.SerializeToString, - hub__pb2.StringValue.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def Broadcast(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/pb.Hub/Broadcast', - hub__pb2.EmptyMessage.SerializeToString, - hub__pb2.UInt32Value.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/lbry/schema/types/v2/purchase_pb2.py b/lbry/schema/types/v2/purchase_pb2.py deleted file mode 100644 index a2f02e584..000000000 --- a/lbry/schema/types/v2/purchase_pb2.py +++ /dev/null @@ -1,69 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: purchase.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='purchase.proto', - package='pb', - syntax='proto3', - serialized_pb=_b('\n\x0epurchase.proto\x12\x02pb\"\x1e\n\x08Purchase\x12\x12\n\nclaim_hash\x18\x01 \x01(\x0c\x62\x06proto3') -) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_PURCHASE = _descriptor.Descriptor( - name='Purchase', - full_name='pb.Purchase', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='claim_hash', full_name='pb.Purchase.claim_hash', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=22, - serialized_end=52, -) - -DESCRIPTOR.message_types_by_name['Purchase'] = _PURCHASE - -Purchase = _reflection.GeneratedProtocolMessageType('Purchase', (_message.Message,), dict( - DESCRIPTOR = _PURCHASE, - __module__ = 'purchase_pb2' - # @@protoc_insertion_point(class_scope:pb.Purchase) - )) -_sym_db.RegisterMessage(Purchase) - - -# @@protoc_insertion_point(module_scope) diff --git a/lbry/schema/types/v2/result_pb2.py b/lbry/schema/types/v2/result_pb2.py deleted file mode 100644 index be36eefdb..000000000 --- a/lbry/schema/types/v2/result_pb2.py +++ /dev/null @@ -1,464 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: result.proto -"""Generated protocol buffer code.""" -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='result.proto', - package='pb', - syntax='proto3', - serialized_options=b'Z$github.com/lbryio/hub/protobuf/go/pb', - create_key=_descriptor._internal_create_key, - serialized_pb=b'\n\x0cresult.proto\x12\x02pb\"\x97\x01\n\x07Outputs\x12\x18\n\x04txos\x18\x01 \x03(\x0b\x32\n.pb.Output\x12\x1e\n\nextra_txos\x18\x02 \x03(\x0b\x32\n.pb.Output\x12\r\n\x05total\x18\x03 \x01(\r\x12\x0e\n\x06offset\x18\x04 \x01(\r\x12\x1c\n\x07\x62locked\x18\x05 \x03(\x0b\x32\x0b.pb.Blocked\x12\x15\n\rblocked_total\x18\x06 \x01(\r\"{\n\x06Output\x12\x0f\n\x07tx_hash\x18\x01 \x01(\x0c\x12\x0c\n\x04nout\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\x1e\n\x05\x63laim\x18\x07 \x01(\x0b\x32\r.pb.ClaimMetaH\x00\x12\x1a\n\x05\x65rror\x18\x0f \x01(\x0b\x32\t.pb.ErrorH\x00\x42\x06\n\x04meta\"\xe6\x02\n\tClaimMeta\x12\x1b\n\x07\x63hannel\x18\x01 \x01(\x0b\x32\n.pb.Output\x12\x1a\n\x06repost\x18\x02 \x01(\x0b\x32\n.pb.Output\x12\x11\n\tshort_url\x18\x03 \x01(\t\x12\x15\n\rcanonical_url\x18\x04 \x01(\t\x12\x16\n\x0eis_controlling\x18\x05 \x01(\x08\x12\x18\n\x10take_over_height\x18\x06 \x01(\r\x12\x17\n\x0f\x63reation_height\x18\x07 \x01(\r\x12\x19\n\x11\x61\x63tivation_height\x18\x08 \x01(\r\x12\x19\n\x11\x65xpiration_height\x18\t \x01(\r\x12\x19\n\x11\x63laims_in_channel\x18\n \x01(\r\x12\x10\n\x08reposted\x18\x0b \x01(\r\x12\x18\n\x10\x65\x66\x66\x65\x63tive_amount\x18\x14 \x01(\x04\x12\x16\n\x0esupport_amount\x18\x15 \x01(\x04\x12\x16\n\x0etrending_score\x18\x16 \x01(\x01\"\x94\x01\n\x05\x45rror\x12\x1c\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x0e.pb.Error.Code\x12\x0c\n\x04text\x18\x02 \x01(\t\x12\x1c\n\x07\x62locked\x18\x03 \x01(\x0b\x32\x0b.pb.Blocked\"A\n\x04\x43ode\x12\x10\n\x0cUNKNOWN_CODE\x10\x00\x12\r\n\tNOT_FOUND\x10\x01\x12\x0b\n\x07INVALID\x10\x02\x12\x0b\n\x07\x42LOCKED\x10\x03\"5\n\x07\x42locked\x12\r\n\x05\x63ount\x18\x01 \x01(\r\x12\x1b\n\x07\x63hannel\x18\x02 \x01(\x0b\x32\n.pb.OutputB&Z$github.com/lbryio/hub/protobuf/go/pbb\x06proto3' -) - - - -_ERROR_CODE = _descriptor.EnumDescriptor( - name='Code', - full_name='pb.Error.Code', - filename=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - values=[ - _descriptor.EnumValueDescriptor( - name='UNKNOWN_CODE', index=0, number=0, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key), - _descriptor.EnumValueDescriptor( - name='NOT_FOUND', index=1, number=1, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key), - _descriptor.EnumValueDescriptor( - name='INVALID', index=2, number=2, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key), - _descriptor.EnumValueDescriptor( - name='BLOCKED', index=3, number=3, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key), - ], - containing_type=None, - serialized_options=None, - serialized_start=744, - serialized_end=809, -) -_sym_db.RegisterEnumDescriptor(_ERROR_CODE) - - -_OUTPUTS = _descriptor.Descriptor( - name='Outputs', - full_name='pb.Outputs', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='txos', full_name='pb.Outputs.txos', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='extra_txos', full_name='pb.Outputs.extra_txos', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='total', full_name='pb.Outputs.total', index=2, - number=3, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='offset', full_name='pb.Outputs.offset', index=3, - number=4, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='blocked', full_name='pb.Outputs.blocked', index=4, - number=5, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='blocked_total', full_name='pb.Outputs.blocked_total', index=5, - number=6, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=21, - serialized_end=172, -) - - -_OUTPUT = _descriptor.Descriptor( - name='Output', - full_name='pb.Output', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='tx_hash', full_name='pb.Output.tx_hash', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='nout', full_name='pb.Output.nout', index=1, - number=2, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='height', full_name='pb.Output.height', index=2, - number=3, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='claim', full_name='pb.Output.claim', index=3, - number=7, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='error', full_name='pb.Output.error', index=4, - number=15, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='meta', full_name='pb.Output.meta', - index=0, containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[]), - ], - serialized_start=174, - serialized_end=297, -) - - -_CLAIMMETA = _descriptor.Descriptor( - name='ClaimMeta', - full_name='pb.ClaimMeta', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='channel', full_name='pb.ClaimMeta.channel', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='repost', full_name='pb.ClaimMeta.repost', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='short_url', full_name='pb.ClaimMeta.short_url', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='canonical_url', full_name='pb.ClaimMeta.canonical_url', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='is_controlling', full_name='pb.ClaimMeta.is_controlling', index=4, - number=5, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='take_over_height', full_name='pb.ClaimMeta.take_over_height', index=5, - number=6, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='creation_height', full_name='pb.ClaimMeta.creation_height', index=6, - number=7, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='activation_height', full_name='pb.ClaimMeta.activation_height', index=7, - number=8, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='expiration_height', full_name='pb.ClaimMeta.expiration_height', index=8, - number=9, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='claims_in_channel', full_name='pb.ClaimMeta.claims_in_channel', index=9, - number=10, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='reposted', full_name='pb.ClaimMeta.reposted', index=10, - number=11, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='effective_amount', full_name='pb.ClaimMeta.effective_amount', index=11, - number=20, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='support_amount', full_name='pb.ClaimMeta.support_amount', index=12, - number=21, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='trending_score', full_name='pb.ClaimMeta.trending_score', index=13, - number=22, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=300, - serialized_end=658, -) - - -_ERROR = _descriptor.Descriptor( - name='Error', - full_name='pb.Error', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='code', full_name='pb.Error.code', index=0, - number=1, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='text', full_name='pb.Error.text', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='blocked', full_name='pb.Error.blocked', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _ERROR_CODE, - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=661, - serialized_end=809, -) - - -_BLOCKED = _descriptor.Descriptor( - name='Blocked', - full_name='pb.Blocked', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='count', full_name='pb.Blocked.count', index=0, - number=1, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='channel', full_name='pb.Blocked.channel', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=811, - serialized_end=864, -) - -_OUTPUTS.fields_by_name['txos'].message_type = _OUTPUT -_OUTPUTS.fields_by_name['extra_txos'].message_type = _OUTPUT -_OUTPUTS.fields_by_name['blocked'].message_type = _BLOCKED -_OUTPUT.fields_by_name['claim'].message_type = _CLAIMMETA -_OUTPUT.fields_by_name['error'].message_type = _ERROR -_OUTPUT.oneofs_by_name['meta'].fields.append( - _OUTPUT.fields_by_name['claim']) -_OUTPUT.fields_by_name['claim'].containing_oneof = _OUTPUT.oneofs_by_name['meta'] -_OUTPUT.oneofs_by_name['meta'].fields.append( - _OUTPUT.fields_by_name['error']) -_OUTPUT.fields_by_name['error'].containing_oneof = _OUTPUT.oneofs_by_name['meta'] -_CLAIMMETA.fields_by_name['channel'].message_type = _OUTPUT -_CLAIMMETA.fields_by_name['repost'].message_type = _OUTPUT -_ERROR.fields_by_name['code'].enum_type = _ERROR_CODE -_ERROR.fields_by_name['blocked'].message_type = _BLOCKED -_ERROR_CODE.containing_type = _ERROR -_BLOCKED.fields_by_name['channel'].message_type = _OUTPUT -DESCRIPTOR.message_types_by_name['Outputs'] = _OUTPUTS -DESCRIPTOR.message_types_by_name['Output'] = _OUTPUT -DESCRIPTOR.message_types_by_name['ClaimMeta'] = _CLAIMMETA -DESCRIPTOR.message_types_by_name['Error'] = _ERROR -DESCRIPTOR.message_types_by_name['Blocked'] = _BLOCKED -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -Outputs = _reflection.GeneratedProtocolMessageType('Outputs', (_message.Message,), { - 'DESCRIPTOR' : _OUTPUTS, - '__module__' : 'result_pb2' - # @@protoc_insertion_point(class_scope:pb.Outputs) - }) -_sym_db.RegisterMessage(Outputs) - -Output = _reflection.GeneratedProtocolMessageType('Output', (_message.Message,), { - 'DESCRIPTOR' : _OUTPUT, - '__module__' : 'result_pb2' - # @@protoc_insertion_point(class_scope:pb.Output) - }) -_sym_db.RegisterMessage(Output) - -ClaimMeta = _reflection.GeneratedProtocolMessageType('ClaimMeta', (_message.Message,), { - 'DESCRIPTOR' : _CLAIMMETA, - '__module__' : 'result_pb2' - # @@protoc_insertion_point(class_scope:pb.ClaimMeta) - }) -_sym_db.RegisterMessage(ClaimMeta) - -Error = _reflection.GeneratedProtocolMessageType('Error', (_message.Message,), { - 'DESCRIPTOR' : _ERROR, - '__module__' : 'result_pb2' - # @@protoc_insertion_point(class_scope:pb.Error) - }) -_sym_db.RegisterMessage(Error) - -Blocked = _reflection.GeneratedProtocolMessageType('Blocked', (_message.Message,), { - 'DESCRIPTOR' : _BLOCKED, - '__module__' : 'result_pb2' - # @@protoc_insertion_point(class_scope:pb.Blocked) - }) -_sym_db.RegisterMessage(Blocked) - - -DESCRIPTOR._options = None -# @@protoc_insertion_point(module_scope) diff --git a/lbry/schema/types/v2/result_pb2_grpc.py b/lbry/schema/types/v2/result_pb2_grpc.py deleted file mode 100644 index 2daafffeb..000000000 --- a/lbry/schema/types/v2/result_pb2_grpc.py +++ /dev/null @@ -1,4 +0,0 @@ -# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! -"""Client and server classes corresponding to protobuf-defined services.""" -import grpc - diff --git a/lbry/schema/types/v2/support_pb2.py b/lbry/schema/types/v2/support_pb2.py deleted file mode 100644 index 2e3fdfcb0..000000000 --- a/lbry/schema/types/v2/support_pb2.py +++ /dev/null @@ -1,76 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: support.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='support.proto', - package='pb', - syntax='proto3', - serialized_pb=_b('\n\rsupport.proto\x12\x02pb\")\n\x07Support\x12\r\n\x05\x65moji\x18\x01 \x01(\t\x12\x0f\n\x07\x63omment\x18\x02 \x01(\tb\x06proto3') -) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_SUPPORT = _descriptor.Descriptor( - name='Support', - full_name='pb.Support', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='emoji', full_name='pb.Support.emoji', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='comment', full_name='pb.Support.comment', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=21, - serialized_end=62, -) - -DESCRIPTOR.message_types_by_name['Support'] = _SUPPORT - -Support = _reflection.GeneratedProtocolMessageType('Support', (_message.Message,), dict( - DESCRIPTOR = _SUPPORT, - __module__ = 'support_pb2' - # @@protoc_insertion_point(class_scope:pb.Support) - )) -_sym_db.RegisterMessage(Support) - - -# @@protoc_insertion_point(module_scope) diff --git a/lbry/schema/url.py b/lbry/schema/url.py deleted file mode 100644 index a1081b199..000000000 --- a/lbry/schema/url.py +++ /dev/null @@ -1,130 +0,0 @@ -import re -import unicodedata -from typing import NamedTuple, Tuple - - -def _create_url_regex(): - # see https://spec.lbry.com/ and test_url.py - invalid_names_regex = \ - r"[^=&#:$@%?;\"/\\<>%{}|^~`\[\]" \ - r"\u0000-\u0020\uD800-\uDFFF\uFFFE-\uFFFF]+" - - def _named(name, regex): - return "(?P<" + name + ">" + regex + ")" - - def _group(regex): - return "(?:" + regex + ")" - - def _oneof(*choices): - return _group('|'.join(choices)) - - def _claim(name, prefix=""): - return _group( - _named(name+"_name", prefix + invalid_names_regex) + - _oneof( - _group('[:#]' + _named(name+"_claim_id", "[0-9a-f]{1,40}")), - _group(r'\$' + _named(name+"_amount_order", '[1-9][0-9]*')) - ) + '?' - ) - - return ( - '^' + - _named("scheme", "lbry://") + '?' + - _oneof( - _group(_claim("channel_with_stream", "@") + "/" + _claim("stream_in_channel")), - _claim("channel", "@"), - _claim("stream") - ) + - '$' - ) - - -URL_REGEX = _create_url_regex() - - -def normalize_name(name): - return unicodedata.normalize('NFD', name).casefold() - - -class PathSegment(NamedTuple): - name: str - claim_id: str = None - amount_order: int = None - - @property - def normalized(self): - return normalize_name(self.name) - - @property - def is_shortid(self): - return self.claim_id is not None and len(self.claim_id) < 40 - - @property - def is_fullid(self): - return self.claim_id is not None and len(self.claim_id) == 40 - - def to_dict(self): - q = {'name': self.name} - if self.claim_id is not None: - q['claim_id'] = self.claim_id - if self.amount_order is not None: - q['amount_order'] = self.amount_order - return q - - def __str__(self): - if self.claim_id is not None: - return f"{self.name}:{self.claim_id}" - elif self.amount_order is not None: - return f"{self.name}${self.amount_order}" - return self.name - - -class URL(NamedTuple): - stream: PathSegment - channel: PathSegment - - @property - def has_channel(self): - return self.channel is not None - - @property - def has_stream(self): - return self.stream is not None - - @property - def has_stream_in_channel(self): - return self.has_channel and self.has_stream - - @property - def parts(self) -> Tuple: - if self.has_stream_in_channel: - return self.channel, self.stream - if self.has_channel: - return self.channel, - return self.stream, - - def __str__(self): - return f"lbry://{'/'.join(str(p) for p in self.parts)}" - - @classmethod - def parse(cls, url): - match = re.match(URL_REGEX, url) - - if match is None: - raise ValueError('Invalid LBRY URL') - - segments = {} - parts = match.groupdict() - for segment in ('channel', 'stream', 'channel_with_stream', 'stream_in_channel'): - if parts[f'{segment}_name'] is not None: - segments[segment] = PathSegment( - parts[f'{segment}_name'], - parts[f'{segment}_claim_id'], - parts[f'{segment}_amount_order'] - ) - - if 'channel_with_stream' in segments: - segments['channel'] = segments['channel_with_stream'] - segments['stream'] = segments['stream_in_channel'] - - return cls(segments.get('stream', None), segments.get('channel', None)) diff --git a/lbry/stream/managed_stream.py b/lbry/stream/managed_stream.py index 2a85da66e..69e0b04fe 100644 --- a/lbry/stream/managed_stream.py +++ b/lbry/stream/managed_stream.py @@ -6,7 +6,7 @@ import logging from typing import Optional from aiohttp.web import Request, StreamResponse, HTTPRequestRangeNotSatisfiable from lbry.error import DownloadSDTimeoutError -from lbry.schema.mime_types import guess_media_type +from scribe.schema.mime_types import guess_media_type from lbry.stream.downloader import StreamDownloader from lbry.stream.descriptor import StreamDescriptor, sanitize_file_name from lbry.stream.reflector.client import StreamReflectorClient @@ -16,7 +16,7 @@ from lbry.file.source import ManagedDownloadSource if typing.TYPE_CHECKING: from lbry.conf import Config - from lbry.schema.claim import Claim + from scribe.schema.claim import Claim from lbry.blob.blob_manager import BlobManager from lbry.blob.blob_info import BlobInfo from lbry.dht.node import Node diff --git a/lbry/testcase.py b/lbry/testcase.py index 3977e6efc..5951b4044 100644 --- a/lbry/testcase.py +++ b/lbry/testcase.py @@ -20,7 +20,7 @@ from lbry.wallet.util import satoshis_to_coins from lbry.wallet.dewies import lbc_to_dewies from lbry.wallet.orchstr8 import Conductor from lbry.wallet.orchstr8.node import LBCWalletNode, WalletNode, HubNode -from lbry.schema.claim import Claim +from scribe.schema.claim import Claim from lbry.extras.daemon.daemon import Daemon, jsonrpc_dumps_pretty from lbry.extras.daemon.components import Component, WalletComponent diff --git a/lbry/utils.py b/lbry/utils.py index edcfbb221..975ec157d 100644 --- a/lbry/utils.py +++ b/lbry/utils.py @@ -21,7 +21,7 @@ import pkg_resources import certifi import aiohttp from prometheus_client import Counter -from lbry.schema.claim import Claim +from scribe.schema.claim import Claim log = logging.getLogger(__name__) diff --git a/lbry/wallet/account.py b/lbry/wallet/account.py index c939d7ee9..0b34f3687 100644 --- a/lbry/wallet/account.py +++ b/lbry/wallet/account.py @@ -12,7 +12,7 @@ from typing import Type, Dict, Tuple, Optional, Any, List from lbry.error import InvalidPasswordError from lbry.crypto.crypt import aes_encrypt, aes_decrypt -from .bip32 import PrivateKey, PublicKey, KeyPath, from_extended_key_string +from scribe.bip32 import PrivateKey, PublicKey, KeyPath, from_extended_key_string from .mnemonic import Mnemonic from .constants import COIN, TXO_TYPES from .transaction import Transaction, Input, Output diff --git a/lbry/wallet/bip32.py b/lbry/wallet/bip32.py deleted file mode 100644 index f9d96164e..000000000 --- a/lbry/wallet/bip32.py +++ /dev/null @@ -1,338 +0,0 @@ -from asn1crypto.keys import PrivateKeyInfo, ECPrivateKey -from coincurve import PublicKey as cPublicKey, PrivateKey as cPrivateKey -from coincurve.utils import ( - pem_to_der, lib as libsecp256k1, ffi as libsecp256k1_ffi -) -from coincurve.ecdsa import CDATA_SIG_LENGTH - -from lbry.crypto.hash import hmac_sha512, hash160, double_sha256 -from lbry.crypto.base58 import Base58 -from .util import cachedproperty - - -class KeyPath: - RECEIVE = 0 - CHANGE = 1 - CHANNEL = 2 - - -class DerivationError(Exception): - """ Raised when an invalid derivation occurs. """ - - -class _KeyBase: - """ A BIP32 Key, public or private. """ - - def __init__(self, ledger, chain_code, n, depth, parent): - if not isinstance(chain_code, (bytes, bytearray)): - raise TypeError('chain code must be raw bytes') - if len(chain_code) != 32: - raise ValueError('invalid chain code') - if not 0 <= n < 1 << 32: - raise ValueError('invalid child number') - if not 0 <= depth < 256: - raise ValueError('invalid depth') - if parent is not None: - if not isinstance(parent, type(self)): - raise TypeError('parent key has bad type') - self.ledger = ledger - self.chain_code = chain_code - self.n = n - self.depth = depth - self.parent = parent - - def _hmac_sha512(self, msg): - """ Use SHA-512 to provide an HMAC, returned as a pair of 32-byte objects. """ - hmac = hmac_sha512(self.chain_code, msg) - return hmac[:32], hmac[32:] - - def _extended_key(self, ver_bytes, raw_serkey): - """ Return the 78-byte extended key given prefix version bytes and serialized key bytes. """ - if not isinstance(ver_bytes, (bytes, bytearray)): - raise TypeError('ver_bytes must be raw bytes') - if len(ver_bytes) != 4: - raise ValueError('ver_bytes must have length 4') - if not isinstance(raw_serkey, (bytes, bytearray)): - raise TypeError('raw_serkey must be raw bytes') - if len(raw_serkey) != 33: - raise ValueError('raw_serkey must have length 33') - - return ( - ver_bytes + bytes((self.depth,)) - + self.parent_fingerprint() + self.n.to_bytes(4, 'big') - + self.chain_code + raw_serkey - ) - - def identifier(self): - raise NotImplementedError - - def extended_key(self): - raise NotImplementedError - - def fingerprint(self): - """ Return the key's fingerprint as 4 bytes. """ - return self.identifier()[:4] - - def parent_fingerprint(self): - """ Return the parent key's fingerprint as 4 bytes. """ - return self.parent.fingerprint() if self.parent else bytes((0,)*4) - - def extended_key_string(self): - """ Return an extended key as a base58 string. """ - return Base58.encode_check(self.extended_key()) - - -class PublicKey(_KeyBase): - """ A BIP32 public key. """ - - def __init__(self, ledger, pubkey, chain_code, n, depth, parent=None): - super().__init__(ledger, chain_code, n, depth, parent) - if isinstance(pubkey, cPublicKey): - self.verifying_key = pubkey - else: - self.verifying_key = self._verifying_key_from_pubkey(pubkey) - - @classmethod - def from_compressed(cls, public_key_bytes, ledger=None) -> 'PublicKey': - return cls(ledger, public_key_bytes, bytes((0,)*32), 0, 0) - - @classmethod - def _verifying_key_from_pubkey(cls, pubkey): - """ Converts a 33-byte compressed pubkey into an coincurve.PublicKey object. """ - if not isinstance(pubkey, (bytes, bytearray)): - raise TypeError('pubkey must be raw bytes') - if len(pubkey) != 33: - raise ValueError('pubkey must be 33 bytes') - if pubkey[0] not in (2, 3): - raise ValueError('invalid pubkey prefix byte') - return cPublicKey(pubkey) - - @cachedproperty - def pubkey_bytes(self): - """ Return the compressed public key as 33 bytes. """ - return self.verifying_key.format(True) - - @cachedproperty - def address(self): - """ The public key as a P2PKH address. """ - return self.ledger.public_key_to_address(self.pubkey_bytes) - - def ec_point(self): - return self.verifying_key.point() - - def child(self, n: int) -> 'PublicKey': - """ Return the derived child extended pubkey at index N. """ - if not 0 <= n < (1 << 31): - raise ValueError('invalid BIP32 public key child number') - - msg = self.pubkey_bytes + n.to_bytes(4, 'big') - L_b, R_b = self._hmac_sha512(msg) # pylint: disable=invalid-name - derived_key = self.verifying_key.add(L_b) - return PublicKey(self.ledger, derived_key, R_b, n, self.depth + 1, self) - - def identifier(self): - """ Return the key's identifier as 20 bytes. """ - return hash160(self.pubkey_bytes) - - def extended_key(self): - """ Return a raw extended public key. """ - return self._extended_key( - self.ledger.extended_public_key_prefix, - self.pubkey_bytes - ) - - def verify(self, signature, digest) -> bool: - """ Verify that a signature is valid for a 32 byte digest. """ - - if len(signature) != 64: - raise ValueError('Signature must be 64 bytes long.') - - if len(digest) != 32: - raise ValueError('Digest must be 32 bytes long.') - - key = self.verifying_key - - raw_signature = libsecp256k1_ffi.new('secp256k1_ecdsa_signature *') - - parsed = libsecp256k1.secp256k1_ecdsa_signature_parse_compact( - key.context.ctx, raw_signature, signature - ) - assert parsed == 1 - - normalized_signature = libsecp256k1_ffi.new('secp256k1_ecdsa_signature *') - - libsecp256k1.secp256k1_ecdsa_signature_normalize( - key.context.ctx, normalized_signature, raw_signature - ) - - verified = libsecp256k1.secp256k1_ecdsa_verify( - key.context.ctx, normalized_signature, digest, key.public_key - ) - - return bool(verified) - - -class PrivateKey(_KeyBase): - """A BIP32 private key.""" - - HARDENED = 1 << 31 - - def __init__(self, ledger, privkey, chain_code, n, depth, parent=None): - super().__init__(ledger, chain_code, n, depth, parent) - if isinstance(privkey, cPrivateKey): - self.signing_key = privkey - else: - self.signing_key = self._signing_key_from_privkey(privkey) - - @classmethod - def _signing_key_from_privkey(cls, private_key): - """ Converts a 32-byte private key into an coincurve.PrivateKey object. """ - return cPrivateKey.from_int(PrivateKey._private_key_secret_exponent(private_key)) - - @classmethod - def _private_key_secret_exponent(cls, private_key): - """ Return the private key as a secret exponent if it is a valid private key. """ - if not isinstance(private_key, (bytes, bytearray)): - raise TypeError('private key must be raw bytes') - if len(private_key) != 32: - raise ValueError('private key must be 32 bytes') - return int.from_bytes(private_key, 'big') - - @classmethod - def from_seed(cls, ledger, seed) -> 'PrivateKey': - # This hard-coded message string seems to be coin-independent... - hmac = hmac_sha512(b'Bitcoin seed', seed) - privkey, chain_code = hmac[:32], hmac[32:] - return cls(ledger, privkey, chain_code, 0, 0) - - @classmethod - def from_pem(cls, ledger, pem) -> 'PrivateKey': - der = pem_to_der(pem.encode()) - try: - key_int = ECPrivateKey.load(der).native['private_key'] - except ValueError: - key_int = PrivateKeyInfo.load(der).native['private_key']['private_key'] - private_key = cPrivateKey.from_int(key_int) - return cls(ledger, private_key, bytes((0,)*32), 0, 0) - - @cachedproperty - def private_key_bytes(self): - """ Return the serialized private key (no leading zero byte). """ - return self.signing_key.secret - - @cachedproperty - def public_key(self) -> PublicKey: - """ Return the corresponding extended public key. """ - verifying_key = self.signing_key.public_key - parent_pubkey = self.parent.public_key if self.parent else None - return PublicKey( - self.ledger, verifying_key, self.chain_code, - self.n, self.depth, parent_pubkey - ) - - def ec_point(self): - return self.public_key.ec_point() - - def secret_exponent(self): - """ Return the private key as a secret exponent. """ - return self.signing_key.to_int() - - def wif(self): - """ Return the private key encoded in Wallet Import Format. """ - return self.ledger.private_key_to_wif(self.private_key_bytes) - - @property - def address(self): - """ The public key as a P2PKH address. """ - return self.public_key.address - - def child(self, n) -> 'PrivateKey': - """ Return the derived child extended private key at index N.""" - if not 0 <= n < (1 << 32): - raise ValueError('invalid BIP32 private key child number') - - if n >= self.HARDENED: - serkey = b'\0' + self.private_key_bytes - else: - serkey = self.public_key.pubkey_bytes - - msg = serkey + n.to_bytes(4, 'big') - L_b, R_b = self._hmac_sha512(msg) # pylint: disable=invalid-name - derived_key = self.signing_key.add(L_b) - return PrivateKey(self.ledger, derived_key, R_b, n, self.depth + 1, self) - - def sign(self, data): - """ Produce a signature for piece of data by double hashing it and signing the hash. """ - return self.signing_key.sign(data, hasher=double_sha256) - - def sign_compact(self, digest): - """ Produce a compact signature. """ - key = self.signing_key - - signature = libsecp256k1_ffi.new('secp256k1_ecdsa_signature *') - signed = libsecp256k1.secp256k1_ecdsa_sign( - key.context.ctx, signature, digest, key.secret, - libsecp256k1_ffi.NULL, libsecp256k1_ffi.NULL - ) - - if not signed: - raise ValueError('The private key was invalid.') - - serialized = libsecp256k1_ffi.new('unsigned char[%d]' % CDATA_SIG_LENGTH) - compacted = libsecp256k1.secp256k1_ecdsa_signature_serialize_compact( - key.context.ctx, serialized, signature - ) - if compacted != 1: - raise ValueError('The signature could not be compacted.') - - return bytes(libsecp256k1_ffi.buffer(serialized, CDATA_SIG_LENGTH)) - - def identifier(self): - """Return the key's identifier as 20 bytes.""" - return self.public_key.identifier() - - def extended_key(self): - """Return a raw extended private key.""" - return self._extended_key( - self.ledger.extended_private_key_prefix, - b'\0' + self.private_key_bytes - ) - - def to_pem(self): - return self.signing_key.to_pem() - - -def _from_extended_key(ledger, ekey): - """Return a PublicKey or PrivateKey from an extended key raw bytes.""" - if not isinstance(ekey, (bytes, bytearray)): - raise TypeError('extended key must be raw bytes') - if len(ekey) != 78: - raise ValueError('extended key must have length 78') - - depth = ekey[4] - n = int.from_bytes(ekey[9:13], 'big') - chain_code = ekey[13:45] - - if ekey[:4] == ledger.extended_public_key_prefix: - pubkey = ekey[45:] - key = PublicKey(ledger, pubkey, chain_code, n, depth) - elif ekey[:4] == ledger.extended_private_key_prefix: - if ekey[45] != 0: - raise ValueError('invalid extended private key prefix byte') - privkey = ekey[46:] - key = PrivateKey(ledger, privkey, chain_code, n, depth) - else: - raise ValueError('version bytes unrecognised') - - return key - - -def from_extended_key_string(ledger, ekey_str): - """Given an extended key string, such as - - xpub6BsnM1W2Y7qLMiuhi7f7dbAwQZ5Cz5gYJCRzTNainXzQXYjFwtuQXHd - 3qfi3t3KJtHxshXezfjft93w4UE7BGMtKwhqEHae3ZA7d823DVrL - - return a PublicKey or PrivateKey. - """ - return _from_extended_key(ledger, Base58.decode_check(ekey_str)) diff --git a/lbry/wallet/database.py b/lbry/wallet/database.py index 2ddf9a666..68994b6fb 100644 --- a/lbry/wallet/database.py +++ b/lbry/wallet/database.py @@ -13,7 +13,7 @@ from datetime import date from prometheus_client import Gauge, Counter, Histogram from lbry.utils import LockWithMetrics -from .bip32 import PublicKey +from scribe.bip32 import PublicKey from .transaction import Transaction, Output, OutputScript, TXRefImmutable, Input from .constants import TXO_TYPES, CLAIM_TYPES from .util import date_to_julian_day diff --git a/lbry/wallet/ledger.py b/lbry/wallet/ledger.py index bf411f08d..18c0c44dd 100644 --- a/lbry/wallet/ledger.py +++ b/lbry/wallet/ledger.py @@ -10,8 +10,8 @@ from collections import defaultdict from binascii import hexlify, unhexlify from typing import Dict, Tuple, Type, Iterable, List, Optional, DefaultDict, NamedTuple -from lbry.schema.result import Outputs, INVALID, NOT_FOUND -from lbry.schema.url import URL +from scribe.schema.result import Outputs, INVALID, NOT_FOUND +from scribe.schema.url import URL from lbry.crypto.hash import hash160, double_sha256, sha256 from lbry.crypto.base58 import Base58 from lbry.utils import LRUCacheWithMetrics @@ -26,7 +26,7 @@ from lbry.wallet.transaction import Transaction, Output from lbry.wallet.header import Headers, UnvalidatedHeaders from lbry.wallet.checkpoints import HASHES from lbry.wallet.constants import TXO_TYPES, CLAIM_TYPES, COIN, NULL_HASH32 -from lbry.wallet.bip32 import PublicKey, PrivateKey +from scribe.bip32 import PublicKey, PrivateKey from lbry.wallet.coinselection import CoinSelector log = logging.getLogger(__name__) diff --git a/lbry/wallet/network.py b/lbry/wallet/network.py index 27c57dd1b..dab822603 100644 --- a/lbry/wallet/network.py +++ b/lbry/wallet/network.py @@ -8,8 +8,8 @@ from collections import defaultdict from typing import Dict, Optional, Tuple import aiohttp import grpc -from lbry.schema.types.v2 import hub_pb2_grpc -from lbry.schema.types.v2.hub_pb2 import SearchRequest +from scribe.schema.types.v2 import hub_pb2_grpc +from scribe.schema.types.v2.hub_pb2 import SearchRequest from lbry import __version__ from lbry.utils import resolve_host diff --git a/lbry/wallet/orchstr8/node.py b/lbry/wallet/orchstr8/node.py index 73abaa118..77c991985 100644 --- a/lbry/wallet/orchstr8/node.py +++ b/lbry/wallet/orchstr8/node.py @@ -8,7 +8,6 @@ import tarfile import logging import tempfile import subprocess -import importlib import platform from distutils.util import strtobool @@ -22,9 +21,8 @@ from lbry.conf import KnownHubsList, Config from lbry.wallet.orchstr8 import __hub_url__ from scribe.env import Env -from scribe.server.server import BlockchainReaderServer -from scribe.elasticsearch.sync import ElasticWriter -from scribe.writer.block_processor import BlockProcessor +from scribe.readers import BlockchainReaderServer, ElasticWriter +from scribe.blockchain.block_processor import BlockProcessor log = logging.getLogger(__name__) @@ -235,7 +233,6 @@ class SPVNode: if not self.stopped: log.warning("spv node is already running") return - print("start spv node") self.stopped = False try: self.data_path = tempfile.mkdtemp() @@ -263,9 +260,7 @@ class SPVNode: self.server = BlockchainReaderServer(env) self.es_writer = ElasticWriter(env) await self.writer.open() - print("opened writer, starting") await self.writer.start() - print("started writer") await self.es_writer.start() await self.server.start() except Exception as e: diff --git a/lbry/wallet/server/__init__.py b/lbry/wallet/server/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/lbry/wallet/server/block_processor.py b/lbry/wallet/server/block_processor.py deleted file mode 100644 index e948bc67c..000000000 --- a/lbry/wallet/server/block_processor.py +++ /dev/null @@ -1,1697 +0,0 @@ -import time -import asyncio -import typing -import signal - -from bisect import bisect_right -from struct import pack -from concurrent.futures.thread import ThreadPoolExecutor -from typing import Optional, List, Tuple, Set, DefaultDict, Dict -from prometheus_client import Gauge, Histogram -from collections import defaultdict - -import lbry -from lbry.schema.claim import Claim -from lbry.wallet.ledger import Ledger, TestNetLedger, RegTestLedger -from lbry.wallet.rpc.jsonrpc import RPCError -from lbry.wallet.server.tx import Tx, TxOutput, TxInput -from lbry.wallet.server.hash import hash_to_hex_str -from lbry.wallet.server.util import class_logger -from lbry.crypto.hash import hash160 -from lbry.wallet.server.db.prefixes import ACTIVATED_SUPPORT_TXO_TYPE, ACTIVATED_CLAIM_TXO_TYPE -from lbry.wallet.server.db.prefixes import PendingActivationKey, PendingActivationValue, ClaimToTXOValue -from lbry.wallet.server.prefetcher import Prefetcher -from lbry.wallet.server.db.db import HubDB -from lbry.wallet.transaction import OutputScript, Output, Transaction - -if typing.TYPE_CHECKING: - from lbry.wallet.server.env import Env - from lbry.wallet.server.db.revertable import RevertableOpStack - - -class ChainError(Exception): - """Raised on error processing blocks.""" - - -class StagedClaimtrieItem(typing.NamedTuple): - name: str - normalized_name: str - claim_hash: bytes - amount: int - expiration_height: int - tx_num: int - position: int - root_tx_num: int - root_position: int - channel_signature_is_valid: bool - signing_hash: Optional[bytes] - reposted_claim_hash: Optional[bytes] - - @property - def is_update(self) -> bool: - return (self.tx_num, self.position) != (self.root_tx_num, self.root_position) - - def invalidate_signature(self) -> 'StagedClaimtrieItem': - return StagedClaimtrieItem( - self.name, self.normalized_name, self.claim_hash, self.amount, self.expiration_height, self.tx_num, - self.position, self.root_tx_num, self.root_position, False, None, self.reposted_claim_hash - ) - - -HISTOGRAM_BUCKETS = ( - .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf') -) - - -class BlockProcessor: - """Process blocks and update the DB state to match. - - Employ a prefetcher to prefetch blocks in batches for processing. - Coordinate backing up in case of chain reorganisations. - """ - - block_count_metric = Gauge( - "block_count", "Number of processed blocks", namespace="block_processor" - ) - block_update_time_metric = Histogram( - "block_time", "Block update times", namespace="block_processor", buckets=HISTOGRAM_BUCKETS - ) - reorg_count_metric = Gauge( - "reorg_count", "Number of reorgs", namespace="block_processor" - ) - - def __init__(self, env: 'Env'): - self.cancellable_tasks = [] - - self.env = env - self.state_lock = asyncio.Lock() - self.daemon = env.coin.DAEMON(env.coin, env.daemon_url) - self._chain_executor = ThreadPoolExecutor(1, thread_name_prefix='block-processor') - self.db = HubDB( - env.coin, env.db_dir, env.cache_MB, env.reorg_limit, env.cache_all_claim_txos, env.cache_all_tx_hashes, - max_open_files=env.db_max_open_files, blocking_channel_ids=env.blocking_channel_ids, - filtering_channel_ids=env.filtering_channel_ids, executor=self._chain_executor - ) - self.shutdown_event = asyncio.Event() - self.coin = env.coin - if env.coin.NET == 'mainnet': - self.ledger = Ledger - elif env.coin.NET == 'testnet': - self.ledger = TestNetLedger - else: - self.ledger = RegTestLedger - self.wait_for_blocks_duration = 0.1 - - self._caught_up_event: Optional[asyncio.Event] = None - self.height = 0 - self.tip = bytes.fromhex(self.coin.GENESIS_HASH)[::-1] - self.tx_count = 0 - - self.blocks_event = asyncio.Event() - self.prefetcher = Prefetcher(self.daemon, env.coin, self.blocks_event) - self.logger = class_logger(__name__, self.__class__.__name__) - - # Meta - self.touched_hashXs: Set[bytes] = set() - - # UTXO cache - self.utxo_cache: Dict[Tuple[bytes, int], Tuple[bytes, int]] = {} - - # Claimtrie cache - self.db_op_stack: Optional['RevertableOpStack'] = None - - ################################# - # attributes used for calculating stake activations and takeovers per block - ################################# - - self.taken_over_names: Set[str] = set() - # txo to pending claim - self.txo_to_claim: Dict[Tuple[int, int], StagedClaimtrieItem] = {} - # claim hash to pending claim txo - self.claim_hash_to_txo: Dict[bytes, Tuple[int, int]] = {} - # claim hash to lists of pending support txos - self.support_txos_by_claim: DefaultDict[bytes, List[Tuple[int, int]]] = defaultdict(list) - # support txo: (supported claim hash, support amount) - self.support_txo_to_claim: Dict[Tuple[int, int], Tuple[bytes, int]] = {} - # removed supports {name: {claim_hash: [(tx_num, nout), ...]}} - self.removed_support_txos_by_name_by_claim: DefaultDict[str, DefaultDict[bytes, List[Tuple[int, int]]]] = \ - defaultdict(lambda: defaultdict(list)) - self.abandoned_claims: Dict[bytes, StagedClaimtrieItem] = {} - self.updated_claims: Set[bytes] = set() - # removed activated support amounts by claim hash - self.removed_active_support_amount_by_claim: DefaultDict[bytes, List[int]] = defaultdict(list) - # pending activated support amounts by claim hash - self.activated_support_amount_by_claim: DefaultDict[bytes, List[int]] = defaultdict(list) - # pending activated name and claim hash to claim/update txo amount - self.activated_claim_amount_by_name_and_hash: Dict[Tuple[str, bytes], int] = {} - # pending claim and support activations per claim hash per name, - # used to process takeovers due to added activations - activation_by_claim_by_name_type = DefaultDict[str, DefaultDict[bytes, List[Tuple[PendingActivationKey, int]]]] - self.activation_by_claim_by_name: activation_by_claim_by_name_type = defaultdict(lambda: defaultdict(list)) - # these are used for detecting early takeovers by not yet activated claims/supports - self.possible_future_support_amounts_by_claim_hash: DefaultDict[bytes, List[int]] = defaultdict(list) - self.possible_future_claim_amount_by_name_and_hash: Dict[Tuple[str, bytes], int] = {} - self.possible_future_support_txos_by_claim_hash: DefaultDict[bytes, List[Tuple[int, int]]] = defaultdict(list) - - self.removed_claims_to_send_es = set() # cumulative changes across blocks to send ES - self.touched_claims_to_send_es = set() - - self.removed_claim_hashes: Set[bytes] = set() # per block changes - self.touched_claim_hashes: Set[bytes] = set() - - self.signatures_changed = set() - - self.pending_reposted = set() - self.pending_channel_counts = defaultdict(lambda: 0) - self.pending_support_amount_change = defaultdict(lambda: 0) - - self.pending_channels = {} - self.amount_cache = {} - self.expired_claim_hashes: Set[bytes] = set() - - self.doesnt_have_valid_signature: Set[bytes] = set() - self.claim_channels: Dict[bytes, bytes] = {} - self.hashXs_by_tx: DefaultDict[bytes, List[int]] = defaultdict(list) - - self.pending_transaction_num_mapping: Dict[bytes, int] = {} - self.pending_transactions: Dict[int, bytes] = {} - - self._stopping = False - self._ready_to_stop = asyncio.Event() - - async def run_in_thread_with_lock(self, func, *args): - # Run in a thread to prevent blocking. Shielded so that - # cancellations from shutdown don't lose work - when the task - # completes the data will be flushed and then we shut down. - # Take the state lock to be certain in-memory state is - # consistent and not being updated elsewhere. - async def run_in_thread_locked(): - async with self.state_lock: - return await asyncio.get_event_loop().run_in_executor(self._chain_executor, func, *args) - return await asyncio.shield(run_in_thread_locked()) - - async def run_in_thread(self, func, *args): - async def run_in_thread(): - return await asyncio.get_event_loop().run_in_executor(self._chain_executor, func, *args) - return await asyncio.shield(run_in_thread()) - - async def refresh_mempool(self): - def fetch_mempool(mempool_prefix): - return { - k.tx_hash: v.raw_tx for (k, v) in mempool_prefix.iterate() - } - - def update_mempool(unsafe_commit, mempool_prefix, to_put, to_delete): - for tx_hash, raw_tx in to_put: - mempool_prefix.stage_put((tx_hash,), (raw_tx,)) - for tx_hash, raw_tx in to_delete.items(): - mempool_prefix.stage_delete((tx_hash,), (raw_tx,)) - unsafe_commit() - - async with self.state_lock: - current_mempool = await self.run_in_thread(fetch_mempool, self.db.prefix_db.mempool_tx) - _to_put = [] - try: - mempool_hashes = await self.daemon.mempool_hashes() - except (TypeError, RPCError): - self.logger.warning("failed to get mempool tx hashes, reorg underway?") - return - for hh in mempool_hashes: - tx_hash = bytes.fromhex(hh)[::-1] - if tx_hash in current_mempool: - current_mempool.pop(tx_hash) - else: - try: - _to_put.append((tx_hash, bytes.fromhex(await self.daemon.getrawtransaction(hh)))) - except (TypeError, RPCError): - self.logger.warning("failed to get a mempool tx, reorg underway?") - return - if current_mempool: - if bytes.fromhex(await self.daemon.getbestblockhash())[::-1] != self.coin.header_hash(self.db.headers[-1]): - return - await self.run_in_thread( - update_mempool, self.db.prefix_db.unsafe_commit, self.db.prefix_db.mempool_tx, _to_put, current_mempool - ) - - async def check_and_advance_blocks(self, raw_blocks): - """Process the list of raw blocks passed. Detects and handles - reorgs. - """ - - if not raw_blocks: - return - first = self.height + 1 - blocks = [self.coin.block(raw_block, first + n) - for n, raw_block in enumerate(raw_blocks)] - headers = [block.header for block in blocks] - hprevs = [self.coin.header_prevhash(h) for h in headers] - chain = [self.tip] + [self.coin.header_hash(h) for h in headers[:-1]] - - if hprevs == chain: - total_start = time.perf_counter() - try: - for block in blocks: - if self._stopping: - return - start = time.perf_counter() - await self.run_in_thread_with_lock(self.advance_block, block) - - self.logger.info("writer advanced to %i in %0.3fs", self.height, time.perf_counter() - start) - if self.height == self.coin.nExtendedClaimExpirationForkHeight: - self.logger.warning( - "applying extended claim expiration fork on claims accepted by, %i", self.height - ) - await self.run_in_thread_with_lock(self.db.apply_expiration_extension_fork) - except: - self.logger.exception("advance blocks failed") - raise - processed_time = time.perf_counter() - total_start - self.block_count_metric.set(self.height) - self.block_update_time_metric.observe(processed_time) - self.touched_hashXs.clear() - elif hprevs[0] != chain[0]: - min_start_height = max(self.height - self.coin.REORG_LIMIT, 0) - count = 1 - block_hashes_from_lbrycrd = await self.daemon.block_hex_hashes( - min_start_height, self.coin.REORG_LIMIT - ) - for height, block_hash in zip( - reversed(range(min_start_height, min_start_height + self.coin.REORG_LIMIT)), - reversed(block_hashes_from_lbrycrd)): - if self.db.get_block_hash(height)[::-1].hex() == block_hash: - break - count += 1 - self.logger.warning(f"blockchain reorg detected at {self.height}, unwinding last {count} blocks") - try: - assert count > 0, count - for _ in range(count): - await self.run_in_thread_with_lock(self.backup_block) - self.logger.info(f'backed up to height {self.height:,d}') - - if self.env.cache_all_claim_txos: - await self.db._read_claim_txos() # TODO: don't do this - await self.prefetcher.reset_height(self.height) - self.reorg_count_metric.inc() - except: - self.logger.exception("reorg blocks failed") - raise - finally: - self.logger.info("backed up to block %i", self.height) - else: - # It is probably possible but extremely rare that what - # bitcoind returns doesn't form a chain because it - # reorg-ed the chain as it was processing the batched - # block hash requests. Should this happen it's simplest - # just to reset the prefetcher and try again. - self.logger.warning('daemon blocks do not form a chain; ' - 'resetting the prefetcher') - await self.prefetcher.reset_height(self.height) - - def _add_claim_or_update(self, height: int, txo: 'Output', tx_hash: bytes, tx_num: int, nout: int, - spent_claims: typing.Dict[bytes, typing.Tuple[int, int, str]]): - try: - claim_name = txo.script.values['claim_name'].decode() - except UnicodeDecodeError: - claim_name = ''.join(chr(c) for c in txo.script.values['claim_name']) - try: - normalized_name = txo.normalized_name - except UnicodeDecodeError: - normalized_name = claim_name - if txo.script.is_claim_name: - claim_hash = hash160(tx_hash + pack('>I', nout))[::-1] - # print(f"\tnew {claim_hash.hex()} ({tx_num} {txo.amount})") - else: - claim_hash = txo.claim_hash[::-1] - # print(f"\tupdate {claim_hash.hex()} ({tx_num} {txo.amount})") - - signing_channel_hash = None - channel_signature_is_valid = False - try: - signable = txo.signable - is_repost = txo.claim.is_repost - is_channel = txo.claim.is_channel - if txo.claim.is_signed: - signing_channel_hash = txo.signable.signing_channel_hash[::-1] - except: # google.protobuf.message.DecodeError: Could not parse JSON. - signable = None - is_repost = False - is_channel = False - - reposted_claim_hash = None - - if is_repost: - reposted_claim_hash = txo.claim.repost.reference.claim_hash[::-1] - self.pending_reposted.add(reposted_claim_hash) - - if is_channel: - self.pending_channels[claim_hash] = txo.claim.channel.public_key_bytes - - self.doesnt_have_valid_signature.add(claim_hash) - raw_channel_tx = None - if signable and signable.signing_channel_hash: - signing_channel = self.db.get_claim_txo(signing_channel_hash) - - if signing_channel: - raw_channel_tx = self.db.prefix_db.tx.get( - self.db.get_tx_hash(signing_channel.tx_num), deserialize_value=False - ) - channel_pub_key_bytes = None - try: - if not signing_channel: - if txo.signable.signing_channel_hash[::-1] in self.pending_channels: - channel_pub_key_bytes = self.pending_channels[signing_channel_hash] - elif raw_channel_tx: - chan_output = self.coin.transaction(raw_channel_tx).outputs[signing_channel.position] - chan_script = OutputScript(chan_output.pk_script) - chan_script.parse() - channel_meta = Claim.from_bytes(chan_script.values['claim']) - - channel_pub_key_bytes = channel_meta.channel.public_key_bytes - if channel_pub_key_bytes: - channel_signature_is_valid = Output.is_signature_valid( - txo.signable.signature, txo.get_signature_digest(self.ledger), channel_pub_key_bytes - ) - if channel_signature_is_valid: - self.pending_channel_counts[signing_channel_hash] += 1 - self.doesnt_have_valid_signature.remove(claim_hash) - self.claim_channels[claim_hash] = signing_channel_hash - except: - self.logger.exception(f"error validating channel signature for %s:%i", tx_hash[::-1].hex(), nout) - - if txo.script.is_claim_name: # it's a root claim - root_tx_num, root_idx = tx_num, nout - previous_amount = 0 - else: # it's a claim update - if claim_hash not in spent_claims: - # print(f"\tthis is a wonky tx, contains unlinked claim update {claim_hash.hex()}") - return - if normalized_name != spent_claims[claim_hash][2]: - self.logger.warning( - f"{tx_hash[::-1].hex()} contains mismatched name for claim update {claim_hash.hex()}" - ) - return - (prev_tx_num, prev_idx, _) = spent_claims.pop(claim_hash) - # print(f"\tupdate {claim_hash.hex()} {tx_hash[::-1].hex()} {txo.amount}") - if (prev_tx_num, prev_idx) in self.txo_to_claim: - previous_claim = self.txo_to_claim.pop((prev_tx_num, prev_idx)) - self.claim_hash_to_txo.pop(claim_hash) - root_tx_num, root_idx = previous_claim.root_tx_num, previous_claim.root_position - else: - previous_claim = self._make_pending_claim_txo(claim_hash) - root_tx_num, root_idx = previous_claim.root_tx_num, previous_claim.root_position - activation = self.db.get_activation(prev_tx_num, prev_idx) - claim_name = previous_claim.name - self.get_remove_activate_ops( - ACTIVATED_CLAIM_TXO_TYPE, claim_hash, prev_tx_num, prev_idx, activation, normalized_name, - previous_claim.amount - ) - previous_amount = previous_claim.amount - self.updated_claims.add(claim_hash) - - if self.env.cache_all_claim_txos: - self.db.claim_to_txo[claim_hash] = ClaimToTXOValue( - tx_num, nout, root_tx_num, root_idx, txo.amount, channel_signature_is_valid, claim_name - ) - self.db.txo_to_claim[tx_num][nout] = claim_hash - - pending = StagedClaimtrieItem( - claim_name, normalized_name, claim_hash, txo.amount, self.coin.get_expiration_height(height), tx_num, nout, - root_tx_num, root_idx, channel_signature_is_valid, signing_channel_hash, reposted_claim_hash - ) - self.txo_to_claim[(tx_num, nout)] = pending - self.claim_hash_to_txo[claim_hash] = (tx_num, nout) - self.get_add_claim_utxo_ops(pending) - - def get_add_claim_utxo_ops(self, pending: StagedClaimtrieItem): - # claim tip by claim hash - self.db.prefix_db.claim_to_txo.stage_put( - (pending.claim_hash,), (pending.tx_num, pending.position, pending.root_tx_num, pending.root_position, - pending.amount, pending.channel_signature_is_valid, pending.name) - ) - # claim hash by txo - self.db.prefix_db.txo_to_claim.stage_put( - (pending.tx_num, pending.position), (pending.claim_hash, pending.normalized_name) - ) - - # claim expiration - self.db.prefix_db.claim_expiration.stage_put( - (pending.expiration_height, pending.tx_num, pending.position), - (pending.claim_hash, pending.normalized_name) - ) - - # short url resolution - for prefix_len in range(10): - self.db.prefix_db.claim_short_id.stage_put( - (pending.normalized_name, pending.claim_hash.hex()[:prefix_len + 1], - pending.root_tx_num, pending.root_position), - (pending.tx_num, pending.position) - ) - - if pending.signing_hash and pending.channel_signature_is_valid: - # channel by stream - self.db.prefix_db.claim_to_channel.stage_put( - (pending.claim_hash, pending.tx_num, pending.position), (pending.signing_hash,) - ) - # stream by channel - self.db.prefix_db.channel_to_claim.stage_put( - (pending.signing_hash, pending.normalized_name, pending.tx_num, pending.position), - (pending.claim_hash,) - ) - - if pending.reposted_claim_hash: - self.db.prefix_db.repost.stage_put((pending.claim_hash,), (pending.reposted_claim_hash,)) - self.db.prefix_db.reposted_claim.stage_put( - (pending.reposted_claim_hash, pending.tx_num, pending.position), (pending.claim_hash,) - ) - - def get_remove_claim_utxo_ops(self, pending: StagedClaimtrieItem): - # claim tip by claim hash - self.db.prefix_db.claim_to_txo.stage_delete( - (pending.claim_hash,), (pending.tx_num, pending.position, pending.root_tx_num, pending.root_position, - pending.amount, pending.channel_signature_is_valid, pending.name) - ) - # claim hash by txo - self.db.prefix_db.txo_to_claim.stage_delete( - (pending.tx_num, pending.position), (pending.claim_hash, pending.normalized_name) - ) - - # claim expiration - self.db.prefix_db.claim_expiration.stage_delete( - (pending.expiration_height, pending.tx_num, pending.position), - (pending.claim_hash, pending.normalized_name) - ) - - # short url resolution - for prefix_len in range(10): - self.db.prefix_db.claim_short_id.stage_delete( - (pending.normalized_name, pending.claim_hash.hex()[:prefix_len + 1], - pending.root_tx_num, pending.root_position), - (pending.tx_num, pending.position) - ) - - if pending.signing_hash and pending.channel_signature_is_valid: - # channel by stream - self.db.prefix_db.claim_to_channel.stage_delete( - (pending.claim_hash, pending.tx_num, pending.position), (pending.signing_hash,) - ) - # stream by channel - self.db.prefix_db.channel_to_claim.stage_delete( - (pending.signing_hash, pending.normalized_name, pending.tx_num, pending.position), - (pending.claim_hash,) - ) - - if pending.reposted_claim_hash: - self.db.prefix_db.repost.stage_delete((pending.claim_hash,), (pending.reposted_claim_hash,)) - self.db.prefix_db.reposted_claim.stage_delete( - (pending.reposted_claim_hash, pending.tx_num, pending.position), (pending.claim_hash,) - ) - - def _add_support(self, height: int, txo: 'Output', tx_num: int, nout: int): - supported_claim_hash = txo.claim_hash[::-1] - self.support_txos_by_claim[supported_claim_hash].append((tx_num, nout)) - self.support_txo_to_claim[(tx_num, nout)] = supported_claim_hash, txo.amount - # print(f"\tsupport claim {supported_claim_hash.hex()} +{txo.amount}") - - self.db.prefix_db.claim_to_support.stage_put((supported_claim_hash, tx_num, nout), (txo.amount,)) - self.db.prefix_db.support_to_claim.stage_put((tx_num, nout), (supported_claim_hash,)) - self.pending_support_amount_change[supported_claim_hash] += txo.amount - - def _add_claim_or_support(self, height: int, tx_hash: bytes, tx_num: int, nout: int, txo: 'Output', - spent_claims: typing.Dict[bytes, Tuple[int, int, str]]): - if txo.script.is_claim_name or txo.script.is_update_claim: - self._add_claim_or_update(height, txo, tx_hash, tx_num, nout, spent_claims) - elif txo.script.is_support_claim or txo.script.is_support_claim_data: - self._add_support(height, txo, tx_num, nout) - - def _spend_support_txo(self, height: int, txin: TxInput): - txin_num = self.get_pending_tx_num(txin.prev_hash) - activation = 0 - if (txin_num, txin.prev_idx) in self.support_txo_to_claim: - spent_support, support_amount = self.support_txo_to_claim.pop((txin_num, txin.prev_idx)) - self.support_txos_by_claim[spent_support].remove((txin_num, txin.prev_idx)) - supported_name = self._get_pending_claim_name(spent_support) - self.removed_support_txos_by_name_by_claim[supported_name][spent_support].append((txin_num, txin.prev_idx)) - else: - spent_support, support_amount = self.db.get_supported_claim_from_txo(txin_num, txin.prev_idx) - if not spent_support: # it is not a support - return - supported_name = self._get_pending_claim_name(spent_support) - if supported_name is not None: - self.removed_support_txos_by_name_by_claim[supported_name][spent_support].append( - (txin_num, txin.prev_idx)) - activation = self.db.get_activation(txin_num, txin.prev_idx, is_support=True) - if 0 < activation < self.height + 1: - self.removed_active_support_amount_by_claim[spent_support].append(support_amount) - if supported_name is not None and activation > 0: - self.get_remove_activate_ops( - ACTIVATED_SUPPORT_TXO_TYPE, spent_support, txin_num, txin.prev_idx, activation, supported_name, - support_amount - ) - # print(f"\tspent support for {spent_support.hex()} activation:{activation} {support_amount}") - self.db.prefix_db.claim_to_support.stage_delete((spent_support, txin_num, txin.prev_idx), (support_amount,)) - self.db.prefix_db.support_to_claim.stage_delete((txin_num, txin.prev_idx), (spent_support,)) - self.pending_support_amount_change[spent_support] -= support_amount - - def _spend_claim_txo(self, txin: TxInput, spent_claims: Dict[bytes, Tuple[int, int, str]]) -> bool: - txin_num = self.get_pending_tx_num(txin.prev_hash) - if (txin_num, txin.prev_idx) in self.txo_to_claim: - spent = self.txo_to_claim[(txin_num, txin.prev_idx)] - else: - if not self.db.get_cached_claim_exists(txin_num, txin.prev_idx): - # txo is not a claim - return False - spent_claim_hash_and_name = self.db.get_claim_from_txo( - txin_num, txin.prev_idx - ) - assert spent_claim_hash_and_name is not None - spent = self._make_pending_claim_txo(spent_claim_hash_and_name.claim_hash) - - if self.env.cache_all_claim_txos: - claim_hash = self.db.txo_to_claim[txin_num].pop(txin.prev_idx) - if not self.db.txo_to_claim[txin_num]: - self.db.txo_to_claim.pop(txin_num) - self.db.claim_to_txo.pop(claim_hash) - if spent.reposted_claim_hash: - self.pending_reposted.add(spent.reposted_claim_hash) - if spent.signing_hash and spent.channel_signature_is_valid and spent.signing_hash not in self.abandoned_claims: - self.pending_channel_counts[spent.signing_hash] -= 1 - spent_claims[spent.claim_hash] = (spent.tx_num, spent.position, spent.normalized_name) - # print(f"\tspend lbry://{spent.name}#{spent.claim_hash.hex()}") - self.get_remove_claim_utxo_ops(spent) - return True - - def _spend_claim_or_support_txo(self, height: int, txin: TxInput, spent_claims): - if not self._spend_claim_txo(txin, spent_claims): - self._spend_support_txo(height, txin) - - def _abandon_claim(self, claim_hash: bytes, tx_num: int, nout: int, normalized_name: str): - if (tx_num, nout) in self.txo_to_claim: - pending = self.txo_to_claim.pop((tx_num, nout)) - self.claim_hash_to_txo.pop(claim_hash) - self.abandoned_claims[pending.claim_hash] = pending - claim_root_tx_num, claim_root_idx = pending.root_tx_num, pending.root_position - prev_amount, prev_signing_hash = pending.amount, pending.signing_hash - reposted_claim_hash, name = pending.reposted_claim_hash, pending.name - expiration = self.coin.get_expiration_height(self.height) - signature_is_valid = pending.channel_signature_is_valid - else: - v = self.db.get_claim_txo( - claim_hash - ) - claim_root_tx_num, claim_root_idx, prev_amount = v.root_tx_num, v.root_position, v.amount - signature_is_valid, name = v.channel_signature_is_valid, v.name - prev_signing_hash = self.db.get_channel_for_claim(claim_hash, tx_num, nout) - reposted_claim_hash = self.db.get_repost(claim_hash) - expiration = self.coin.get_expiration_height(bisect_right(self.db.tx_counts, tx_num)) - self.abandoned_claims[claim_hash] = staged = StagedClaimtrieItem( - name, normalized_name, claim_hash, prev_amount, expiration, tx_num, nout, claim_root_tx_num, - claim_root_idx, signature_is_valid, prev_signing_hash, reposted_claim_hash - ) - for support_txo_to_clear in self.support_txos_by_claim[claim_hash]: - self.support_txo_to_claim.pop(support_txo_to_clear) - self.support_txos_by_claim[claim_hash].clear() - self.support_txos_by_claim.pop(claim_hash) - if normalized_name.startswith('@'): # abandon a channel, invalidate signatures - self._invalidate_channel_signatures(claim_hash) - - def _get_invalidate_signature_ops(self, pending: StagedClaimtrieItem): - if not pending.signing_hash: - return - self.db.prefix_db.claim_to_channel.stage_delete( - (pending.claim_hash, pending.tx_num, pending.position), (pending.signing_hash,) - ) - if pending.channel_signature_is_valid: - self.db.prefix_db.channel_to_claim.stage_delete( - (pending.signing_hash, pending.normalized_name, pending.tx_num, pending.position), - (pending.claim_hash,) - ) - self.db.prefix_db.claim_to_txo.stage_delete( - (pending.claim_hash,), - (pending.tx_num, pending.position, pending.root_tx_num, pending.root_position, pending.amount, - pending.channel_signature_is_valid, pending.name) - ) - self.db.prefix_db.claim_to_txo.stage_put( - (pending.claim_hash,), - (pending.tx_num, pending.position, pending.root_tx_num, pending.root_position, pending.amount, - False, pending.name) - ) - - def _invalidate_channel_signatures(self, claim_hash: bytes): - for (signed_claim_hash, ) in self.db.prefix_db.channel_to_claim.iterate( - prefix=(claim_hash, ), include_key=False): - if signed_claim_hash in self.abandoned_claims or signed_claim_hash in self.expired_claim_hashes: - continue - # there is no longer a signing channel for this claim as of this block - if signed_claim_hash in self.doesnt_have_valid_signature: - continue - # the signing channel changed in this block - if signed_claim_hash in self.claim_channels and signed_claim_hash != self.claim_channels[signed_claim_hash]: - continue - - # if the claim with an invalidated signature is in this block, update the StagedClaimtrieItem - # so that if we later try to spend it in this block we won't try to delete the channel info twice - if signed_claim_hash in self.claim_hash_to_txo: - signed_claim_txo = self.claim_hash_to_txo[signed_claim_hash] - claim = self.txo_to_claim[signed_claim_txo] - if claim.signing_hash != claim_hash: # claim was already invalidated this block - continue - self.txo_to_claim[signed_claim_txo] = claim.invalidate_signature() - else: - claim = self._make_pending_claim_txo(signed_claim_hash) - self.signatures_changed.add(signed_claim_hash) - self.pending_channel_counts[claim_hash] -= 1 - self._get_invalidate_signature_ops(claim) - - for staged in list(self.txo_to_claim.values()): - needs_invalidate = staged.claim_hash not in self.doesnt_have_valid_signature - if staged.signing_hash == claim_hash and needs_invalidate: - self._get_invalidate_signature_ops(staged) - self.txo_to_claim[self.claim_hash_to_txo[staged.claim_hash]] = staged.invalidate_signature() - self.signatures_changed.add(staged.claim_hash) - self.pending_channel_counts[claim_hash] -= 1 - - def _make_pending_claim_txo(self, claim_hash: bytes): - claim = self.db.get_claim_txo(claim_hash) - if claim_hash in self.doesnt_have_valid_signature: - signing_hash = None - else: - signing_hash = self.db.get_channel_for_claim(claim_hash, claim.tx_num, claim.position) - reposted_claim_hash = self.db.get_repost(claim_hash) - return StagedClaimtrieItem( - claim.name, claim.normalized_name, claim_hash, claim.amount, - self.coin.get_expiration_height( - bisect_right(self.db.tx_counts, claim.tx_num), - extended=self.height >= self.coin.nExtendedClaimExpirationForkHeight - ), - claim.tx_num, claim.position, claim.root_tx_num, claim.root_position, - claim.channel_signature_is_valid, signing_hash, reposted_claim_hash - ) - - def _expire_claims(self, height: int): - expired = self.db.get_expired_by_height(height) - self.expired_claim_hashes.update(set(expired.keys())) - spent_claims = {} - for expired_claim_hash, (tx_num, position, name, txi) in expired.items(): - if (tx_num, position) not in self.txo_to_claim: - self._spend_claim_txo(txi, spent_claims) - if expired: - # abandon the channels last to handle abandoned signed claims in the same tx, - # see test_abandon_channel_and_claims_in_same_tx - expired_channels = {} - for abandoned_claim_hash, (tx_num, nout, normalized_name) in spent_claims.items(): - self._abandon_claim(abandoned_claim_hash, tx_num, nout, normalized_name) - - if normalized_name.startswith('@'): - expired_channels[abandoned_claim_hash] = (tx_num, nout, normalized_name) - else: - # print(f"\texpire {abandoned_claim_hash.hex()} {tx_num} {nout}") - self._abandon_claim(abandoned_claim_hash, tx_num, nout, normalized_name) - - # do this to follow the same content claim removing pathway as if a claim (possible channel) was abandoned - for abandoned_claim_hash, (tx_num, nout, normalized_name) in expired_channels.items(): - # print(f"\texpire {abandoned_claim_hash.hex()} {tx_num} {nout}") - self._abandon_claim(abandoned_claim_hash, tx_num, nout, normalized_name) - - def _cached_get_active_amount(self, claim_hash: bytes, txo_type: int, height: int) -> int: - if (claim_hash, txo_type, height) in self.amount_cache: - return self.amount_cache[(claim_hash, txo_type, height)] - if txo_type == ACTIVATED_CLAIM_TXO_TYPE: - if claim_hash in self.claim_hash_to_txo: - amount = self.txo_to_claim[self.claim_hash_to_txo[claim_hash]].amount - else: - amount = self.db.get_active_amount_as_of_height( - claim_hash, height - ) - self.amount_cache[(claim_hash, txo_type, height)] = amount - else: - self.amount_cache[(claim_hash, txo_type, height)] = amount = self.db._get_active_amount( - claim_hash, txo_type, height - ) - return amount - - def _get_pending_claim_amount(self, name: str, claim_hash: bytes, height=None) -> int: - if (name, claim_hash) in self.activated_claim_amount_by_name_and_hash: - if claim_hash in self.claim_hash_to_txo: - return self.txo_to_claim[self.claim_hash_to_txo[claim_hash]].amount - return self.activated_claim_amount_by_name_and_hash[(name, claim_hash)] - if (name, claim_hash) in self.possible_future_claim_amount_by_name_and_hash: - return self.possible_future_claim_amount_by_name_and_hash[(name, claim_hash)] - return self._cached_get_active_amount(claim_hash, ACTIVATED_CLAIM_TXO_TYPE, height or (self.height + 1)) - - def _get_pending_claim_name(self, claim_hash: bytes) -> Optional[str]: - assert claim_hash is not None - if claim_hash in self.claim_hash_to_txo: - return self.txo_to_claim[self.claim_hash_to_txo[claim_hash]].normalized_name - claim_info = self.db.get_claim_txo(claim_hash) - if claim_info: - return claim_info.normalized_name - - def _get_pending_supported_amount(self, claim_hash: bytes, height: Optional[int] = None) -> int: - amount = self._cached_get_active_amount(claim_hash, ACTIVATED_SUPPORT_TXO_TYPE, height or (self.height + 1)) - if claim_hash in self.activated_support_amount_by_claim: - amount += sum(self.activated_support_amount_by_claim[claim_hash]) - if claim_hash in self.possible_future_support_amounts_by_claim_hash: - amount += sum(self.possible_future_support_amounts_by_claim_hash[claim_hash]) - if claim_hash in self.removed_active_support_amount_by_claim: - return amount - sum(self.removed_active_support_amount_by_claim[claim_hash]) - return amount - - def _get_pending_effective_amount(self, name: str, claim_hash: bytes, height: Optional[int] = None) -> int: - claim_amount = self._get_pending_claim_amount(name, claim_hash, height=height) - support_amount = self._get_pending_supported_amount(claim_hash, height=height) - return claim_amount + support_amount - - def get_activate_ops(self, txo_type: int, claim_hash: bytes, tx_num: int, position: int, - activation_height: int, name: str, amount: int): - self.db.prefix_db.activated.stage_put( - (txo_type, tx_num, position), (activation_height, claim_hash, name) - ) - self.db.prefix_db.pending_activation.stage_put( - (activation_height, txo_type, tx_num, position), (claim_hash, name) - ) - self.db.prefix_db.active_amount.stage_put( - (claim_hash, txo_type, activation_height, tx_num, position), (amount,) - ) - - def get_remove_activate_ops(self, txo_type: int, claim_hash: bytes, tx_num: int, position: int, - activation_height: int, name: str, amount: int): - self.db.prefix_db.activated.stage_delete( - (txo_type, tx_num, position), (activation_height, claim_hash, name) - ) - self.db.prefix_db.pending_activation.stage_delete( - (activation_height, txo_type, tx_num, position), (claim_hash, name) - ) - self.db.prefix_db.active_amount.stage_delete( - (claim_hash, txo_type, activation_height, tx_num, position), (amount,) - ) - - def _get_takeover_ops(self, height: int): - - # cache for controlling claims as of the previous block - controlling_claims = {} - - def get_controlling(_name): - if _name not in controlling_claims: - _controlling = self.db.get_controlling_claim(_name) - controlling_claims[_name] = _controlling - else: - _controlling = controlling_claims[_name] - return _controlling - - names_with_abandoned_or_updated_controlling_claims: List[str] = [] - - # get the claims and supports previously scheduled to be activated at this block - activated_at_height = self.db.get_activated_at_height(height) - activate_in_future = defaultdict(lambda: defaultdict(list)) - future_activations = defaultdict(dict) - - def get_delayed_activate_ops(name: str, claim_hash: bytes, is_new_claim: bool, tx_num: int, nout: int, - amount: int, is_support: bool): - controlling = get_controlling(name) - nothing_is_controlling = not controlling - staged_is_controlling = False if not controlling else claim_hash == controlling.claim_hash - controlling_is_abandoned = False if not controlling else \ - name in names_with_abandoned_or_updated_controlling_claims - - if nothing_is_controlling or staged_is_controlling or controlling_is_abandoned: - delay = 0 - elif is_new_claim: - delay = self.coin.get_delay_for_name(height - controlling.height) - else: - controlling_effective_amount = self._get_pending_effective_amount(name, controlling.claim_hash) - staged_effective_amount = self._get_pending_effective_amount(name, claim_hash) - staged_update_could_cause_takeover = staged_effective_amount > controlling_effective_amount - delay = 0 if not staged_update_could_cause_takeover else self.coin.get_delay_for_name( - height - controlling.height - ) - if delay == 0: # if delay was 0 it needs to be considered for takeovers - activated_at_height[PendingActivationValue(claim_hash, name)].append( - PendingActivationKey( - height, ACTIVATED_SUPPORT_TXO_TYPE if is_support else ACTIVATED_CLAIM_TXO_TYPE, tx_num, nout - ) - ) - else: # if the delay was higher if still needs to be considered if something else triggers a takeover - activate_in_future[name][claim_hash].append(( - PendingActivationKey( - height + delay, ACTIVATED_SUPPORT_TXO_TYPE if is_support else ACTIVATED_CLAIM_TXO_TYPE, - tx_num, nout - ), amount - )) - if is_support: - self.possible_future_support_txos_by_claim_hash[claim_hash].append((tx_num, nout)) - self.get_activate_ops( - ACTIVATED_SUPPORT_TXO_TYPE if is_support else ACTIVATED_CLAIM_TXO_TYPE, claim_hash, tx_num, nout, - height + delay, name, amount - ) - - # determine names needing takeover/deletion due to controlling claims being abandoned - # and add ops to deactivate abandoned claims - for claim_hash, staged in self.abandoned_claims.items(): - controlling = get_controlling(staged.normalized_name) - if controlling and controlling.claim_hash == claim_hash: - names_with_abandoned_or_updated_controlling_claims.append(staged.normalized_name) - # print(f"\t{staged.name} needs takeover") - activation = self.db.get_activation(staged.tx_num, staged.position) - if activation > 0: # db returns -1 for non-existent txos - # removed queued future activation from the db - self.get_remove_activate_ops( - ACTIVATED_CLAIM_TXO_TYPE, staged.claim_hash, staged.tx_num, staged.position, - activation, staged.normalized_name, staged.amount - ) - else: - # it hadn't yet been activated - pass - - # get the removed activated supports for controlling claims to determine if takeovers are possible - abandoned_support_check_need_takeover = defaultdict(list) - for claim_hash, amounts in self.removed_active_support_amount_by_claim.items(): - name = self._get_pending_claim_name(claim_hash) - if name is None: - continue - controlling = get_controlling(name) - if controlling and controlling.claim_hash == claim_hash and \ - name not in names_with_abandoned_or_updated_controlling_claims: - abandoned_support_check_need_takeover[(name, claim_hash)].extend(amounts) - - # get the controlling claims with updates to the claim to check if takeover is needed - for claim_hash in self.updated_claims: - if claim_hash in self.abandoned_claims: - continue - name = self._get_pending_claim_name(claim_hash) - if name is None: - continue - controlling = get_controlling(name) - if controlling and controlling.claim_hash == claim_hash and \ - name not in names_with_abandoned_or_updated_controlling_claims: - names_with_abandoned_or_updated_controlling_claims.append(name) - - # prepare to activate or delay activation of the pending claims being added this block - for (tx_num, nout), staged in self.txo_to_claim.items(): - is_delayed = not staged.is_update - prev_txo = self.db.get_cached_claim_txo(staged.claim_hash) - if prev_txo: - prev_activation = self.db.get_activation(prev_txo.tx_num, prev_txo.position) - if height < prev_activation or prev_activation < 0: - is_delayed = True - get_delayed_activate_ops( - staged.normalized_name, staged.claim_hash, is_delayed, tx_num, nout, staged.amount, - is_support=False - ) - - # and the supports - for (tx_num, nout), (claim_hash, amount) in self.support_txo_to_claim.items(): - if claim_hash in self.abandoned_claims: - continue - elif claim_hash in self.claim_hash_to_txo: - name = self.txo_to_claim[self.claim_hash_to_txo[claim_hash]].normalized_name - staged_is_new_claim = not self.txo_to_claim[self.claim_hash_to_txo[claim_hash]].is_update - else: - supported_claim_info = self.db.get_claim_txo(claim_hash) - if not supported_claim_info: - # the supported claim doesn't exist - continue - else: - v = supported_claim_info - name = v.normalized_name - staged_is_new_claim = (v.root_tx_num, v.root_position) == (v.tx_num, v.position) - get_delayed_activate_ops( - name, claim_hash, staged_is_new_claim, tx_num, nout, amount, is_support=True - ) - - # add the activation/delayed-activation ops - for activated, activated_txos in activated_at_height.items(): - controlling = get_controlling(activated.normalized_name) - if activated.claim_hash in self.abandoned_claims: - continue - reactivate = False - if not controlling or controlling.claim_hash == activated.claim_hash: - # there is no delay for claims to a name without a controlling value or to the controlling value - reactivate = True - for activated_txo in activated_txos: - if activated_txo.is_support and (activated_txo.tx_num, activated_txo.position) in \ - self.removed_support_txos_by_name_by_claim[activated.normalized_name][activated.claim_hash]: - # print("\tskip activate support for pending abandoned claim") - continue - if activated_txo.is_claim: - txo_type = ACTIVATED_CLAIM_TXO_TYPE - txo_tup = (activated_txo.tx_num, activated_txo.position) - if txo_tup in self.txo_to_claim: - amount = self.txo_to_claim[txo_tup].amount - else: - amount = self.db.get_claim_txo_amount( - activated.claim_hash - ) - if amount is None: - # print("\tskip activate for non existent claim") - continue - self.activated_claim_amount_by_name_and_hash[(activated.normalized_name, activated.claim_hash)] = amount - else: - txo_type = ACTIVATED_SUPPORT_TXO_TYPE - txo_tup = (activated_txo.tx_num, activated_txo.position) - if txo_tup in self.support_txo_to_claim: - amount = self.support_txo_to_claim[txo_tup][1] - else: - amount = self.db.get_support_txo_amount( - activated.claim_hash, activated_txo.tx_num, activated_txo.position - ) - if amount is None: - # print("\tskip activate support for non existent claim") - continue - self.activated_support_amount_by_claim[activated.claim_hash].append(amount) - self.activation_by_claim_by_name[activated.normalized_name][activated.claim_hash].append((activated_txo, amount)) - # print(f"\tactivate {'support' if txo_type == ACTIVATED_SUPPORT_TXO_TYPE else 'claim'} " - # f"{activated.claim_hash.hex()} @ {activated_txo.height}") - - # go through claims where the controlling claim or supports to the controlling claim have been abandoned - # check if takeovers are needed or if the name node is now empty - need_reactivate_if_takes_over = {} - for need_takeover in names_with_abandoned_or_updated_controlling_claims: - existing = self.db.get_claim_txos_for_name(need_takeover) - has_candidate = False - # add existing claims to the queue for the takeover - # track that we need to reactivate these if one of them becomes controlling - for candidate_claim_hash, (tx_num, nout) in existing.items(): - if candidate_claim_hash in self.abandoned_claims: - continue - has_candidate = True - existing_activation = self.db.get_activation(tx_num, nout) - activate_key = PendingActivationKey( - existing_activation, ACTIVATED_CLAIM_TXO_TYPE, tx_num, nout - ) - self.activation_by_claim_by_name[need_takeover][candidate_claim_hash].append(( - activate_key, self.db.get_claim_txo_amount(candidate_claim_hash) - )) - need_reactivate_if_takes_over[(need_takeover, candidate_claim_hash)] = activate_key - # print(f"\tcandidate to takeover abandoned controlling claim for " - # f"{activate_key.tx_num}:{activate_key.position} {activate_key.is_claim}") - if not has_candidate: - # remove name takeover entry, the name is now unclaimed - controlling = get_controlling(need_takeover) - self.db.prefix_db.claim_takeover.stage_delete( - (need_takeover,), (controlling.claim_hash, controlling.height) - ) - - # scan for possible takeovers out of the accumulated activations, of these make sure there - # aren't any future activations for the taken over names with yet higher amounts, if there are - # these need to get activated now and take over instead. for example: - # claim A is winning for 0.1 for long enough for a > 1 takeover delay - # claim B is made for 0.2 - # a block later, claim C is made for 0.3, it will schedule to activate 1 (or rarely 2) block(s) after B - # upon the delayed activation of B, we need to detect to activate C and make it take over early instead - - claim_exists = {} - for activated, activated_claim_txo in self.db.get_future_activated(height).items(): - # uses the pending effective amount for the future activation height, not the current height - future_amount = self._get_pending_claim_amount( - activated.normalized_name, activated.claim_hash, activated_claim_txo.height + 1 - ) - if activated.claim_hash not in claim_exists: - claim_exists[activated.claim_hash] = activated.claim_hash in self.claim_hash_to_txo or ( - self.db.get_claim_txo(activated.claim_hash) is not None) - if claim_exists[activated.claim_hash] and activated.claim_hash not in self.abandoned_claims: - v = future_amount, activated, activated_claim_txo - future_activations[activated.normalized_name][activated.claim_hash] = v - - for name, future_activated in activate_in_future.items(): - for claim_hash, activated in future_activated.items(): - if claim_hash not in claim_exists: - claim_exists[claim_hash] = claim_hash in self.claim_hash_to_txo or ( - self.db.get_claim_txo(claim_hash) is not None) - if not claim_exists[claim_hash]: - continue - if claim_hash in self.abandoned_claims: - continue - for txo in activated: - v = txo[1], PendingActivationValue(claim_hash, name), txo[0] - future_activations[name][claim_hash] = v - if txo[0].is_claim: - self.possible_future_claim_amount_by_name_and_hash[(name, claim_hash)] = txo[1] - else: - self.possible_future_support_amounts_by_claim_hash[claim_hash].append(txo[1]) - - # process takeovers - checked_names = set() - for name, activated in self.activation_by_claim_by_name.items(): - checked_names.add(name) - controlling = controlling_claims[name] - amounts = { - claim_hash: self._get_pending_effective_amount(name, claim_hash) - for claim_hash in activated.keys() if claim_hash not in self.abandoned_claims - } - # if there is a controlling claim include it in the amounts to ensure it remains the max - if controlling and controlling.claim_hash not in self.abandoned_claims: - amounts[controlling.claim_hash] = self._get_pending_effective_amount(name, controlling.claim_hash) - winning_claim_hash = max(amounts, key=lambda x: amounts[x]) - if not controlling or (winning_claim_hash != controlling.claim_hash and - name in names_with_abandoned_or_updated_controlling_claims) or \ - ((winning_claim_hash != controlling.claim_hash) and (amounts[winning_claim_hash] > amounts[controlling.claim_hash])): - amounts_with_future_activations = {claim_hash: amount for claim_hash, amount in amounts.items()} - amounts_with_future_activations.update( - { - claim_hash: self._get_pending_effective_amount( - name, claim_hash, self.height + 1 + self.coin.maxTakeoverDelay - ) for claim_hash in future_activations[name] - } - ) - winning_including_future_activations = max( - amounts_with_future_activations, key=lambda x: amounts_with_future_activations[x] - ) - future_winning_amount = amounts_with_future_activations[winning_including_future_activations] - - if winning_claim_hash != winning_including_future_activations and \ - future_winning_amount > amounts[winning_claim_hash]: - # print(f"\ttakeover by {winning_claim_hash.hex()} triggered early activation and " - # f"takeover by {winning_including_future_activations.hex()} at {height}") - # handle a pending activated claim jumping the takeover delay when another name takes over - if winning_including_future_activations not in self.claim_hash_to_txo: - claim = self.db.get_claim_txo(winning_including_future_activations) - tx_num = claim.tx_num - position = claim.position - amount = claim.amount - activation = self.db.get_activation(tx_num, position) - else: - tx_num, position = self.claim_hash_to_txo[winning_including_future_activations] - amount = self.txo_to_claim[(tx_num, position)].amount - activation = None - for (k, tx_amount) in activate_in_future[name][winning_including_future_activations]: - if (k.tx_num, k.position) == (tx_num, position): - activation = k.height - break - if activation is None: - # TODO: reproduce this in an integration test (block 604718) - _k = PendingActivationValue(winning_including_future_activations, name) - if _k in activated_at_height: - for pending_activation in activated_at_height[_k]: - if (pending_activation.tx_num, pending_activation.position) == (tx_num, position): - activation = pending_activation.height - break - assert None not in (amount, activation) - # update the claim that's activating early - self.get_remove_activate_ops( - ACTIVATED_CLAIM_TXO_TYPE, winning_including_future_activations, tx_num, - position, activation, name, amount - ) - self.get_activate_ops( - ACTIVATED_CLAIM_TXO_TYPE, winning_including_future_activations, tx_num, - position, height, name, amount - ) - - for (k, amount) in activate_in_future[name][winning_including_future_activations]: - txo = (k.tx_num, k.position) - if txo in self.possible_future_support_txos_by_claim_hash[winning_including_future_activations]: - self.get_remove_activate_ops( - ACTIVATED_SUPPORT_TXO_TYPE, winning_including_future_activations, k.tx_num, - k.position, k.height, name, amount - ) - self.get_activate_ops( - ACTIVATED_SUPPORT_TXO_TYPE, winning_including_future_activations, k.tx_num, - k.position, height, name, amount - ) - self.taken_over_names.add(name) - if controlling: - self.db.prefix_db.claim_takeover.stage_delete( - (name,), (controlling.claim_hash, controlling.height) - ) - self.db.prefix_db.claim_takeover.stage_put((name,), (winning_including_future_activations, height)) - self.touched_claim_hashes.add(winning_including_future_activations) - if controlling and controlling.claim_hash not in self.abandoned_claims: - self.touched_claim_hashes.add(controlling.claim_hash) - elif not controlling or (winning_claim_hash != controlling.claim_hash and - name in names_with_abandoned_or_updated_controlling_claims) or \ - ((winning_claim_hash != controlling.claim_hash) and (amounts[winning_claim_hash] > amounts[controlling.claim_hash])): - # print(f"\ttakeover by {winning_claim_hash.hex()} at {height}") - if (name, winning_claim_hash) in need_reactivate_if_takes_over: - previous_pending_activate = need_reactivate_if_takes_over[(name, winning_claim_hash)] - amount = self.db.get_claim_txo_amount( - winning_claim_hash - ) - if winning_claim_hash in self.claim_hash_to_txo: - tx_num, position = self.claim_hash_to_txo[winning_claim_hash] - amount = self.txo_to_claim[(tx_num, position)].amount - else: - tx_num, position = previous_pending_activate.tx_num, previous_pending_activate.position - if previous_pending_activate.height > height: - # the claim had a pending activation in the future, move it to now - if tx_num < self.tx_count: - self.get_remove_activate_ops( - ACTIVATED_CLAIM_TXO_TYPE, winning_claim_hash, tx_num, - position, previous_pending_activate.height, name, amount - ) - self.get_activate_ops( - ACTIVATED_CLAIM_TXO_TYPE, winning_claim_hash, tx_num, - position, height, name, amount - ) - self.taken_over_names.add(name) - if controlling: - self.db.prefix_db.claim_takeover.stage_delete( - (name,), (controlling.claim_hash, controlling.height) - ) - self.db.prefix_db.claim_takeover.stage_put((name,), (winning_claim_hash, height)) - if controlling and controlling.claim_hash not in self.abandoned_claims: - self.touched_claim_hashes.add(controlling.claim_hash) - self.touched_claim_hashes.add(winning_claim_hash) - elif winning_claim_hash == controlling.claim_hash: - # print("\tstill winning") - pass - else: - # print("\tno takeover") - pass - - # handle remaining takeovers from abandoned supports - for (name, claim_hash), amounts in abandoned_support_check_need_takeover.items(): - if name in checked_names: - continue - checked_names.add(name) - controlling = get_controlling(name) - amounts = { - claim_hash: self._get_pending_effective_amount(name, claim_hash) - for claim_hash in self.db.get_claims_for_name(name) if claim_hash not in self.abandoned_claims - } - if controlling and controlling.claim_hash not in self.abandoned_claims: - amounts[controlling.claim_hash] = self._get_pending_effective_amount(name, controlling.claim_hash) - winning = max(amounts, key=lambda x: amounts[x]) - - if (controlling and winning != controlling.claim_hash) or (not controlling and winning): - self.taken_over_names.add(name) - # print(f"\ttakeover from abandoned support {controlling.claim_hash.hex()} -> {winning.hex()}") - if controlling: - self.db.prefix_db.claim_takeover.stage_delete( - (name,), (controlling.claim_hash, controlling.height) - ) - self.db.prefix_db.claim_takeover.stage_put((name,), (winning, height)) - if controlling: - self.touched_claim_hashes.add(controlling.claim_hash) - self.touched_claim_hashes.add(winning) - - def _get_cumulative_update_ops(self, height: int): - # update the last takeover height for names with takeovers - for name in self.taken_over_names: - self.touched_claim_hashes.update( - {claim_hash for claim_hash in self.db.get_claims_for_name(name) - if claim_hash not in self.abandoned_claims} - ) - - # gather cumulative removed/touched sets to update the search index - self.removed_claim_hashes.update(set(self.abandoned_claims.keys())) - self.touched_claim_hashes.difference_update(self.removed_claim_hashes) - self.touched_claim_hashes.update( - set( - map(lambda item: item[1], self.activated_claim_amount_by_name_and_hash.keys()) - ).union( - set(self.claim_hash_to_txo.keys()) - ).union( - self.removed_active_support_amount_by_claim.keys() - ).union( - self.signatures_changed - ).union( - set(self.removed_active_support_amount_by_claim.keys()) - ).union( - set(self.activated_support_amount_by_claim.keys()) - ).union( - set(self.pending_support_amount_change.keys()) - ).difference( - self.removed_claim_hashes - ) - ) - - # update support amount totals - for supported_claim, amount in self.pending_support_amount_change.items(): - existing = self.db.prefix_db.support_amount.get(supported_claim) - total = amount - if existing is not None: - total += existing.amount - self.db.prefix_db.support_amount.stage_delete((supported_claim,), existing) - self.db.prefix_db.support_amount.stage_put((supported_claim,), (total,)) - - # use the cumulative changes to update bid ordered resolve - for removed in self.removed_claim_hashes: - removed_claim = self.db.get_claim_txo(removed) - if removed_claim: - amt = self.db.get_url_effective_amount( - removed_claim.normalized_name, removed - ) - if amt: - self.db.prefix_db.effective_amount.stage_delete( - (removed_claim.normalized_name, amt.effective_amount, amt.tx_num, amt.position), (removed,) - ) - for touched in self.touched_claim_hashes: - prev_effective_amount = 0 - - if touched in self.claim_hash_to_txo: - pending = self.txo_to_claim[self.claim_hash_to_txo[touched]] - name, tx_num, position = pending.normalized_name, pending.tx_num, pending.position - claim_from_db = self.db.get_claim_txo(touched) - if claim_from_db: - claim_amount_info = self.db.get_url_effective_amount(name, touched) - if claim_amount_info: - prev_effective_amount = claim_amount_info.effective_amount - self.db.prefix_db.effective_amount.stage_delete( - (name, claim_amount_info.effective_amount, claim_amount_info.tx_num, - claim_amount_info.position), (touched,) - ) - else: - v = self.db.get_claim_txo(touched) - if not v: - continue - name, tx_num, position = v.normalized_name, v.tx_num, v.position - amt = self.db.get_url_effective_amount(name, touched) - if amt: - prev_effective_amount = amt.effective_amount - self.db.prefix_db.effective_amount.stage_delete( - (name, prev_effective_amount, amt.tx_num, amt.position), (touched,) - ) - - new_effective_amount = self._get_pending_effective_amount(name, touched) - self.db.prefix_db.effective_amount.stage_put( - (name, new_effective_amount, tx_num, position), (touched,) - ) - if touched in self.claim_hash_to_txo or touched in self.removed_claim_hashes \ - or touched in self.pending_support_amount_change: - # exclude sending notifications for claims/supports that activated but - # weren't added/spent in this block - self.db.prefix_db.trending_notification.stage_put( - (height, touched), (prev_effective_amount, new_effective_amount) - ) - - for channel_hash, count in self.pending_channel_counts.items(): - if count != 0: - channel_count_val = self.db.prefix_db.channel_count.get(channel_hash) - channel_count = 0 if not channel_count_val else channel_count_val.count - if channel_count_val is not None: - self.db.prefix_db.channel_count.stage_delete((channel_hash,), (channel_count,)) - self.db.prefix_db.channel_count.stage_put((channel_hash,), (channel_count + count,)) - - self.touched_claim_hashes.update( - {k for k in self.pending_reposted if k not in self.removed_claim_hashes} - ) - self.touched_claim_hashes.update( - {k for k, v in self.pending_channel_counts.items() if v != 0 and k not in self.removed_claim_hashes} - ) - self.touched_claims_to_send_es.update(self.touched_claim_hashes) - self.touched_claims_to_send_es.difference_update(self.removed_claim_hashes) - self.removed_claims_to_send_es.update(self.removed_claim_hashes) - - def advance_block(self, block): - height = self.height + 1 - # print("advance ", height) - # Use local vars for speed in the loops - tx_count = self.tx_count - spend_utxo = self.spend_utxo - add_utxo = self.add_utxo - spend_claim_or_support_txo = self._spend_claim_or_support_txo - add_claim_or_support = self._add_claim_or_support - txs: List[Tuple[Tx, bytes]] = block.transactions - - self.db.prefix_db.block_hash.stage_put(key_args=(height,), value_args=(self.coin.header_hash(block.header),)) - self.db.prefix_db.header.stage_put(key_args=(height,), value_args=(block.header,)) - self.db.prefix_db.block_txs.stage_put(key_args=(height,), value_args=([tx_hash for tx, tx_hash in txs],)) - - for tx, tx_hash in txs: - spent_claims = {} - txos = Transaction(tx.raw).outputs - - # clean up mempool, delete txs that were already in mempool/staged to be added - # leave txs in mempool that werent in the block - mempool_tx = self.db.prefix_db.mempool_tx.get_pending(tx_hash) - if mempool_tx: - self.db.prefix_db.mempool_tx.stage_delete((tx_hash,), mempool_tx) - - self.db.prefix_db.tx.stage_put(key_args=(tx_hash,), value_args=(tx.raw,)) - self.db.prefix_db.tx_num.stage_put(key_args=(tx_hash,), value_args=(tx_count,)) - self.db.prefix_db.tx_hash.stage_put(key_args=(tx_count,), value_args=(tx_hash,)) - - # Spend the inputs - for txin in tx.inputs: - if txin.is_generation(): - continue - # spend utxo for address histories - hashX = spend_utxo(txin.prev_hash, txin.prev_idx) - if hashX: - if tx_count not in self.hashXs_by_tx[hashX]: - self.hashXs_by_tx[hashX].append(tx_count) - # spend claim/support txo - spend_claim_or_support_txo(height, txin, spent_claims) - - # Add the new UTXOs - for nout, txout in enumerate(tx.outputs): - # Get the hashX. Ignore unspendable outputs - hashX = add_utxo(tx_hash, tx_count, nout, txout) - if hashX: - # self._set_hashX_cache(hashX) - if tx_count not in self.hashXs_by_tx[hashX]: - self.hashXs_by_tx[hashX].append(tx_count) - # add claim/support txo - add_claim_or_support( - height, tx_hash, tx_count, nout, txos[nout], spent_claims - ) - - # Handle abandoned claims - abandoned_channels = {} - # abandon the channels last to handle abandoned signed claims in the same tx, - # see test_abandon_channel_and_claims_in_same_tx - for abandoned_claim_hash, (tx_num, nout, normalized_name) in spent_claims.items(): - if normalized_name.startswith('@'): - abandoned_channels[abandoned_claim_hash] = (tx_num, nout, normalized_name) - else: - # print(f"\tabandon {normalized_name} {abandoned_claim_hash.hex()} {tx_num} {nout}") - self._abandon_claim(abandoned_claim_hash, tx_num, nout, normalized_name) - - for abandoned_claim_hash, (tx_num, nout, normalized_name) in abandoned_channels.items(): - # print(f"\tabandon {normalized_name} {abandoned_claim_hash.hex()} {tx_num} {nout}") - self._abandon_claim(abandoned_claim_hash, tx_num, nout, normalized_name) - self.pending_transactions[tx_count] = tx_hash - self.pending_transaction_num_mapping[tx_hash] = tx_count - if self.env.cache_all_tx_hashes: - self.db.total_transactions.append(tx_hash) - self.db.tx_num_mapping[tx_hash] = tx_count - tx_count += 1 - - # handle expired claims - self._expire_claims(height) - - # activate claims and process takeovers - self._get_takeover_ops(height) - - # update effective amount and update sets of touched and deleted claims - self._get_cumulative_update_ops(height) - - self.db.prefix_db.touched_hashX.stage_put((height,), (list(sorted(self.touched_hashXs)),)) - - self.db.prefix_db.tx_count.stage_put(key_args=(height,), value_args=(tx_count,)) - - for hashX, new_history in self.hashXs_by_tx.items(): - if not new_history: - continue - self.db.prefix_db.hashX_history.stage_put(key_args=(hashX, height), value_args=(new_history,)) - - self.tx_count = tx_count - self.db.tx_counts.append(self.tx_count) - - cached_max_reorg_depth = self.daemon.cached_height() - self.env.reorg_limit - - # if height >= cached_max_reorg_depth: - self.db.prefix_db.touched_or_deleted.stage_put( - key_args=(height,), value_args=(self.touched_claim_hashes, self.removed_claim_hashes) - ) - - self.height = height - self.db.headers.append(block.header) - self.tip = self.coin.header_hash(block.header) - - self.db.fs_height = self.height - self.db.fs_tx_count = self.tx_count - self.db.hist_flush_count += 1 - self.db.hist_unflushed_count = 0 - self.db.utxo_flush_count = self.db.hist_flush_count - self.db.db_height = self.height - self.db.db_tx_count = self.tx_count - self.db.db_tip = self.tip - self.db.last_flush_tx_count = self.db.fs_tx_count - now = time.time() - self.db.wall_time += now - self.db.last_flush - self.db.last_flush = now - self.db.write_db_state() - - # flush the changes - save_undo = (self.daemon.cached_height() - self.height) <= self.env.reorg_limit - - if save_undo: - self.db.prefix_db.commit(self.height, self.tip) - else: - self.db.prefix_db.unsafe_commit() - self.clear_after_advance_or_reorg() - self.db.assert_db_state() - - def clear_after_advance_or_reorg(self): - self.txo_to_claim.clear() - self.claim_hash_to_txo.clear() - self.support_txos_by_claim.clear() - self.support_txo_to_claim.clear() - self.removed_support_txos_by_name_by_claim.clear() - self.abandoned_claims.clear() - self.removed_active_support_amount_by_claim.clear() - self.activated_support_amount_by_claim.clear() - self.activated_claim_amount_by_name_and_hash.clear() - self.activation_by_claim_by_name.clear() - self.possible_future_claim_amount_by_name_and_hash.clear() - self.possible_future_support_amounts_by_claim_hash.clear() - self.possible_future_support_txos_by_claim_hash.clear() - self.pending_channels.clear() - self.amount_cache.clear() - self.signatures_changed.clear() - self.expired_claim_hashes.clear() - self.doesnt_have_valid_signature.clear() - self.claim_channels.clear() - self.utxo_cache.clear() - self.hashXs_by_tx.clear() - self.removed_claim_hashes.clear() - self.touched_claim_hashes.clear() - self.pending_reposted.clear() - self.pending_channel_counts.clear() - self.updated_claims.clear() - self.taken_over_names.clear() - self.pending_transaction_num_mapping.clear() - self.pending_transactions.clear() - self.pending_support_amount_change.clear() - self.touched_hashXs.clear() - - def backup_block(self): - assert len(self.db.prefix_db._op_stack) == 0 - touched_and_deleted = self.db.prefix_db.touched_or_deleted.get(self.height) - self.touched_claims_to_send_es.update(touched_and_deleted.touched_claims) - self.removed_claims_to_send_es.difference_update(touched_and_deleted.touched_claims) - self.removed_claims_to_send_es.update(touched_and_deleted.deleted_claims) - - # self.db.assert_flushed(self.flush_data()) - self.logger.info("backup block %i", self.height) - # Check and update self.tip - - self.db.tx_counts.pop() - reverted_block_hash = self.coin.header_hash(self.db.headers.pop()) - self.tip = self.coin.header_hash(self.db.headers[-1]) - if self.env.cache_all_tx_hashes: - while len(self.db.total_transactions) > self.db.tx_counts[-1]: - self.db.tx_num_mapping.pop(self.db.total_transactions.pop()) - self.tx_count -= 1 - else: - self.tx_count = self.db.tx_counts[-1] - self.height -= 1 - - # self.touched can include other addresses which is - # harmless, but remove None. - self.touched_hashXs.discard(None) - - assert self.height < self.db.db_height - assert not self.db.hist_unflushed - - start_time = time.time() - tx_delta = self.tx_count - self.db.last_flush_tx_count - ### - self.db.fs_tx_count = self.tx_count - # Truncate header_mc: header count is 1 more than the height. - self.db.header_mc.truncate(self.height + 1) - ### - # Not certain this is needed, but it doesn't hurt - self.db.hist_flush_count += 1 - - while self.db.fs_height > self.height: - self.db.fs_height -= 1 - self.db.utxo_flush_count = self.db.hist_flush_count - self.db.db_height = self.height - self.db.db_tx_count = self.tx_count - self.db.db_tip = self.tip - # Flush state last as it reads the wall time. - now = time.time() - self.db.wall_time += now - self.db.last_flush - self.db.last_flush = now - self.db.last_flush_tx_count = self.db.fs_tx_count - - # rollback - self.db.prefix_db.rollback(self.height + 1, reverted_block_hash) - self.db.es_sync_height = self.height - self.db.write_db_state() - self.db.prefix_db.unsafe_commit() - - self.clear_after_advance_or_reorg() - self.db.assert_db_state() - - elapsed = self.db.last_flush - start_time - self.logger.warning(f'backup flush #{self.db.hist_flush_count:,d} took {elapsed:.1f}s. ' - f'Height {self.height:,d} txs: {self.tx_count:,d} ({tx_delta:+,d})') - - def add_utxo(self, tx_hash: bytes, tx_num: int, nout: int, txout: 'TxOutput') -> Optional[bytes]: - hashX = self.coin.hashX_from_script(txout.pk_script) - if hashX: - self.touched_hashXs.add(hashX) - self.utxo_cache[(tx_hash, nout)] = (hashX, txout.value) - self.db.prefix_db.utxo.stage_put((hashX, tx_num, nout), (txout.value,)) - self.db.prefix_db.hashX_utxo.stage_put((tx_hash[:4], tx_num, nout), (hashX,)) - return hashX - - def get_pending_tx_num(self, tx_hash: bytes) -> int: - if tx_hash in self.pending_transaction_num_mapping: - return self.pending_transaction_num_mapping[tx_hash] - else: - return self.db.get_tx_num(tx_hash) - - def spend_utxo(self, tx_hash: bytes, nout: int): - hashX, amount = self.utxo_cache.pop((tx_hash, nout), (None, None)) - txin_num = self.get_pending_tx_num(tx_hash) - if not hashX: - hashX_value = self.db.prefix_db.hashX_utxo.get(tx_hash[:4], txin_num, nout) - if not hashX_value: - return - hashX = hashX_value.hashX - utxo_value = self.db.prefix_db.utxo.get(hashX, txin_num, nout) - if not utxo_value: - self.logger.warning( - "%s:%s is not found in UTXO db for %s", hash_to_hex_str(tx_hash), nout, hash_to_hex_str(hashX) - ) - raise ChainError( - f"{hash_to_hex_str(tx_hash)}:{nout} is not found in UTXO db for {hash_to_hex_str(hashX)}" - ) - self.touched_hashXs.add(hashX) - self.db.prefix_db.hashX_utxo.stage_delete((tx_hash[:4], txin_num, nout), hashX_value) - self.db.prefix_db.utxo.stage_delete((hashX, txin_num, nout), utxo_value) - return hashX - elif amount is not None: - self.db.prefix_db.hashX_utxo.stage_delete((tx_hash[:4], txin_num, nout), (hashX,)) - self.db.prefix_db.utxo.stage_delete((hashX, txin_num, nout), (amount,)) - self.touched_hashXs.add(hashX) - return hashX - - async def process_blocks_and_mempool_forever(self): - """Loop forever processing blocks as they arrive.""" - try: - while not self._stopping: - if self.height == self.daemon.cached_height(): - if not self._caught_up_event.is_set(): - await self._first_caught_up() - self._caught_up_event.set() - try: - await asyncio.wait_for(self.blocks_event.wait(), self.wait_for_blocks_duration) - except asyncio.TimeoutError: - pass - self.blocks_event.clear() - blocks = self.prefetcher.get_prefetched_blocks() - if self._stopping: - break - if not blocks: - try: - await self.refresh_mempool() - except asyncio.CancelledError: - raise - except Exception: - self.logger.exception("error while updating mempool txs") - raise - else: - try: - await self.check_and_advance_blocks(blocks) - except asyncio.CancelledError: - raise - except Exception: - self.logger.exception("error while processing txs") - raise - finally: - self._ready_to_stop.set() - - async def _first_caught_up(self): - self.logger.info(f'caught up to height {self.height}') - # Flush everything but with first_sync->False state. - first_sync = self.db.first_sync - self.db.first_sync = False - - def flush(): - assert len(self.db.prefix_db._op_stack) == 0 - self.db.write_db_state() - self.db.prefix_db.unsafe_commit() - self.db.assert_db_state() - - await self.run_in_thread_with_lock(flush) - - if first_sync: - self.logger.info(f'{lbry.__version__} synced to ' - f'height {self.height:,d}, halting here.') - self.shutdown_event.set() - - async def open(self): - self.db.open_db() - self.height = self.db.db_height - self.tip = self.db.db_tip - self.tx_count = self.db.db_tx_count - await self.db.initialize_caches() - - async def fetch_and_process_blocks(self, caught_up_event): - """Fetch, process and index blocks from the daemon. - - Sets caught_up_event when first caught up. Flushes to disk - and shuts down cleanly if cancelled. - - This is mainly because if, during initial sync ElectrumX is - asked to shut down when a large number of blocks have been - processed but not written to disk, it should write those to - disk before exiting, as otherwise a significant amount of work - could be lost. - """ - - await self.open() - - self._caught_up_event = caught_up_event - try: - await asyncio.wait([ - self.prefetcher.main_loop(self.height), - self.process_blocks_and_mempool_forever() - ]) - except asyncio.CancelledError: - raise - except: - self.logger.exception("Block processing failed!") - raise - finally: - # Shut down block processing - self.logger.info('closing the DB for a clean shutdown...') - self._chain_executor.shutdown(wait=True) - self.db.close() - - async def start(self): - self._stopping = False - env = self.env - min_str, max_str = env.coin.SESSIONCLS.protocol_min_max_strings() - self.logger.info(f'software version: {lbry.__version__}') - self.logger.info(f'supported protocol versions: {min_str}-{max_str}') - self.logger.info(f'event loop policy: {env.loop_policy}') - self.logger.info(f'reorg limit is {env.reorg_limit:,d} blocks') - - await self.daemon.height() - - def _start_cancellable(run, *args): - _flag = asyncio.Event() - self.cancellable_tasks.append(asyncio.ensure_future(run(*args, _flag))) - return _flag.wait() - - await _start_cancellable(self.fetch_and_process_blocks) - - async def stop(self): - self._stopping = True - await self._ready_to_stop.wait() - for task in reversed(self.cancellable_tasks): - task.cancel() - await asyncio.wait(self.cancellable_tasks) - self.shutdown_event.set() - await self.daemon.close() - - def run(self): - loop = asyncio.get_event_loop() - loop.set_default_executor(self._chain_executor) - - def __exit(): - raise SystemExit() - try: - loop.add_signal_handler(signal.SIGINT, __exit) - loop.add_signal_handler(signal.SIGTERM, __exit) - loop.run_until_complete(self.start()) - loop.run_until_complete(self.shutdown_event.wait()) - except (SystemExit, KeyboardInterrupt): - pass - finally: - loop.run_until_complete(self.stop()) diff --git a/lbry/wallet/server/chain_reader.py b/lbry/wallet/server/chain_reader.py deleted file mode 100644 index a08599e5c..000000000 --- a/lbry/wallet/server/chain_reader.py +++ /dev/null @@ -1,285 +0,0 @@ -import signal -import logging -import asyncio -import typing -from concurrent.futures.thread import ThreadPoolExecutor -from prometheus_client import Gauge, Histogram -import lbry -from lbry.wallet.server.mempool import MemPool -from lbry.wallet.server.db.prefixes import DBState -from lbry.wallet.udp import StatusServer -from lbry.wallet.server.db.db import HubDB -from lbry.wallet.server.db.elasticsearch.notifier import ElasticNotifierClientProtocol -from lbry.wallet.server.session import LBRYSessionManager -from lbry.prometheus import PrometheusServer - - -HISTOGRAM_BUCKETS = ( - .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf') -) - - -class BlockchainReader: - block_count_metric = Gauge( - "block_count", "Number of processed blocks", namespace="blockchain_reader" - ) - block_update_time_metric = Histogram( - "block_time", "Block update times", namespace="blockchain_reader", buckets=HISTOGRAM_BUCKETS - ) - reorg_count_metric = Gauge( - "reorg_count", "Number of reorgs", namespace="blockchain_reader" - ) - - def __init__(self, env, secondary_name: str, thread_workers: int = 1, thread_prefix: str = 'blockchain-reader'): - self.env = env - self.log = logging.getLogger(__name__).getChild(self.__class__.__name__) - self.shutdown_event = asyncio.Event() - self.cancellable_tasks = [] - self._thread_workers = thread_workers - self._thread_prefix = thread_prefix - self._executor = ThreadPoolExecutor(thread_workers, thread_name_prefix=thread_prefix) - self.db = HubDB( - env.coin, env.db_dir, env.cache_MB, env.reorg_limit, env.cache_all_claim_txos, env.cache_all_tx_hashes, - secondary_name=secondary_name, max_open_files=-1, blocking_channel_ids=env.blocking_channel_ids, - filtering_channel_ids=env.filtering_channel_ids, executor=self._executor - ) - self.last_state: typing.Optional[DBState] = None - self._refresh_interval = 0.1 - self._lock = asyncio.Lock() - - def _detect_changes(self): - try: - self.db.prefix_db.try_catch_up_with_primary() - except: - self.log.exception('failed to update secondary db') - raise - state = self.db.prefix_db.db_state.get() - if not state or state.height <= 0: - return - if self.last_state and self.last_state.height > state.height: - self.log.warning("reorg detected, waiting until the writer has flushed the new blocks to advance") - return - last_height = 0 if not self.last_state else self.last_state.height - rewound = False - if self.last_state: - while True: - if self.db.headers[-1] == self.db.prefix_db.header.get(last_height, deserialize_value=False): - self.log.debug("connects to block %i", last_height) - break - else: - self.log.warning("disconnect block %i", last_height) - self.unwind() - rewound = True - last_height -= 1 - if rewound: - self.reorg_count_metric.inc() - self.db.read_db_state() - if not self.last_state or last_height < state.height: - for height in range(last_height + 1, state.height + 1): - self.log.info("advancing to %i", height) - self.advance(height) - self.clear_caches() - self.last_state = state - self.block_count_metric.set(self.last_state.height) - self.db.blocked_streams, self.db.blocked_channels = self.db.get_streams_and_channels_reposted_by_channel_hashes( - self.db.blocking_channel_hashes - ) - self.db.filtered_streams, self.db.filtered_channels = self.db.get_streams_and_channels_reposted_by_channel_hashes( - self.db.filtering_channel_hashes - ) - - async def poll_for_changes(self): - await asyncio.get_event_loop().run_in_executor(self._executor, self._detect_changes) - - async def refresh_blocks_forever(self, synchronized: asyncio.Event): - while True: - try: - async with self._lock: - await self.poll_for_changes() - except asyncio.CancelledError: - raise - except: - self.log.exception("blockchain reader main loop encountered an unexpected error") - raise - await asyncio.sleep(self._refresh_interval) - synchronized.set() - - def clear_caches(self): - pass - - def advance(self, height: int): - tx_count = self.db.prefix_db.tx_count.get(height).tx_count - assert tx_count not in self.db.tx_counts, f'boom {tx_count} in {len(self.db.tx_counts)} tx counts' - assert len(self.db.tx_counts) == height, f"{len(self.db.tx_counts)} != {height}" - self.db.tx_counts.append(tx_count) - self.db.headers.append(self.db.prefix_db.header.get(height, deserialize_value=False)) - - def unwind(self): - self.db.tx_counts.pop() - self.db.headers.pop() - - async def start(self): - if not self._executor: - self._executor = ThreadPoolExecutor(self._thread_workers, thread_name_prefix=self._thread_prefix) - self.db._executor = self._executor - - -class BlockchainReaderServer(BlockchainReader): - block_count_metric = Gauge( - "block_count", "Number of processed blocks", namespace="wallet_server" - ) - block_update_time_metric = Histogram( - "block_time", "Block update times", namespace="wallet_server", buckets=HISTOGRAM_BUCKETS - ) - reorg_count_metric = Gauge( - "reorg_count", "Number of reorgs", namespace="wallet_server" - ) - - def __init__(self, env): - super().__init__(env, 'lbry-reader', thread_workers=max(1, env.max_query_workers), thread_prefix='hub-worker') - self.history_cache = {} - self.resolve_outputs_cache = {} - self.resolve_cache = {} - self.notifications_to_send = [] - self.mempool_notifications = set() - self.status_server = StatusServer() - self.daemon = env.coin.DAEMON(env.coin, env.daemon_url) # only needed for broadcasting txs - self.prometheus_server: typing.Optional[PrometheusServer] = None - self.mempool = MemPool(self.env.coin, self.db) - self.session_manager = LBRYSessionManager( - env, self.db, self.mempool, self.history_cache, self.resolve_cache, - self.resolve_outputs_cache, self.daemon, - self.shutdown_event, - on_available_callback=self.status_server.set_available, - on_unavailable_callback=self.status_server.set_unavailable - ) - self.mempool.session_manager = self.session_manager - self.es_notifications = asyncio.Queue() - self.es_notification_client = ElasticNotifierClientProtocol(self.es_notifications) - self.synchronized = asyncio.Event() - self._es_height = None - self._es_block_hash = None - - def clear_caches(self): - self.history_cache.clear() - self.resolve_outputs_cache.clear() - self.resolve_cache.clear() - # self.clear_search_cache() - # self.mempool.notified_mempool_txs.clear() - - def clear_search_cache(self): - self.session_manager.search_index.clear_caches() - - def advance(self, height: int): - super().advance(height) - touched_hashXs = self.db.prefix_db.touched_hashX.get(height).touched_hashXs - self.notifications_to_send.append((set(touched_hashXs), height)) - - def _detect_changes(self): - super()._detect_changes() - self.mempool_notifications.update(self.mempool.refresh()) - - async def poll_for_changes(self): - await super().poll_for_changes() - if self.db.fs_height <= 0: - return - self.status_server.set_height(self.db.fs_height, self.db.db_tip) - if self.notifications_to_send: - for (touched, height) in self.notifications_to_send: - await self.mempool.on_block(touched, height) - self.log.info("reader advanced to %i", height) - if self._es_height == self.db.db_height: - self.synchronized.set() - if self.mempool_notifications: - await self.mempool.on_mempool( - set(self.mempool.touched_hashXs), self.mempool_notifications, self.db.db_height - ) - self.mempool_notifications.clear() - self.notifications_to_send.clear() - - async def receive_es_notifications(self, synchronized: asyncio.Event): - await asyncio.get_event_loop().create_connection( - lambda: self.es_notification_client, '127.0.0.1', self.env.elastic_notifier_port - ) - synchronized.set() - try: - while True: - self._es_height, self._es_block_hash = await self.es_notifications.get() - self.clear_search_cache() - if self.last_state and self._es_block_hash == self.last_state.tip: - self.synchronized.set() - self.log.info("es and reader are in sync at block %i", self.last_state.height) - else: - self.log.info("es and reader are not yet in sync (block %s vs %s)", self._es_height, - self.db.db_height) - finally: - self.es_notification_client.close() - - async def start(self): - await super().start() - env = self.env - min_str, max_str = env.coin.SESSIONCLS.protocol_min_max_strings() - self.log.info(f'software version: {lbry.__version__}') - self.log.info(f'supported protocol versions: {min_str}-{max_str}') - self.log.info(f'event loop policy: {env.loop_policy}') - self.log.info(f'reorg limit is {env.reorg_limit:,d} blocks') - await self.daemon.height() - - def _start_cancellable(run, *args): - _flag = asyncio.Event() - self.cancellable_tasks.append(asyncio.ensure_future(run(*args, _flag))) - return _flag.wait() - - self.db.open_db() - await self.db.initialize_caches() - - self.last_state = self.db.read_db_state() - - await self.start_prometheus() - if self.env.udp_port and int(self.env.udp_port): - await self.status_server.start( - 0, bytes.fromhex(self.env.coin.GENESIS_HASH)[::-1], self.env.country, - self.env.host, self.env.udp_port, self.env.allow_lan_udp - ) - await _start_cancellable(self.receive_es_notifications) - await _start_cancellable(self.refresh_blocks_forever) - await self.session_manager.search_index.start() - await _start_cancellable(self.session_manager.serve, self.mempool) - - async def stop(self): - await self.status_server.stop() - async with self._lock: - while self.cancellable_tasks: - t = self.cancellable_tasks.pop() - if not t.done(): - t.cancel() - await self.session_manager.search_index.stop() - self.db.close() - if self.prometheus_server: - await self.prometheus_server.stop() - self.prometheus_server = None - await self.daemon.close() - self._executor.shutdown(wait=True) - self._executor = None - self.shutdown_event.set() - - def run(self): - loop = asyncio.get_event_loop() - loop.set_default_executor(self._executor) - - def __exit(): - raise SystemExit() - try: - loop.add_signal_handler(signal.SIGINT, __exit) - loop.add_signal_handler(signal.SIGTERM, __exit) - loop.run_until_complete(self.start()) - loop.run_until_complete(self.shutdown_event.wait()) - except (SystemExit, KeyboardInterrupt): - pass - finally: - loop.run_until_complete(self.stop()) - - async def start_prometheus(self): - if not self.prometheus_server and self.env.prometheus_port: - self.prometheus_server = PrometheusServer() - await self.prometheus_server.start("0.0.0.0", self.env.prometheus_port) diff --git a/lbry/wallet/server/cli.py b/lbry/wallet/server/cli.py deleted file mode 100644 index f415cb3b4..000000000 --- a/lbry/wallet/server/cli.py +++ /dev/null @@ -1,64 +0,0 @@ -import logging -import traceback -import argparse -from lbry.wallet.server.env import Env -from lbry.wallet.server.block_processor import BlockProcessor -from lbry.wallet.server.chain_reader import BlockchainReaderServer -from lbry.wallet.server.db.elasticsearch.sync import ElasticWriter - - -def get_arg_parser(name): - parser = argparse.ArgumentParser( - prog=name - ) - Env.contribute_to_arg_parser(parser) - return parser - - -def setup_logging(): - logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-4s %(name)s:%(lineno)d: %(message)s") - logging.getLogger('aiohttp').setLevel(logging.WARNING) - logging.getLogger('elasticsearch').setLevel(logging.WARNING) - - -def run_writer_forever(): - setup_logging() - args = get_arg_parser('lbry-hub-writer').parse_args() - try: - block_processor = BlockProcessor(Env.from_arg_parser(args)) - block_processor.run() - except Exception: - traceback.print_exc() - logging.critical('block processor terminated abnormally') - else: - logging.info('block processor terminated normally') - - -def run_server_forever(): - setup_logging() - args = get_arg_parser('lbry-hub-server').parse_args() - - try: - server = BlockchainReaderServer(Env.from_arg_parser(args)) - server.run() - except Exception: - traceback.print_exc() - logging.critical('server terminated abnormally') - else: - logging.info('server terminated normally') - - -def run_es_sync_forever(): - setup_logging() - parser = get_arg_parser('lbry-hub-elastic-sync') - parser.add_argument('--reindex', type=bool, default=False) - args = parser.parse_args() - - try: - server = ElasticWriter(Env.from_arg_parser(args)) - server.run(args.reindex) - except Exception: - traceback.print_exc() - logging.critical('es writer terminated abnormally') - else: - logging.info('es writer terminated normally') diff --git a/lbry/wallet/server/coin.py b/lbry/wallet/server/coin.py deleted file mode 100644 index afe2a9423..000000000 --- a/lbry/wallet/server/coin.py +++ /dev/null @@ -1,381 +0,0 @@ -import re -import struct -from typing import List -from hashlib import sha256 -from decimal import Decimal -from collections import namedtuple - -import lbry.wallet.server.tx as lib_tx -from lbry.wallet.script import OutputScript, OP_CLAIM_NAME, OP_UPDATE_CLAIM, OP_SUPPORT_CLAIM -from lbry.wallet.server.tx import DeserializerSegWit -from lbry.wallet.server.util import cachedproperty, subclasses -from lbry.wallet.server.hash import Base58, hash160, double_sha256, hash_to_hex_str, HASHX_LEN -from lbry.wallet.server.daemon import Daemon, LBCDaemon -from lbry.wallet.server.script import ScriptPubKey, OpCodes -from lbry.wallet.server.session import LBRYElectrumX, LBRYSessionManager - - -Block = namedtuple("Block", "raw header transactions") -OP_RETURN = OpCodes.OP_RETURN - - -class CoinError(Exception): - """Exception raised for coin-related errors.""" - - -class Coin: - """Base class of coin hierarchy.""" - - REORG_LIMIT = 200 - # Not sure if these are coin-specific - RPC_URL_REGEX = re.compile('.+@(\\[[0-9a-fA-F:]+\\]|[^:]+)(:[0-9]+)?') - VALUE_PER_COIN = 100000000 - CHUNK_SIZE = 2016 - BASIC_HEADER_SIZE = 80 - STATIC_BLOCK_HEADERS = True - SESSIONCLS = LBRYElectrumX - DESERIALIZER = lib_tx.Deserializer - DAEMON = Daemon - SESSION_MANAGER = LBRYSessionManager - HEADER_VALUES = [ - 'version', 'prev_block_hash', 'merkle_root', 'timestamp', 'bits', 'nonce' - ] - HEADER_UNPACK = struct.Struct('< I 32s 32s I I I').unpack_from - MEMPOOL_HISTOGRAM_REFRESH_SECS = 500 - XPUB_VERBYTES = bytes('????', 'utf-8') - XPRV_VERBYTES = bytes('????', 'utf-8') - ENCODE_CHECK = Base58.encode_check - DECODE_CHECK = Base58.decode_check - # Peer discovery - PEER_DEFAULT_PORTS = {'t': '50001', 's': '50002'} - PEERS: List[str] = [] - - @classmethod - def lookup_coin_class(cls, name, net): - """Return a coin class given name and network. - - Raise an exception if unrecognised.""" - req_attrs = ['TX_COUNT', 'TX_COUNT_HEIGHT', 'TX_PER_BLOCK'] - for coin in subclasses(Coin): - if (coin.NAME.lower() == name.lower() and - coin.NET.lower() == net.lower()): - coin_req_attrs = req_attrs.copy() - missing = [attr for attr in coin_req_attrs - if not hasattr(coin, attr)] - if missing: - raise CoinError(f'coin {name} missing {missing} attributes') - return coin - raise CoinError(f'unknown coin {name} and network {net} combination') - - @classmethod - def sanitize_url(cls, url): - # Remove surrounding ws and trailing /s - url = url.strip().rstrip('/') - match = cls.RPC_URL_REGEX.match(url) - if not match: - raise CoinError(f'invalid daemon URL: "{url}"') - if match.groups()[1] is None: - url += f':{cls.RPC_PORT:d}' - if not url.startswith('http://') and not url.startswith('https://'): - url = 'http://' + url - return url + '/' - - @classmethod - def genesis_block(cls, block): - """Check the Genesis block is the right one for this coin. - - Return the block less its unspendable coinbase. - """ - header = cls.block_header(block, 0) - header_hex_hash = hash_to_hex_str(cls.header_hash(header)) - if header_hex_hash != cls.GENESIS_HASH: - raise CoinError(f'genesis block has hash {header_hex_hash} expected {cls.GENESIS_HASH}') - - return header + bytes(1) - - @classmethod - def hashX_from_script(cls, script): - """Returns a hashX from a script, or None if the script is provably - unspendable so the output can be dropped. - """ - if script and script[0] == OP_RETURN: - return None - return sha256(script).digest()[:HASHX_LEN] - - @staticmethod - def lookup_xverbytes(verbytes): - """Return a (is_xpub, coin_class) pair given xpub/xprv verbytes.""" - # Order means BTC testnet will override NMC testnet - for coin in subclasses(Coin): - if verbytes == coin.XPUB_VERBYTES: - return True, coin - if verbytes == coin.XPRV_VERBYTES: - return False, coin - raise CoinError('version bytes unrecognised') - - @classmethod - def address_to_hashX(cls, address): - """Return a hashX given a coin address.""" - return cls.hashX_from_script(cls.pay_to_address_script(address)) - - @classmethod - def P2PKH_address_from_hash160(cls, hash160): - """Return a P2PKH address given a public key.""" - assert len(hash160) == 20 - return cls.ENCODE_CHECK(cls.P2PKH_VERBYTE + hash160) - - @classmethod - def P2PKH_address_from_pubkey(cls, pubkey): - """Return a coin address given a public key.""" - return cls.P2PKH_address_from_hash160(hash160(pubkey)) - - @classmethod - def P2SH_address_from_hash160(cls, hash160): - """Return a coin address given a hash160.""" - assert len(hash160) == 20 - return cls.ENCODE_CHECK(cls.P2SH_VERBYTES[0] + hash160) - - @classmethod - def hash160_to_P2PKH_script(cls, hash160): - return ScriptPubKey.P2PKH_script(hash160) - - @classmethod - def hash160_to_P2PKH_hashX(cls, hash160): - return cls.hashX_from_script(cls.hash160_to_P2PKH_script(hash160)) - - @classmethod - def pay_to_address_script(cls, address): - """Return a pubkey script that pays to a pubkey hash. - - Pass the address (either P2PKH or P2SH) in base58 form. - """ - raw = cls.DECODE_CHECK(address) - - # Require version byte(s) plus hash160. - verbyte = -1 - verlen = len(raw) - 20 - if verlen > 0: - verbyte, hash160 = raw[:verlen], raw[verlen:] - - if verbyte == cls.P2PKH_VERBYTE: - return cls.hash160_to_P2PKH_script(hash160) - if verbyte in cls.P2SH_VERBYTES: - return ScriptPubKey.P2SH_script(hash160) - - raise CoinError(f'invalid address: {address}') - - @classmethod - def privkey_WIF(cls, privkey_bytes, compressed): - """Return the private key encoded in Wallet Import Format.""" - payload = bytearray(cls.WIF_BYTE) + privkey_bytes - if compressed: - payload.append(0x01) - return cls.ENCODE_CHECK(payload) - - @classmethod - def header_hash(cls, header): - """Given a header return hash""" - return double_sha256(header) - - @classmethod - def header_prevhash(cls, header): - """Given a header return previous hash""" - return header[4:36] - - @classmethod - def static_header_offset(cls, height): - """Given a header height return its offset in the headers file. - - If header sizes change at some point, this is the only code - that needs updating.""" - assert cls.STATIC_BLOCK_HEADERS - return height * cls.BASIC_HEADER_SIZE - - @classmethod - def static_header_len(cls, height): - """Given a header height return its length.""" - return (cls.static_header_offset(height + 1) - - cls.static_header_offset(height)) - - @classmethod - def block_header(cls, block, height): - """Returns the block header given a block and its height.""" - return block[:cls.static_header_len(height)] - - @classmethod - def block(cls, raw_block, height): - """Return a Block namedtuple given a raw block and its height.""" - header = cls.block_header(raw_block, height) - txs = cls.DESERIALIZER(raw_block, start=len(header)).read_tx_block() - return Block(raw_block, header, txs) - - @classmethod - def transaction(cls, raw_tx: bytes): - """Return a Block namedtuple given a raw block and its height.""" - return cls.DESERIALIZER(raw_tx).read_tx() - - @classmethod - def decimal_value(cls, value): - """Return the number of standard coin units as a Decimal given a - quantity of smallest units. - - For example 1 BTC is returned for 100 million satoshis. - """ - return Decimal(value) / cls.VALUE_PER_COIN - - @classmethod - def electrum_header(cls, header, height): - h = dict(zip(cls.HEADER_VALUES, cls.HEADER_UNPACK(header))) - # Add the height that is not present in the header itself - h['block_height'] = height - # Convert bytes to str - h['prev_block_hash'] = hash_to_hex_str(h['prev_block_hash']) - h['merkle_root'] = hash_to_hex_str(h['merkle_root']) - return h - - -class LBC(Coin): - DAEMON = LBCDaemon - SESSIONCLS = LBRYElectrumX - SESSION_MANAGER = LBRYSessionManager - DESERIALIZER = DeserializerSegWit - NAME = "LBRY" - SHORTNAME = "LBC" - NET = "mainnet" - BASIC_HEADER_SIZE = 112 - CHUNK_SIZE = 96 - XPUB_VERBYTES = bytes.fromhex("0488b21e") - XPRV_VERBYTES = bytes.fromhex("0488ade4") - P2PKH_VERBYTE = bytes.fromhex("55") - P2SH_VERBYTES = bytes.fromhex("7A") - WIF_BYTE = bytes.fromhex("1C") - GENESIS_HASH = ('9c89283ba0f3227f6c03b70216b9f665' - 'f0118d5e0fa729cedf4fb34d6a34f463') - TX_COUNT = 2716936 - TX_COUNT_HEIGHT = 329554 - TX_PER_BLOCK = 1 - RPC_PORT = 9245 - REORG_LIMIT = 200 - - nOriginalClaimExpirationTime = 262974 - nExtendedClaimExpirationTime = 2102400 - nExtendedClaimExpirationForkHeight = 400155 - nNormalizedNameForkHeight = 539940 # targeting 21 March 2019 - nMinTakeoverWorkaroundHeight = 496850 - nMaxTakeoverWorkaroundHeight = 658300 # targeting 30 Oct 2019 - nWitnessForkHeight = 680770 # targeting 11 Dec 2019 - nAllClaimsInMerkleForkHeight = 658310 # targeting 30 Oct 2019 - proportionalDelayFactor = 32 - maxTakeoverDelay = 4032 - - PEERS = [ - ] - - @classmethod - def genesis_block(cls, block): - '''Check the Genesis block is the right one for this coin. - - Return the block less its unspendable coinbase. - ''' - header = cls.block_header(block, 0) - header_hex_hash = hash_to_hex_str(cls.header_hash(header)) - if header_hex_hash != cls.GENESIS_HASH: - raise CoinError(f'genesis block has hash {header_hex_hash} expected {cls.GENESIS_HASH}') - - return block - - @classmethod - def electrum_header(cls, header, height): - version, = struct.unpack(' int: - if extended: - return last_updated_height + cls.nExtendedClaimExpirationTime - if last_updated_height < cls.nExtendedClaimExpirationForkHeight: - return last_updated_height + cls.nOriginalClaimExpirationTime - return last_updated_height + cls.nExtendedClaimExpirationTime - - @classmethod - def get_delay_for_name(cls, blocks_of_continuous_ownership: int) -> int: - return min(blocks_of_continuous_ownership // cls.proportionalDelayFactor, cls.maxTakeoverDelay) - - -class LBCRegTest(LBC): - NET = "regtest" - GENESIS_HASH = '6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556' - XPUB_VERBYTES = bytes.fromhex('043587cf') - XPRV_VERBYTES = bytes.fromhex('04358394') - P2PKH_VERBYTE = bytes.fromhex("6f") - P2SH_VERBYTES = bytes.fromhex("c4") - - nOriginalClaimExpirationTime = 500 - nExtendedClaimExpirationTime = 600 - nExtendedClaimExpirationForkHeight = 800 - nNormalizedNameForkHeight = 250 - nMinTakeoverWorkaroundHeight = -1 - nMaxTakeoverWorkaroundHeight = -1 - nWitnessForkHeight = 150 - nAllClaimsInMerkleForkHeight = 350 - - -class LBCTestNet(LBCRegTest): - NET = "testnet" - GENESIS_HASH = '9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463' diff --git a/lbry/wallet/server/daemon.py b/lbry/wallet/server/daemon.py deleted file mode 100644 index bcaadc35d..000000000 --- a/lbry/wallet/server/daemon.py +++ /dev/null @@ -1,345 +0,0 @@ -import asyncio -import itertools -import json -import time -from functools import wraps - -import aiohttp -from prometheus_client import Gauge, Histogram -from lbry.utils import LRUCacheWithMetrics -from lbry.wallet.rpc.jsonrpc import RPCError -from lbry.wallet.server.util import hex_to_bytes, class_logger -from lbry.wallet.rpc import JSONRPC - - -class DaemonError(Exception): - """Raised when the daemon returns an error in its results.""" - - -class WarmingUpError(Exception): - """Internal - when the daemon is warming up.""" - - -class WorkQueueFullError(Exception): - """Internal - when the daemon's work queue is full.""" - - -NAMESPACE = "wallet_server" - - -class Daemon: - """Handles connections to a daemon at the given URL.""" - - WARMING_UP = -28 - id_counter = itertools.count() - - lbrycrd_request_time_metric = Histogram( - "lbrycrd_request", "lbrycrd requests count", namespace=NAMESPACE, labelnames=("method",) - ) - lbrycrd_pending_count_metric = Gauge( - "lbrycrd_pending_count", "Number of lbrycrd rpcs that are in flight", namespace=NAMESPACE, - labelnames=("method",) - ) - - def __init__(self, coin, url, max_workqueue=10, init_retry=0.25, - max_retry=4.0): - self.coin = coin - self.logger = class_logger(__name__, self.__class__.__name__) - self.set_url(url) - # Limit concurrent RPC calls to this number. - # See DEFAULT_HTTP_WORKQUEUE in bitcoind, which is typically 16 - self.workqueue_semaphore = asyncio.Semaphore(value=max_workqueue) - self.init_retry = init_retry - self.max_retry = max_retry - self._height = None - self.available_rpcs = {} - self.connector = aiohttp.TCPConnector(ssl=False) - self._block_hash_cache = LRUCacheWithMetrics(100000) - self._block_cache = LRUCacheWithMetrics(2 ** 13, metric_name='block', namespace=NAMESPACE) - - async def close(self): - if self.connector: - await self.connector.close() - self.connector = None - - def set_url(self, url): - """Set the URLS to the given list, and switch to the first one.""" - urls = url.split(',') - urls = [self.coin.sanitize_url(url) for url in urls] - for n, url in enumerate(urls): - status = '' if n else ' (current)' - logged_url = self.logged_url(url) - self.logger.info(f'daemon #{n + 1} at {logged_url}{status}') - self.url_index = 0 - self.urls = urls - - def current_url(self): - """Returns the current daemon URL.""" - return self.urls[self.url_index] - - def logged_url(self, url=None): - """The host and port part, for logging.""" - url = url or self.current_url() - return url[url.rindex('@') + 1:] - - def failover(self): - """Call to fail-over to the next daemon URL. - - Returns False if there is only one, otherwise True. - """ - if len(self.urls) > 1: - self.url_index = (self.url_index + 1) % len(self.urls) - self.logger.info(f'failing over to {self.logged_url()}') - return True - return False - - def client_session(self): - """An aiohttp client session.""" - return aiohttp.ClientSession(connector=self.connector, connector_owner=False) - - async def _send_data(self, data): - if not self.connector: - raise asyncio.CancelledError('Tried to send request during shutdown.') - async with self.workqueue_semaphore: - async with self.client_session() as session: - async with session.post(self.current_url(), data=data) as resp: - kind = resp.headers.get('Content-Type', None) - if kind == 'application/json': - return await resp.json() - # bitcoind's HTTP protocol "handling" is a bad joke - text = await resp.text() - if 'Work queue depth exceeded' in text: - raise WorkQueueFullError - text = text.strip() or resp.reason - self.logger.error(text) - raise DaemonError(text) - - async def _send(self, payload, processor): - """Send a payload to be converted to JSON. - - Handles temporary connection issues. Daemon response errors - are raise through DaemonError. - """ - - def log_error(error): - nonlocal last_error_log, retry - now = time.time() - if now - last_error_log > 60: - last_error_log = now - self.logger.error(f'{error} Retrying occasionally...') - if retry == self.max_retry and self.failover(): - retry = 0 - - on_good_message = None - last_error_log = 0 - data = json.dumps(payload) - retry = self.init_retry - methods = tuple( - [payload['method']] if isinstance(payload, dict) else [request['method'] for request in payload] - ) - while True: - try: - for method in methods: - self.lbrycrd_pending_count_metric.labels(method=method).inc() - result = await self._send_data(data) - result = processor(result) - if on_good_message: - self.logger.info(on_good_message) - return result - except asyncio.TimeoutError: - log_error('timeout error.') - except aiohttp.ServerDisconnectedError: - log_error('disconnected.') - on_good_message = 'connection restored' - except aiohttp.ClientConnectionError: - log_error('connection problem - is your daemon running?') - on_good_message = 'connection restored' - except aiohttp.ClientError as e: - log_error(f'daemon error: {e}') - on_good_message = 'running normally' - except WarmingUpError: - log_error('starting up checking blocks.') - on_good_message = 'running normally' - except WorkQueueFullError: - log_error('work queue full.') - on_good_message = 'running normally' - finally: - for method in methods: - self.lbrycrd_pending_count_metric.labels(method=method).dec() - await asyncio.sleep(retry) - retry = max(min(self.max_retry, retry * 2), self.init_retry) - - async def _send_single(self, method, params=None): - """Send a single request to the daemon.""" - - start = time.perf_counter() - - def processor(result): - err = result['error'] - if not err: - return result['result'] - if err.get('code') == self.WARMING_UP: - raise WarmingUpError - raise DaemonError(err) - - payload = {'method': method, 'id': next(self.id_counter)} - if params: - payload['params'] = params - result = await self._send(payload, processor) - self.lbrycrd_request_time_metric.labels(method=method).observe(time.perf_counter() - start) - return result - - async def _send_vector(self, method, params_iterable, replace_errs=False): - """Send several requests of the same method. - - The result will be an array of the same length as params_iterable. - If replace_errs is true, any item with an error is returned as None, - otherwise an exception is raised.""" - - start = time.perf_counter() - - def processor(result): - errs = [item['error'] for item in result if item['error']] - if any(err.get('code') == self.WARMING_UP for err in errs): - raise WarmingUpError - if not errs or replace_errs: - return [item['result'] for item in result] - raise DaemonError(errs) - - payload = [{'method': method, 'params': p, 'id': next(self.id_counter)} - for p in params_iterable] - result = [] - if payload: - result = await self._send(payload, processor) - self.lbrycrd_request_time_metric.labels(method=method).observe(time.perf_counter() - start) - return result - - async def _is_rpc_available(self, method): - """Return whether given RPC method is available in the daemon. - - Results are cached and the daemon will generally not be queried with - the same method more than once.""" - available = self.available_rpcs.get(method) - if available is None: - available = True - try: - await self._send_single(method) - except DaemonError as e: - err = e.args[0] - error_code = err.get("code") - available = error_code != JSONRPC.METHOD_NOT_FOUND - self.available_rpcs[method] = available - return available - - async def block_hex_hashes(self, first, count): - """Return the hex hashes of count block starting at height first.""" - if first + count < (self.cached_height() or 0) - 200: - return await self._cached_block_hex_hashes(first, count) - params_iterable = ((h, ) for h in range(first, first + count)) - return await self._send_vector('getblockhash', params_iterable) - - async def _cached_block_hex_hashes(self, first, count): - """Return the hex hashes of count block starting at height first.""" - cached = self._block_hash_cache.get((first, count)) - if cached: - return cached - params_iterable = ((h, ) for h in range(first, first + count)) - self._block_hash_cache[(first, count)] = await self._send_vector('getblockhash', params_iterable) - return self._block_hash_cache[(first, count)] - - async def deserialised_block(self, hex_hash): - """Return the deserialised block with the given hex hash.""" - if hex_hash not in self._block_cache: - block = await self._send_single('getblock', (hex_hash, 1)) - self._block_cache[hex_hash] = block - return block - return self._block_cache[hex_hash] - - async def raw_blocks(self, hex_hashes): - """Return the raw binary blocks with the given hex hashes.""" - params_iterable = ((h, 0) for h in hex_hashes) - blocks = await self._send_vector('getblock', params_iterable) - # Convert hex string to bytes - return [hex_to_bytes(block) for block in blocks] - - async def mempool_hashes(self): - """Update our record of the daemon's mempool hashes.""" - return await self._send_single('getrawmempool') - - async def estimatefee(self, block_count): - """Return the fee estimate for the block count. Units are whole - currency units per KB, e.g. 0.00000995, or -1 if no estimate - is available. - """ - args = (block_count, ) - if await self._is_rpc_available('estimatesmartfee'): - estimate = await self._send_single('estimatesmartfee', args) - return estimate.get('feerate', -1) - return await self._send_single('estimatefee', args) - - async def getnetworkinfo(self): - """Return the result of the 'getnetworkinfo' RPC call.""" - return await self._send_single('getnetworkinfo') - - async def relayfee(self): - """The minimum fee a low-priority tx must pay in order to be accepted - to the daemon's memory pool.""" - network_info = await self.getnetworkinfo() - return network_info['relayfee'] - - async def getrawtransaction(self, hex_hash, verbose=False): - """Return the serialized raw transaction with the given hash.""" - # Cast to int because some coin daemons are old and require it - return await self._send_single('getrawtransaction', - (hex_hash, int(verbose))) - - async def getrawtransactions(self, hex_hashes, replace_errs=True): - """Return the serialized raw transactions with the given hashes. - - Replaces errors with None by default.""" - params_iterable = ((hex_hash, 0) for hex_hash in hex_hashes) - txs = await self._send_vector('getrawtransaction', params_iterable, - replace_errs=replace_errs) - # Convert hex strings to bytes - return [hex_to_bytes(tx) if tx else None for tx in txs] - - async def broadcast_transaction(self, raw_tx): - """Broadcast a transaction to the network.""" - return await self._send_single('sendrawtransaction', (raw_tx, )) - - async def height(self): - """Query the daemon for its current height.""" - self._height = await self._send_single('getblockcount') - return self._height - - def cached_height(self): - """Return the cached daemon height. - - If the daemon has not been queried yet this returns None.""" - return self._height - - -def handles_errors(decorated_function): - @wraps(decorated_function) - async def wrapper(*args, **kwargs): - try: - return await decorated_function(*args, **kwargs) - except DaemonError as daemon_error: - raise RPCError(1, daemon_error.args[0]) - return wrapper - - -class LBCDaemon(Daemon): - @handles_errors - async def getrawtransaction(self, hex_hash, verbose=False): - return await super().getrawtransaction(hex_hash=hex_hash, verbose=verbose) - - @handles_errors - async def getclaimsforname(self, name): - '''Given a name, retrieves all claims matching that name.''' - return await self._send_single('getclaimsforname', (name,)) - - @handles_errors - async def getbestblockhash(self): - '''Given a name, retrieves all claims matching that name.''' - return await self._send_single('getbestblockhash') diff --git a/lbry/wallet/server/db/__init__.py b/lbry/wallet/server/db/__init__.py deleted file mode 100644 index 8daf91d1f..000000000 --- a/lbry/wallet/server/db/__init__.py +++ /dev/null @@ -1,49 +0,0 @@ -import enum -from typing import NamedTuple, Optional - - -@enum.unique -class DB_PREFIXES(enum.Enum): - claim_to_support = b'K' - support_to_claim = b'L' - - claim_to_txo = b'E' - txo_to_claim = b'G' - - claim_to_channel = b'I' - channel_to_claim = b'J' - - claim_short_id_prefix = b'F' - effective_amount = b'D' - claim_expiration = b'O' - - claim_takeover = b'P' - pending_activation = b'Q' - activated_claim_and_support = b'R' - active_amount = b'S' - - repost = b'V' - reposted_claim = b'W' - - undo = b'M' - touched_or_deleted = b'Y' - - tx = b'B' - block_hash = b'C' - header = b'H' - tx_num = b'N' - tx_count = b'T' - tx_hash = b'X' - utxo = b'u' - hashx_utxo = b'h' - hashx_history = b'x' - db_state = b's' - channel_count = b'Z' - support_amount = b'a' - block_tx = b'b' - trending_notifications = b'c' - mempool_tx = b'd' - touched_hashX = b'e' - - -COLUMN_SETTINGS = {} # this is updated by the PrefixRow metaclass diff --git a/lbry/wallet/server/db/common.py b/lbry/wallet/server/db/common.py deleted file mode 100644 index 3024053f9..000000000 --- a/lbry/wallet/server/db/common.py +++ /dev/null @@ -1,477 +0,0 @@ -import typing -from typing import Optional -from lbry.error import ResolveCensoredError - -CLAIM_TYPES = { - 'stream': 1, - 'channel': 2, - 'repost': 3, - 'collection': 4, -} - -STREAM_TYPES = { - 'video': 1, - 'audio': 2, - 'image': 3, - 'document': 4, - 'binary': 5, - 'model': 6, -} - -# 9/21/2020 -MOST_USED_TAGS = { - "gaming", - "people & blogs", - "entertainment", - "music", - "pop culture", - "education", - "technology", - "blockchain", - "news", - "funny", - "science & technology", - "learning", - "gameplay", - "news & politics", - "comedy", - "bitcoin", - "beliefs", - "nature", - "art", - "economics", - "film & animation", - "lets play", - "games", - "sports", - "howto & style", - "game", - "cryptocurrency", - "playstation 4", - "automotive", - "crypto", - "mature", - "sony interactive entertainment", - "walkthrough", - "tutorial", - "video game", - "weapons", - "playthrough", - "pc", - "anime", - "how to", - "btc", - "fun", - "ethereum", - "food", - "travel & events", - "minecraft", - "science", - "autos & vehicles", - "play", - "politics", - "commentary", - "twitch", - "ps4live", - "love", - "ps4", - "nonprofits & activism", - "ps4share", - "fortnite", - "xbox", - "porn", - "video games", - "trump", - "español", - "money", - "music video", - "nintendo", - "movie", - "coronavirus", - "donald trump", - "steam", - "trailer", - "android", - "podcast", - "xbox one", - "survival", - "audio", - "linux", - "travel", - "funny moments", - "litecoin", - "animation", - "gamer", - "lets", - "playstation", - "bitcoin news", - "history", - "xxx", - "fox news", - "dance", - "god", - "adventure", - "liberal", - "2020", - "horror", - "government", - "freedom", - "reaction", - "meme", - "photography", - "truth", - "health", - "lbry", - "family", - "online", - "eth", - "crypto news", - "diy", - "trading", - "gold", - "memes", - "world", - "space", - "lol", - "covid-19", - "rpg", - "humor", - "democrat", - "film", - "call of duty", - "tech", - "religion", - "conspiracy", - "rap", - "cnn", - "hangoutsonair", - "unboxing", - "fiction", - "conservative", - "cars", - "hoa", - "epic", - "programming", - "progressive", - "cryptocurrency news", - "classical", - "jesus", - "movies", - "book", - "ps3", - "republican", - "fitness", - "books", - "multiplayer", - "animals", - "pokemon", - "bitcoin price", - "facebook", - "sharefactory", - "criptomonedas", - "cod", - "bible", - "business", - "stream", - "comics", - "how", - "fail", - "nsfw", - "new music", - "satire", - "pets & animals", - "computer", - "classical music", - "indie", - "musica", - "msnbc", - "fps", - "mod", - "sport", - "sony", - "ripple", - "auto", - "rock", - "marvel", - "complete", - "mining", - "political", - "mobile", - "pubg", - "hip hop", - "flat earth", - "xbox 360", - "reviews", - "vlogging", - "latest news", - "hack", - "tarot", - "iphone", - "media", - "cute", - "christian", - "free speech", - "trap", - "war", - "remix", - "ios", - "xrp", - "spirituality", - "song", - "league of legends", - "cat" -} - -MATURE_TAGS = [ - 'nsfw', 'porn', 'xxx', 'mature', 'adult', 'sex' -] - - -def normalize_tag(tag): - return tag.replace(" ", "_").replace("&", "and").replace("-", "_") - - -COMMON_TAGS = { - tag: normalize_tag(tag) for tag in list(MOST_USED_TAGS) -} - -INDEXED_LANGUAGES = [ - 'none', - 'en', - 'aa', - 'ab', - 'ae', - 'af', - 'ak', - 'am', - 'an', - 'ar', - 'as', - 'av', - 'ay', - 'az', - 'ba', - 'be', - 'bg', - 'bh', - 'bi', - 'bm', - 'bn', - 'bo', - 'br', - 'bs', - 'ca', - 'ce', - 'ch', - 'co', - 'cr', - 'cs', - 'cu', - 'cv', - 'cy', - 'da', - 'de', - 'dv', - 'dz', - 'ee', - 'el', - 'eo', - 'es', - 'et', - 'eu', - 'fa', - 'ff', - 'fi', - 'fj', - 'fo', - 'fr', - 'fy', - 'ga', - 'gd', - 'gl', - 'gn', - 'gu', - 'gv', - 'ha', - 'he', - 'hi', - 'ho', - 'hr', - 'ht', - 'hu', - 'hy', - 'hz', - 'ia', - 'id', - 'ie', - 'ig', - 'ii', - 'ik', - 'io', - 'is', - 'it', - 'iu', - 'ja', - 'jv', - 'ka', - 'kg', - 'ki', - 'kj', - 'kk', - 'kl', - 'km', - 'kn', - 'ko', - 'kr', - 'ks', - 'ku', - 'kv', - 'kw', - 'ky', - 'la', - 'lb', - 'lg', - 'li', - 'ln', - 'lo', - 'lt', - 'lu', - 'lv', - 'mg', - 'mh', - 'mi', - 'mk', - 'ml', - 'mn', - 'mr', - 'ms', - 'mt', - 'my', - 'na', - 'nb', - 'nd', - 'ne', - 'ng', - 'nl', - 'nn', - 'no', - 'nr', - 'nv', - 'ny', - 'oc', - 'oj', - 'om', - 'or', - 'os', - 'pa', - 'pi', - 'pl', - 'ps', - 'pt', - 'qu', - 'rm', - 'rn', - 'ro', - 'ru', - 'rw', - 'sa', - 'sc', - 'sd', - 'se', - 'sg', - 'si', - 'sk', - 'sl', - 'sm', - 'sn', - 'so', - 'sq', - 'sr', - 'ss', - 'st', - 'su', - 'sv', - 'sw', - 'ta', - 'te', - 'tg', - 'th', - 'ti', - 'tk', - 'tl', - 'tn', - 'to', - 'tr', - 'ts', - 'tt', - 'tw', - 'ty', - 'ug', - 'uk', - 'ur', - 'uz', - 've', - 'vi', - 'vo', - 'wa', - 'wo', - 'xh', - 'yi', - 'yo', - 'za', - 'zh', - 'zu' -] - - -class ResolveResult(typing.NamedTuple): - name: str - normalized_name: str - claim_hash: bytes - tx_num: int - position: int - tx_hash: bytes - height: int - amount: int - short_url: str - is_controlling: bool - canonical_url: str - creation_height: int - activation_height: int - expiration_height: int - effective_amount: int - support_amount: int - reposted: int - last_takeover_height: typing.Optional[int] - claims_in_channel: typing.Optional[int] - channel_hash: typing.Optional[bytes] - reposted_claim_hash: typing.Optional[bytes] - signature_valid: typing.Optional[bool] - - -class TrendingNotification(typing.NamedTuple): - height: int - prev_amount: int - new_amount: int - - -class UTXO(typing.NamedTuple): - tx_num: int - tx_pos: int - tx_hash: bytes - height: int - value: int - - -OptionalResolveResultOrError = Optional[typing.Union[ResolveResult, ResolveCensoredError, LookupError, ValueError]] - - -class ExpandedResolveResult(typing.NamedTuple): - stream: OptionalResolveResultOrError - channel: OptionalResolveResultOrError - repost: OptionalResolveResultOrError - reposted_channel: OptionalResolveResultOrError - - -class DBError(Exception): - """Raised on general DB errors generally indicating corruption.""" diff --git a/lbry/wallet/server/db/db.py b/lbry/wallet/server/db/db.py deleted file mode 100644 index 26937f60a..000000000 --- a/lbry/wallet/server/db/db.py +++ /dev/null @@ -1,1152 +0,0 @@ -import os -import asyncio -import array -import time -import typing -import struct -import zlib -import base64 -from typing import Optional, Iterable, Tuple, DefaultDict, Set, Dict, List, TYPE_CHECKING -from functools import partial -from asyncio import sleep -from bisect import bisect_right -from collections import defaultdict -from concurrent.futures.thread import ThreadPoolExecutor - -from lbry.error import ResolveCensoredError -from lbry.schema.result import Censor -from lbry.utils import LRUCacheWithMetrics -from lbry.schema.url import URL, normalize_name -from lbry.wallet.server import util -from lbry.wallet.server.hash import hash_to_hex_str -from lbry.wallet.server.tx import TxInput -from lbry.wallet.server.merkle import Merkle, MerkleCache -from lbry.wallet.server.db.common import ResolveResult, STREAM_TYPES, CLAIM_TYPES, ExpandedResolveResult, DBError, UTXO -from lbry.wallet.server.db.prefixes import PendingActivationValue, ClaimTakeoverValue, ClaimToTXOValue, PrefixDB -from lbry.wallet.server.db.prefixes import ACTIVATED_CLAIM_TXO_TYPE, ACTIVATED_SUPPORT_TXO_TYPE -from lbry.wallet.server.db.prefixes import PendingActivationKey, TXOToClaimValue, DBStatePrefixRow, MempoolTXPrefixRow -from lbry.wallet.transaction import OutputScript -from lbry.schema.claim import Claim, guess_stream_type -from lbry.wallet.ledger import Ledger, RegTestLedger, TestNetLedger - - -if TYPE_CHECKING: - from lbry.wallet.server.db.prefixes import EffectiveAmountKey - - -TXO_STRUCT = struct.Struct(b'>LH') -TXO_STRUCT_unpack = TXO_STRUCT.unpack -TXO_STRUCT_pack = TXO_STRUCT.pack - - -class HubDB: - DB_VERSIONS = HIST_DB_VERSIONS = [7] - - def __init__(self, coin, db_dir: str, cache_MB: int = 512, reorg_limit: int = 200, - cache_all_claim_txos: bool = False, cache_all_tx_hashes: bool = False, - secondary_name: str = '', max_open_files: int = 64, blocking_channel_ids: List[str] = None, - filtering_channel_ids: List[str] = None, executor: ThreadPoolExecutor = None): - self.logger = util.class_logger(__name__, self.__class__.__name__) - self.coin = coin - self._executor = executor - self._db_dir = db_dir - - self._cache_MB = cache_MB - self._reorg_limit = reorg_limit - self._cache_all_claim_txos = cache_all_claim_txos - self._cache_all_tx_hashes = cache_all_tx_hashes - self._secondary_name = secondary_name - if secondary_name: - assert max_open_files == -1, 'max open files must be -1 for secondary readers' - self._db_max_open_files = max_open_files - self.prefix_db: typing.Optional[PrefixDB] = None - - self.hist_unflushed = defaultdict(partial(array.array, 'I')) - self.hist_unflushed_count = 0 - self.hist_flush_count = 0 - self.hist_comp_flush_count = -1 - self.hist_comp_cursor = -1 - - self.es_sync_height = 0 - - # blocking/filtering dicts - blocking_channels = blocking_channel_ids or [] #self.env.default('BLOCKING_CHANNEL_IDS', '').split(' ') - filtering_channels = filtering_channel_ids or [] #self.env.default('FILTERING_CHANNEL_IDS', '').split(' ') - self.blocked_streams = {} - self.blocked_channels = {} - self.blocking_channel_hashes = { - bytes.fromhex(channel_id) for channel_id in blocking_channels if channel_id - } - self.filtered_streams = {} - - self.filtered_channels = {} - self.filtering_channel_hashes = { - bytes.fromhex(channel_id) for channel_id in filtering_channels if channel_id - } - - self.tx_counts = None - self.headers = None - self.encoded_headers = LRUCacheWithMetrics(1 << 21, metric_name='encoded_headers', namespace='wallet_server') - self.last_flush = time.time() - - # Header merkle cache - self.merkle = Merkle() - self.header_mc = MerkleCache(self.merkle, self.fs_block_hashes) - - self._tx_and_merkle_cache = LRUCacheWithMetrics(2 ** 16, metric_name='tx_and_merkle', namespace="wallet_server") - - # these are only used if the cache_all_tx_hashes setting is on - self.total_transactions: List[bytes] = [] - self.tx_num_mapping: Dict[bytes, int] = {} - - # these are only used if the cache_all_claim_txos setting is on - self.claim_to_txo: Dict[bytes, ClaimToTXOValue] = {} - self.txo_to_claim: DefaultDict[int, Dict[int, bytes]] = defaultdict(dict) - - self.genesis_bytes = bytes.fromhex(self.coin.GENESIS_HASH) - - if coin.NET == 'mainnet': - self.ledger = Ledger - elif coin.NET == 'testnet': - self.ledger = TestNetLedger - else: - self.ledger = RegTestLedger - - def get_claim_from_txo(self, tx_num: int, tx_idx: int) -> Optional[TXOToClaimValue]: - claim_hash_and_name = self.prefix_db.txo_to_claim.get(tx_num, tx_idx) - if not claim_hash_and_name: - return - return claim_hash_and_name - - def get_repost(self, claim_hash) -> Optional[bytes]: - repost = self.prefix_db.repost.get(claim_hash) - if repost: - return repost.reposted_claim_hash - return - - def get_reposted_count(self, claim_hash: bytes) -> int: - return sum( - 1 for _ in self.prefix_db.reposted_claim.iterate(prefix=(claim_hash,), include_value=False) - ) - - def get_activation(self, tx_num, position, is_support=False) -> int: - activation = self.prefix_db.activated.get( - ACTIVATED_SUPPORT_TXO_TYPE if is_support else ACTIVATED_CLAIM_TXO_TYPE, tx_num, position - ) - if activation: - return activation.height - return -1 - - def get_supported_claim_from_txo(self, tx_num: int, position: int) -> typing.Tuple[Optional[bytes], Optional[int]]: - supported_claim_hash = self.prefix_db.support_to_claim.get(tx_num, position) - if supported_claim_hash: - packed_support_amount = self.prefix_db.claim_to_support.get( - supported_claim_hash.claim_hash, tx_num, position - ) - if packed_support_amount: - return supported_claim_hash.claim_hash, packed_support_amount.amount - return None, None - - def get_support_amount(self, claim_hash: bytes): - support_amount_val = self.prefix_db.support_amount.get(claim_hash) - if support_amount_val is None: - return 0 - return support_amount_val.amount - - def get_supports(self, claim_hash: bytes): - return [ - (k.tx_num, k.position, v.amount) for k, v in self.prefix_db.claim_to_support.iterate(prefix=(claim_hash,)) - ] - - def get_short_claim_id_url(self, name: str, normalized_name: str, claim_hash: bytes, - root_tx_num: int, root_position: int) -> str: - claim_id = claim_hash.hex() - for prefix_len in range(10): - for k in self.prefix_db.claim_short_id.iterate(prefix=(normalized_name, claim_id[:prefix_len+1]), - include_value=False): - if k.root_tx_num == root_tx_num and k.root_position == root_position: - return f'{name}#{k.partial_claim_id}' - break - print(f"{claim_id} has a collision") - return f'{name}#{claim_id}' - - def _prepare_resolve_result(self, tx_num: int, position: int, claim_hash: bytes, name: str, - root_tx_num: int, root_position: int, activation_height: int, - signature_valid: bool) -> ResolveResult: - try: - normalized_name = normalize_name(name) - except UnicodeDecodeError: - normalized_name = name - controlling_claim = self.get_controlling_claim(normalized_name) - - tx_hash = self.get_tx_hash(tx_num) - height = bisect_right(self.tx_counts, tx_num) - created_height = bisect_right(self.tx_counts, root_tx_num) - last_take_over_height = controlling_claim.height - - expiration_height = self.coin.get_expiration_height(height) - support_amount = self.get_support_amount(claim_hash) - claim_amount = self.get_cached_claim_txo(claim_hash).amount - - effective_amount = self.get_effective_amount(claim_hash) - channel_hash = self.get_channel_for_claim(claim_hash, tx_num, position) - reposted_claim_hash = self.get_repost(claim_hash) - short_url = self.get_short_claim_id_url(name, normalized_name, claim_hash, root_tx_num, root_position) - canonical_url = short_url - claims_in_channel = self.get_claims_in_channel_count(claim_hash) - if channel_hash: - channel_vals = self.get_cached_claim_txo(channel_hash) - if channel_vals: - channel_short_url = self.get_short_claim_id_url( - channel_vals.name, channel_vals.normalized_name, channel_hash, channel_vals.root_tx_num, - channel_vals.root_position - ) - canonical_url = f'{channel_short_url}/{short_url}' - return ResolveResult( - name, normalized_name, claim_hash, tx_num, position, tx_hash, height, claim_amount, short_url=short_url, - is_controlling=controlling_claim.claim_hash == claim_hash, canonical_url=canonical_url, - last_takeover_height=last_take_over_height, claims_in_channel=claims_in_channel, - creation_height=created_height, activation_height=activation_height, - expiration_height=expiration_height, effective_amount=effective_amount, support_amount=support_amount, - channel_hash=channel_hash, reposted_claim_hash=reposted_claim_hash, - reposted=self.get_reposted_count(claim_hash), - signature_valid=None if not channel_hash else signature_valid - ) - - def _resolve_parsed_url(self, name: str, claim_id: Optional[str] = None, - amount_order: Optional[int] = None) -> Optional[ResolveResult]: - """ - :param normalized_name: name - :param claim_id: partial or complete claim id - :param amount_order: '$' suffix to a url, defaults to 1 (winning) if no claim id modifier is provided - """ - try: - normalized_name = normalize_name(name) - except UnicodeDecodeError: - normalized_name = name - if (not amount_order and not claim_id) or amount_order == 1: - # winning resolution - controlling = self.get_controlling_claim(normalized_name) - if not controlling: - # print(f"none controlling for lbry://{normalized_name}") - return - # print(f"resolved controlling lbry://{normalized_name}#{controlling.claim_hash.hex()}") - return self._fs_get_claim_by_hash(controlling.claim_hash) - - amount_order = max(int(amount_order or 1), 1) - - if claim_id: - if len(claim_id) == 40: # a full claim id - claim_txo = self.get_claim_txo(bytes.fromhex(claim_id)) - if not claim_txo or normalized_name != claim_txo.normalized_name: - return - return self._prepare_resolve_result( - claim_txo.tx_num, claim_txo.position, bytes.fromhex(claim_id), claim_txo.name, - claim_txo.root_tx_num, claim_txo.root_position, - self.get_activation(claim_txo.tx_num, claim_txo.position), claim_txo.channel_signature_is_valid - ) - # resolve by partial/complete claim id - for key, claim_txo in self.prefix_db.claim_short_id.iterate(prefix=(normalized_name, claim_id[:10])): - full_claim_hash = self.get_cached_claim_hash(claim_txo.tx_num, claim_txo.position) - c = self.get_cached_claim_txo(full_claim_hash) - - non_normalized_name = c.name - signature_is_valid = c.channel_signature_is_valid - return self._prepare_resolve_result( - claim_txo.tx_num, claim_txo.position, full_claim_hash, non_normalized_name, key.root_tx_num, - key.root_position, self.get_activation(claim_txo.tx_num, claim_txo.position), - signature_is_valid - ) - return - - # resolve by amount ordering, 1 indexed - for idx, (key, claim_val) in enumerate(self.prefix_db.effective_amount.iterate(prefix=(normalized_name,))): - if amount_order > idx + 1: - continue - claim_txo = self.get_cached_claim_txo(claim_val.claim_hash) - activation = self.get_activation(key.tx_num, key.position) - return self._prepare_resolve_result( - key.tx_num, key.position, claim_val.claim_hash, key.normalized_name, claim_txo.root_tx_num, - claim_txo.root_position, activation, claim_txo.channel_signature_is_valid - ) - return - - def _resolve_claim_in_channel(self, channel_hash: bytes, normalized_name: str): - candidates = [] - for key, stream in self.prefix_db.channel_to_claim.iterate(prefix=(channel_hash, normalized_name)): - effective_amount = self.get_effective_amount(stream.claim_hash) - if not candidates or candidates[-1][-1] == effective_amount: - candidates.append((stream.claim_hash, key.tx_num, key.position, effective_amount)) - else: - break - if not candidates: - return - return list(sorted(candidates, key=lambda item: item[1]))[0] - - def _resolve(self, url) -> ExpandedResolveResult: - try: - parsed = URL.parse(url) - except ValueError as e: - return ExpandedResolveResult(e, None, None, None) - - stream = channel = resolved_channel = resolved_stream = None - if parsed.has_stream_in_channel: - channel = parsed.channel - stream = parsed.stream - elif parsed.has_channel: - channel = parsed.channel - elif parsed.has_stream: - stream = parsed.stream - if channel: - resolved_channel = self._resolve_parsed_url(channel.name, channel.claim_id, channel.amount_order) - if not resolved_channel: - return ExpandedResolveResult(None, LookupError(f'Could not find channel in "{url}".'), None, None) - if stream: - if resolved_channel: - stream_claim = self._resolve_claim_in_channel(resolved_channel.claim_hash, stream.normalized) - if stream_claim: - stream_claim_id, stream_tx_num, stream_tx_pos, effective_amount = stream_claim - resolved_stream = self._fs_get_claim_by_hash(stream_claim_id) - else: - resolved_stream = self._resolve_parsed_url(stream.name, stream.claim_id, stream.amount_order) - if not channel and not resolved_channel and resolved_stream and resolved_stream.channel_hash: - resolved_channel = self._fs_get_claim_by_hash(resolved_stream.channel_hash) - if not resolved_stream: - return ExpandedResolveResult(LookupError(f'Could not find claim at "{url}".'), None, None, None) - - repost = None - reposted_channel = None - if resolved_stream or resolved_channel: - claim_hash = resolved_stream.claim_hash if resolved_stream else resolved_channel.claim_hash - claim = resolved_stream if resolved_stream else resolved_channel - reposted_claim_hash = resolved_stream.reposted_claim_hash if resolved_stream else None - blocker_hash = self.blocked_streams.get(claim_hash) or self.blocked_streams.get( - reposted_claim_hash) or self.blocked_channels.get(claim_hash) or self.blocked_channels.get( - reposted_claim_hash) or self.blocked_channels.get(claim.channel_hash) - if blocker_hash: - reason_row = self._fs_get_claim_by_hash(blocker_hash) - return ExpandedResolveResult( - None, ResolveCensoredError(url, blocker_hash, censor_row=reason_row), None, None - ) - if claim.reposted_claim_hash: - repost = self._fs_get_claim_by_hash(claim.reposted_claim_hash) - if repost and repost.channel_hash and repost.signature_valid: - reposted_channel = self._fs_get_claim_by_hash(repost.channel_hash) - return ExpandedResolveResult(resolved_stream, resolved_channel, repost, reposted_channel) - - async def resolve(self, url) -> ExpandedResolveResult: - return await asyncio.get_event_loop().run_in_executor(self._executor, self._resolve, url) - - def _fs_get_claim_by_hash(self, claim_hash): - claim = self.get_cached_claim_txo(claim_hash) - if claim: - activation = self.get_activation(claim.tx_num, claim.position) - return self._prepare_resolve_result( - claim.tx_num, claim.position, claim_hash, claim.name, claim.root_tx_num, claim.root_position, - activation, claim.channel_signature_is_valid - ) - - async def fs_getclaimbyid(self, claim_id): - return await asyncio.get_event_loop().run_in_executor( - self._executor, self._fs_get_claim_by_hash, bytes.fromhex(claim_id) - ) - - def get_claim_txo_amount(self, claim_hash: bytes) -> Optional[int]: - claim = self.get_claim_txo(claim_hash) - if claim: - return claim.amount - - def get_block_hash(self, height: int) -> Optional[bytes]: - v = self.prefix_db.block_hash.get(height) - if v: - return v.block_hash - - def get_support_txo_amount(self, claim_hash: bytes, tx_num: int, position: int) -> Optional[int]: - v = self.prefix_db.claim_to_support.get(claim_hash, tx_num, position) - return None if not v else v.amount - - def get_claim_txo(self, claim_hash: bytes) -> Optional[ClaimToTXOValue]: - assert claim_hash - return self.prefix_db.claim_to_txo.get(claim_hash) - - def _get_active_amount(self, claim_hash: bytes, txo_type: int, height: int) -> int: - return sum( - v.amount for v in self.prefix_db.active_amount.iterate( - start=(claim_hash, txo_type, 0), stop=(claim_hash, txo_type, height), include_key=False - ) - ) - - def get_active_amount_as_of_height(self, claim_hash: bytes, height: int) -> int: - for v in self.prefix_db.active_amount.iterate( - start=(claim_hash, ACTIVATED_CLAIM_TXO_TYPE, 0), stop=(claim_hash, ACTIVATED_CLAIM_TXO_TYPE, height), - include_key=False, reverse=True): - return v.amount - return 0 - - def get_effective_amount(self, claim_hash: bytes) -> int: - return self._get_active_amount( - claim_hash, ACTIVATED_SUPPORT_TXO_TYPE, self.db_height + 1 - ) + self._get_active_amount(claim_hash, ACTIVATED_CLAIM_TXO_TYPE, self.db_height + 1) - - def get_url_effective_amount(self, name: str, claim_hash: bytes) -> Optional['EffectiveAmountKey']: - for k, v in self.prefix_db.effective_amount.iterate(prefix=(name,)): - if v.claim_hash == claim_hash: - return k - - def get_claims_for_name(self, name): - claims = [] - prefix = self.prefix_db.claim_short_id.pack_partial_key(name) + bytes([1]) - stop = self.prefix_db.claim_short_id.pack_partial_key(name) + int(2).to_bytes(1, byteorder='big') - cf = self.prefix_db.column_families[self.prefix_db.claim_short_id.prefix] - for _v in self.prefix_db.iterator(column_family=cf, start=prefix, iterate_upper_bound=stop, include_key=False): - v = self.prefix_db.claim_short_id.unpack_value(_v) - claim_hash = self.get_claim_from_txo(v.tx_num, v.position).claim_hash - if claim_hash not in claims: - claims.append(claim_hash) - return claims - - def get_claims_in_channel_count(self, channel_hash) -> int: - channel_count_val = self.prefix_db.channel_count.get(channel_hash) - if channel_count_val is None: - return 0 - return channel_count_val.count - - async def reload_blocking_filtering_streams(self): - def reload(): - self.blocked_streams, self.blocked_channels = self.get_streams_and_channels_reposted_by_channel_hashes( - self.blocking_channel_hashes - ) - self.filtered_streams, self.filtered_channels = self.get_streams_and_channels_reposted_by_channel_hashes( - self.filtering_channel_hashes - ) - await asyncio.get_event_loop().run_in_executor(self._executor, reload) - - def get_streams_and_channels_reposted_by_channel_hashes(self, reposter_channel_hashes: Set[bytes]): - streams, channels = {}, {} - for reposter_channel_hash in reposter_channel_hashes: - for stream in self.prefix_db.channel_to_claim.iterate((reposter_channel_hash, ), include_key=False): - repost = self.get_repost(stream.claim_hash) - if repost: - txo = self.get_claim_txo(repost) - if txo: - if txo.normalized_name.startswith('@'): - channels[repost] = reposter_channel_hash - else: - streams[repost] = reposter_channel_hash - return streams, channels - - def get_channel_for_claim(self, claim_hash, tx_num, position) -> Optional[bytes]: - v = self.prefix_db.claim_to_channel.get(claim_hash, tx_num, position) - if v: - return v.signing_hash - - def get_expired_by_height(self, height: int) -> Dict[bytes, Tuple[int, int, str, TxInput]]: - expired = {} - for k, v in self.prefix_db.claim_expiration.iterate(prefix=(height,)): - tx_hash = self.get_tx_hash(k.tx_num) - tx = self.coin.transaction(self.prefix_db.tx.get(tx_hash, deserialize_value=False)) - # treat it like a claim spend so it will delete/abandon properly - # the _spend_claim function this result is fed to expects a txi, so make a mock one - # print(f"\texpired lbry://{v.name} {v.claim_hash.hex()}") - expired[v.claim_hash] = ( - k.tx_num, k.position, v.normalized_name, - TxInput(prev_hash=tx_hash, prev_idx=k.position, script=tx.outputs[k.position].pk_script, sequence=0) - ) - return expired - - def get_controlling_claim(self, name: str) -> Optional[ClaimTakeoverValue]: - controlling = self.prefix_db.claim_takeover.get(name) - if not controlling: - return - return controlling - - def get_claim_txos_for_name(self, name: str): - txos = {} - prefix = self.prefix_db.claim_short_id.pack_partial_key(name) + int(1).to_bytes(1, byteorder='big') - stop = self.prefix_db.claim_short_id.pack_partial_key(name) + int(2).to_bytes(1, byteorder='big') - cf = self.prefix_db.column_families[self.prefix_db.claim_short_id.prefix] - for v in self.prefix_db.iterator(column_family=cf, start=prefix, iterate_upper_bound=stop, include_key=False): - tx_num, nout = self.prefix_db.claim_short_id.unpack_value(v) - txos[self.get_claim_from_txo(tx_num, nout).claim_hash] = tx_num, nout - return txos - - def get_claim_metadata(self, tx_hash, nout): - raw = self.prefix_db.tx.get(tx_hash, deserialize_value=False) - try: - output = self.coin.transaction(raw).outputs[nout] - script = OutputScript(output.pk_script) - script.parse() - return Claim.from_bytes(script.values['claim']) - except: - self.logger.error("claim parsing for ES failed with tx: %s", tx_hash[::-1].hex()) - return - - def _prepare_claim_metadata(self, claim_hash: bytes, claim: ResolveResult): - metadata = self.get_claim_metadata(claim.tx_hash, claim.position) - if not metadata: - return - metadata = metadata - if not metadata.is_stream or not metadata.stream.has_fee: - fee_amount = 0 - else: - fee_amount = int(max(metadata.stream.fee.amount or 0, 0) * 1000) - if fee_amount >= 9223372036854775807: - return - reposted_claim_hash = None if not metadata.is_repost else metadata.repost.reference.claim_hash[::-1] - reposted_claim = None - reposted_metadata = None - if reposted_claim_hash: - reposted_claim = self.get_cached_claim_txo(reposted_claim_hash) - if not reposted_claim: - return - reposted_metadata = self.get_claim_metadata( - self.get_tx_hash(reposted_claim.tx_num), reposted_claim.position - ) - if not reposted_metadata: - return - reposted_tags = [] - reposted_languages = [] - reposted_has_source = False - reposted_claim_type = None - reposted_stream_type = None - reposted_media_type = None - reposted_fee_amount = None - reposted_fee_currency = None - reposted_duration = None - if reposted_claim: - reposted_tx_hash = self.get_tx_hash(reposted_claim.tx_num) - raw_reposted_claim_tx = self.prefix_db.tx.get(reposted_tx_hash, deserialize_value=False) - try: - reposted_claim_txo = self.coin.transaction( - raw_reposted_claim_tx - ).outputs[reposted_claim.position] - reposted_script = OutputScript(reposted_claim_txo.pk_script) - reposted_script.parse() - reposted_metadata = Claim.from_bytes(reposted_script.values['claim']) - except: - self.logger.error("failed to parse reposted claim in tx %s that was reposted by %s", - reposted_tx_hash[::-1].hex(), claim_hash.hex()) - return - if reposted_metadata: - if reposted_metadata.is_stream: - meta = reposted_metadata.stream - elif reposted_metadata.is_channel: - meta = reposted_metadata.channel - elif reposted_metadata.is_collection: - meta = reposted_metadata.collection - elif reposted_metadata.is_repost: - meta = reposted_metadata.repost - else: - return - reposted_tags = [tag for tag in meta.tags] - reposted_languages = [lang.language or 'none' for lang in meta.languages] or ['none'] - reposted_has_source = False if not reposted_metadata.is_stream else reposted_metadata.stream.has_source - reposted_claim_type = CLAIM_TYPES[reposted_metadata.claim_type] - reposted_stream_type = STREAM_TYPES[guess_stream_type(reposted_metadata.stream.source.media_type)] \ - if reposted_has_source else 0 - reposted_media_type = reposted_metadata.stream.source.media_type if reposted_metadata.is_stream else 0 - if not reposted_metadata.is_stream or not reposted_metadata.stream.has_fee: - reposted_fee_amount = 0 - else: - reposted_fee_amount = int(max(reposted_metadata.stream.fee.amount or 0, 0) * 1000) - if reposted_fee_amount >= 9223372036854775807: - return - reposted_fee_currency = None if not reposted_metadata.is_stream else reposted_metadata.stream.fee.currency - reposted_duration = None - if reposted_metadata.is_stream and \ - (reposted_metadata.stream.video.duration or reposted_metadata.stream.audio.duration): - reposted_duration = reposted_metadata.stream.video.duration or reposted_metadata.stream.audio.duration - if metadata.is_stream: - meta = metadata.stream - elif metadata.is_channel: - meta = metadata.channel - elif metadata.is_collection: - meta = metadata.collection - elif metadata.is_repost: - meta = metadata.repost - else: - return - claim_tags = [tag for tag in meta.tags] - claim_languages = [lang.language or 'none' for lang in meta.languages] or ['none'] - - tags = list(set(claim_tags).union(set(reposted_tags))) - languages = list(set(claim_languages).union(set(reposted_languages))) - blocked_hash = self.blocked_streams.get(claim_hash) or self.blocked_streams.get( - reposted_claim_hash) or self.blocked_channels.get(claim_hash) or self.blocked_channels.get( - reposted_claim_hash) or self.blocked_channels.get(claim.channel_hash) - filtered_hash = self.filtered_streams.get(claim_hash) or self.filtered_streams.get( - reposted_claim_hash) or self.filtered_channels.get(claim_hash) or self.filtered_channels.get( - reposted_claim_hash) or self.filtered_channels.get(claim.channel_hash) - value = { - 'claim_id': claim_hash.hex(), - 'claim_name': claim.name, - 'normalized_name': claim.normalized_name, - 'tx_id': claim.tx_hash[::-1].hex(), - 'tx_num': claim.tx_num, - 'tx_nout': claim.position, - 'amount': claim.amount, - 'timestamp': self.estimate_timestamp(claim.height), - 'creation_timestamp': self.estimate_timestamp(claim.creation_height), - 'height': claim.height, - 'creation_height': claim.creation_height, - 'activation_height': claim.activation_height, - 'expiration_height': claim.expiration_height, - 'effective_amount': claim.effective_amount, - 'support_amount': claim.support_amount, - 'is_controlling': bool(claim.is_controlling), - 'last_take_over_height': claim.last_takeover_height, - 'short_url': claim.short_url, - 'canonical_url': claim.canonical_url, - 'title': None if not metadata.is_stream else metadata.stream.title, - 'author': None if not metadata.is_stream else metadata.stream.author, - 'description': None if not metadata.is_stream else metadata.stream.description, - 'claim_type': CLAIM_TYPES[metadata.claim_type], - 'has_source': reposted_has_source if metadata.is_repost else ( - False if not metadata.is_stream else metadata.stream.has_source), - 'sd_hash': metadata.stream.source.sd_hash if metadata.is_stream and metadata.stream.has_source else None, - 'stream_type': STREAM_TYPES[guess_stream_type(metadata.stream.source.media_type)] - if metadata.is_stream and metadata.stream.has_source - else reposted_stream_type if metadata.is_repost else 0, - 'media_type': metadata.stream.source.media_type - if metadata.is_stream else reposted_media_type if metadata.is_repost else None, - 'fee_amount': fee_amount if not metadata.is_repost else reposted_fee_amount, - 'fee_currency': metadata.stream.fee.currency - if metadata.is_stream else reposted_fee_currency if metadata.is_repost else None, - 'repost_count': self.get_reposted_count(claim_hash), - 'reposted_claim_id': None if not reposted_claim_hash else reposted_claim_hash.hex(), - 'reposted_claim_type': reposted_claim_type, - 'reposted_has_source': reposted_has_source, - 'channel_id': None if not metadata.is_signed else metadata.signing_channel_hash[::-1].hex(), - 'public_key_id': None if not metadata.is_channel else - self.ledger.public_key_to_address(metadata.channel.public_key_bytes), - 'signature': (metadata.signature or b'').hex() or None, - # 'signature_digest': metadata.signature, - 'is_signature_valid': bool(claim.signature_valid), - 'tags': tags, - 'languages': languages, - 'censor_type': Censor.RESOLVE if blocked_hash else Censor.SEARCH if filtered_hash else Censor.NOT_CENSORED, - 'censoring_channel_id': (blocked_hash or filtered_hash or b'').hex() or None, - 'claims_in_channel': None if not metadata.is_channel else self.get_claims_in_channel_count(claim_hash) - } - - if metadata.is_repost and reposted_duration is not None: - value['duration'] = reposted_duration - elif metadata.is_stream and (metadata.stream.video.duration or metadata.stream.audio.duration): - value['duration'] = metadata.stream.video.duration or metadata.stream.audio.duration - if metadata.is_stream: - value['release_time'] = metadata.stream.release_time or value['creation_timestamp'] - elif metadata.is_repost or metadata.is_collection: - value['release_time'] = value['creation_timestamp'] - return value - - async def all_claims_producer(self, batch_size=500_000): - batch = [] - if self._cache_all_claim_txos: - claim_iterator = self.claim_to_txo.items() - else: - claim_iterator = map(lambda item: (item[0].claim_hash, item[1]), self.prefix_db.claim_to_txo.iterate()) - - for claim_hash, claim_txo in claim_iterator: - # TODO: fix the couple of claim txos that dont have controlling names - if not self.prefix_db.claim_takeover.get(claim_txo.normalized_name): - continue - activation = self.get_activation(claim_txo.tx_num, claim_txo.position) - claim = self._prepare_resolve_result( - claim_txo.tx_num, claim_txo.position, claim_hash, claim_txo.name, claim_txo.root_tx_num, - claim_txo.root_position, activation, claim_txo.channel_signature_is_valid - ) - if claim: - batch.append(claim) - if len(batch) == batch_size: - batch.sort(key=lambda x: x.tx_hash) # sort is to improve read-ahead hits - for claim in batch: - meta = self._prepare_claim_metadata(claim.claim_hash, claim) - if meta: - yield meta - batch.clear() - batch.sort(key=lambda x: x.tx_hash) - for claim in batch: - meta = self._prepare_claim_metadata(claim.claim_hash, claim) - if meta: - yield meta - batch.clear() - - def claim_producer(self, claim_hash: bytes) -> Optional[Dict]: - claim_txo = self.get_cached_claim_txo(claim_hash) - if not claim_txo: - self.logger.warning("can't sync non existent claim to ES: %s", claim_hash.hex()) - return - if not self.prefix_db.claim_takeover.get(claim_txo.normalized_name): - self.logger.warning("can't sync non existent claim to ES: %s", claim_hash.hex()) - return - activation = self.get_activation(claim_txo.tx_num, claim_txo.position) - claim = self._prepare_resolve_result( - claim_txo.tx_num, claim_txo.position, claim_hash, claim_txo.name, claim_txo.root_tx_num, - claim_txo.root_position, activation, claim_txo.channel_signature_is_valid - ) - if not claim: - self.logger.warning("wat") - return - return self._prepare_claim_metadata(claim.claim_hash, claim) - - def claims_producer(self, claim_hashes: Set[bytes]): - batch = [] - results = [] - - for claim_hash in claim_hashes: - claim_txo = self.get_cached_claim_txo(claim_hash) - if not claim_txo: - self.logger.warning("can't sync non existent claim to ES: %s", claim_hash.hex()) - continue - if not self.prefix_db.claim_takeover.get(claim_txo.normalized_name): - self.logger.warning("can't sync non existent claim to ES: %s", claim_hash.hex()) - continue - - activation = self.get_activation(claim_txo.tx_num, claim_txo.position) - claim = self._prepare_resolve_result( - claim_txo.tx_num, claim_txo.position, claim_hash, claim_txo.name, claim_txo.root_tx_num, - claim_txo.root_position, activation, claim_txo.channel_signature_is_valid - ) - if claim: - batch.append(claim) - - batch.sort(key=lambda x: x.tx_hash) - - for claim in batch: - _meta = self._prepare_claim_metadata(claim.claim_hash, claim) - if _meta: - results.append(_meta) - return results - - def get_activated_at_height(self, height: int) -> DefaultDict[PendingActivationValue, List[PendingActivationKey]]: - activated = defaultdict(list) - for k, v in self.prefix_db.pending_activation.iterate(prefix=(height,)): - activated[v].append(k) - return activated - - def get_future_activated(self, height: int) -> typing.Dict[PendingActivationValue, PendingActivationKey]: - results = {} - for k, v in self.prefix_db.pending_activation.iterate( - start=(height + 1,), stop=(height + 1 + self.coin.maxTakeoverDelay,), reverse=True): - if v not in results: - results[v] = k - return results - - async def _read_tx_counts(self): - if self.tx_counts is not None: - return - # tx_counts[N] has the cumulative number of txs at the end of - # height N. So tx_counts[0] is 1 - the genesis coinbase - - def get_counts(): - return [ - v.tx_count for v in self.prefix_db.tx_count.iterate( - start=(0,), stop=(self.db_height + 1,), include_key=False, fill_cache=False - ) - ] - - tx_counts = await asyncio.get_event_loop().run_in_executor(self._executor, get_counts) - assert len(tx_counts) == self.db_height + 1, f"{len(tx_counts)} vs {self.db_height + 1}" - self.tx_counts = array.array('I', tx_counts) - - if self.tx_counts: - assert self.db_tx_count == self.tx_counts[-1], \ - f"{self.db_tx_count} vs {self.tx_counts[-1]} ({len(self.tx_counts)} counts)" - else: - assert self.db_tx_count == 0 - - async def _read_claim_txos(self): - def read_claim_txos(): - set_claim_to_txo = self.claim_to_txo.__setitem__ - for k, v in self.prefix_db.claim_to_txo.iterate(fill_cache=False): - set_claim_to_txo(k.claim_hash, v) - self.txo_to_claim[v.tx_num][v.position] = k.claim_hash - - self.claim_to_txo.clear() - self.txo_to_claim.clear() - start = time.perf_counter() - self.logger.info("loading claims") - await asyncio.get_event_loop().run_in_executor(self._executor, read_claim_txos) - ts = time.perf_counter() - start - self.logger.info("loaded %i claim txos in %ss", len(self.claim_to_txo), round(ts, 4)) - - async def _read_headers(self): - if self.headers is not None: - return - - def get_headers(): - return [ - header for header in self.prefix_db.header.iterate( - start=(0, ), stop=(self.db_height + 1, ), include_key=False, fill_cache=False, deserialize_value=False - ) - ] - - headers = await asyncio.get_event_loop().run_in_executor(self._executor, get_headers) - assert len(headers) - 1 == self.db_height, f"{len(headers)} vs {self.db_height}" - self.headers = headers - - async def _read_tx_hashes(self): - def _read_tx_hashes(): - return list(self.prefix_db.tx_hash.iterate(start=(0,), stop=(self.db_tx_count + 1), include_key=False, fill_cache=False, deserialize_value=False)) - - self.logger.info("loading tx hashes") - self.total_transactions.clear() - self.tx_num_mapping.clear() - start = time.perf_counter() - self.total_transactions.extend(await asyncio.get_event_loop().run_in_executor(self._executor, _read_tx_hashes)) - self.tx_num_mapping = { - tx_hash: tx_num for tx_num, tx_hash in enumerate(self.total_transactions) - } - ts = time.perf_counter() - start - self.logger.info("loaded %i tx hashes in %ss", len(self.total_transactions), round(ts, 4)) - - def estimate_timestamp(self, height: int) -> int: - if height < len(self.headers): - return struct.unpack(' 0: - await self.populate_header_merkle_cache() - - def close(self): - self.prefix_db.close() - self.prefix_db = None - - def get_tx_hash(self, tx_num: int) -> bytes: - if self._cache_all_tx_hashes: - return self.total_transactions[tx_num] - return self.prefix_db.tx_hash.get(tx_num, deserialize_value=False) - - def get_tx_num(self, tx_hash: bytes) -> int: - if self._cache_all_tx_hashes: - return self.tx_num_mapping[tx_hash] - return self.prefix_db.tx_num.get(tx_hash).tx_num - - def get_cached_claim_txo(self, claim_hash: bytes) -> Optional[ClaimToTXOValue]: - if self._cache_all_claim_txos: - return self.claim_to_txo.get(claim_hash) - return self.prefix_db.claim_to_txo.get_pending(claim_hash) - - def get_cached_claim_hash(self, tx_num: int, position: int) -> Optional[bytes]: - if self._cache_all_claim_txos: - if tx_num not in self.txo_to_claim: - return - return self.txo_to_claim[tx_num].get(position, None) - v = self.prefix_db.txo_to_claim.get_pending(tx_num, position) - return None if not v else v.claim_hash - - def get_cached_claim_exists(self, tx_num: int, position: int) -> bool: - return self.get_cached_claim_hash(tx_num, position) is not None - - # Header merkle cache - - async def populate_header_merkle_cache(self): - self.logger.info('populating header merkle cache...') - length = max(1, self.db_height - self._reorg_limit) - start = time.time() - await self.header_mc.initialize(length) - elapsed = time.time() - start - self.logger.info(f'header merkle cache populated in {elapsed:.1f}s') - - async def header_branch_and_root(self, length, height): - return await self.header_mc.branch_and_root(length, height) - - async def raw_header(self, height): - """Return the binary header at the given height.""" - header, n = await self.read_headers(height, 1) - if n != 1: - raise IndexError(f'height {height:,d} out of range') - return header - - def encode_headers(self, start_height, count, headers): - key = (start_height, count) - if not self.encoded_headers.get(key): - compressobj = zlib.compressobj(wbits=-15, level=1, memLevel=9) - headers = base64.b64encode(compressobj.compress(headers) + compressobj.flush()).decode() - if start_height % 1000 != 0: - return headers - self.encoded_headers[key] = headers - return self.encoded_headers.get(key) - - async def read_headers(self, start_height, count) -> typing.Tuple[bytes, int]: - """Requires start_height >= 0, count >= 0. Reads as many headers as - are available starting at start_height up to count. This - would be zero if start_height is beyond self.db_height, for - example. - - Returns a (binary, n) pair where binary is the concatenated - binary headers, and n is the count of headers returned. - """ - - if start_height < 0 or count < 0: - raise DBError(f'{count:,d} headers starting at {start_height:,d} not on disk') - - disk_count = max(0, min(count, self.db_height + 1 - start_height)) - - def read_headers(): - x = b''.join( - self.prefix_db.header.iterate( - start=(start_height,), stop=(start_height+disk_count,), include_key=False, deserialize_value=False - ) - ) - return x - - if disk_count: - return await asyncio.get_event_loop().run_in_executor(self._executor, read_headers), disk_count - return b'', 0 - - def fs_tx_hash(self, tx_num): - """Return a par (tx_hash, tx_height) for the given tx number. - - If the tx_height is not on disk, returns (None, tx_height).""" - tx_height = bisect_right(self.tx_counts, tx_num) - if tx_height > self.db_height: - return None, tx_height - try: - return self.get_tx_hash(tx_num), tx_height - except IndexError: - self.logger.exception( - "Failed to access a cached transaction, known bug #3142 " - "should be fixed in #3205" - ) - return None, tx_height - - def get_block_txs(self, height: int) -> List[bytes]: - return self.prefix_db.block_txs.get(height).tx_hashes - - async def get_transactions_and_merkles(self, tx_hashes: Iterable[str]): - tx_infos = {} - for tx_hash in tx_hashes: - tx_infos[tx_hash] = await asyncio.get_event_loop().run_in_executor( - self._executor, self._get_transaction_and_merkle, tx_hash - ) - await asyncio.sleep(0) - return tx_infos - - def _get_transaction_and_merkle(self, tx_hash): - cached_tx = self._tx_and_merkle_cache.get(tx_hash) - if cached_tx: - tx, merkle = cached_tx - else: - tx_hash_bytes = bytes.fromhex(tx_hash)[::-1] - tx_num = self.prefix_db.tx_num.get(tx_hash_bytes) - tx = None - tx_height = -1 - tx_num = None if not tx_num else tx_num.tx_num - if tx_num is not None: - if self._cache_all_claim_txos: - fill_cache = tx_num in self.txo_to_claim and len(self.txo_to_claim[tx_num]) > 0 - else: - fill_cache = False - tx_height = bisect_right(self.tx_counts, tx_num) - tx = self.prefix_db.tx.get(tx_hash_bytes, fill_cache=fill_cache, deserialize_value=False) - if tx_height == -1: - merkle = { - 'block_height': -1 - } - else: - tx_pos = tx_num - self.tx_counts[tx_height - 1] - branch, root = self.merkle.branch_and_root( - self.get_block_txs(tx_height), tx_pos - ) - merkle = { - 'block_height': tx_height, - 'merkle': [ - hash_to_hex_str(hash) - for hash in branch - ], - 'pos': tx_pos - } - if tx_height + 10 < self.db_height: - self._tx_and_merkle_cache[tx_hash] = tx, merkle - return (None if not tx else tx.hex(), merkle) - - async def fs_block_hashes(self, height, count): - if height + count > len(self.headers): - raise DBError(f'only got {len(self.headers) - height:,d} headers starting at {height:,d}, not {count:,d}') - return [self.coin.header_hash(header) for header in self.headers[height:height + count]] - - def read_history(self, hashX: bytes, limit: int = 1000) -> List[Tuple[bytes, int]]: - txs = [] - txs_extend = txs.extend - for hist in self.prefix_db.hashX_history.iterate(prefix=(hashX,), include_key=False): - txs_extend(hist) - if len(txs) >= limit: - break - return [ - (self.get_tx_hash(tx_num), bisect_right(self.tx_counts, tx_num)) - for tx_num in txs - ] - - async def limited_history(self, hashX, *, limit=1000): - """Return an unpruned, sorted list of (tx_hash, height) tuples of - confirmed transactions that touched the address, earliest in - the blockchain first. Includes both spending and receiving - transactions. By default returns at most 1000 entries. Set - limit to None to get them all. - """ - return await asyncio.get_event_loop().run_in_executor(self._executor, self.read_history, hashX, limit) - - # -- Undo information - - def min_undo_height(self, max_height): - """Returns a height from which we should store undo info.""" - return max_height - self._reorg_limit + 1 - - def apply_expiration_extension_fork(self): - # TODO: this can't be reorged - for k, v in self.prefix_db.claim_expiration.iterate(): - self.prefix_db.claim_expiration.stage_delete(k, v) - self.prefix_db.claim_expiration.stage_put( - (bisect_right(self.tx_counts, k.tx_num) + self.coin.nExtendedClaimExpirationTime, - k.tx_num, k.position), v - ) - self.prefix_db.unsafe_commit() - - def write_db_state(self): - """Write (UTXO) state to the batch.""" - if self.db_height > 0: - self.prefix_db.db_state.stage_delete((), self.prefix_db.db_state.get()) - self.prefix_db.db_state.stage_put((), ( - self.genesis_bytes, self.db_height, self.db_tx_count, self.db_tip, - self.utxo_flush_count, int(self.wall_time), self.first_sync, self.db_version, - self.hist_flush_count, self.hist_comp_flush_count, self.hist_comp_cursor, - self.es_sync_height - ) - ) - - def read_db_state(self): - state = self.prefix_db.db_state.get() - - if not state: - self.db_height = -1 - self.db_tx_count = 0 - self.db_tip = b'\0' * 32 - self.db_version = max(self.DB_VERSIONS) - self.utxo_flush_count = 0 - self.wall_time = 0 - self.first_sync = True - self.hist_flush_count = 0 - self.hist_comp_flush_count = -1 - self.hist_comp_cursor = -1 - self.hist_db_version = max(self.DB_VERSIONS) - self.es_sync_height = 0 - else: - self.db_version = state.db_version - if self.db_version not in self.DB_VERSIONS: - raise DBError(f'your DB version is {self.db_version} but this ' - f'software only handles versions {self.DB_VERSIONS}') - # backwards compat - genesis_hash = state.genesis - if genesis_hash.hex() != self.coin.GENESIS_HASH: - raise DBError(f'DB genesis hash {genesis_hash} does not ' - f'match coin {self.coin.GENESIS_HASH}') - self.db_height = state.height - self.db_tx_count = state.tx_count - self.db_tip = state.tip - self.utxo_flush_count = state.utxo_flush_count - self.wall_time = state.wall_time - self.first_sync = state.first_sync - self.hist_flush_count = state.hist_flush_count - self.hist_comp_flush_count = state.comp_flush_count - self.hist_comp_cursor = state.comp_cursor - self.hist_db_version = state.db_version - self.es_sync_height = state.es_sync_height - return state - - def assert_db_state(self): - state = self.prefix_db.db_state.get() - assert self.db_version == state.db_version, f"{self.db_version} != {state.db_version}" - assert self.db_height == state.height, f"{self.db_height} != {state.height}" - assert self.db_tx_count == state.tx_count, f"{self.db_tx_count} != {state.tx_count}" - assert self.db_tip == state.tip, f"{self.db_tip} != {state.tip}" - assert self.first_sync == state.first_sync, f"{self.first_sync} != {state.first_sync}" - assert self.es_sync_height == state.es_sync_height, f"{self.es_sync_height} != {state.es_sync_height}" - - async def all_utxos(self, hashX): - """Return all UTXOs for an address sorted in no particular order.""" - def read_utxos(): - utxos = [] - utxos_append = utxos.append - fs_tx_hash = self.fs_tx_hash - for k, v in self.prefix_db.utxo.iterate(prefix=(hashX, )): - tx_hash, height = fs_tx_hash(k.tx_num) - utxos_append(UTXO(k.tx_num, k.nout, tx_hash, height, v.amount)) - return utxos - - while True: - utxos = await asyncio.get_event_loop().run_in_executor(self._executor, read_utxos) - if all(utxo.tx_hash is not None for utxo in utxos): - return utxos - self.logger.warning(f'all_utxos: tx hash not ' - f'found (reorg?), retrying...') - await sleep(0.25) - - async def lookup_utxos(self, prevouts): - def lookup_utxos(): - utxos = [] - utxo_append = utxos.append - for (tx_hash, nout) in prevouts: - tx_num_val = self.prefix_db.tx_num.get(tx_hash) - if not tx_num_val: - print("no tx num for ", tx_hash[::-1].hex()) - continue - tx_num = tx_num_val.tx_num - hashX_val = self.prefix_db.hashX_utxo.get(tx_hash[:4], tx_num, nout) - if not hashX_val: - continue - hashX = hashX_val.hashX - utxo_value = self.prefix_db.utxo.get(hashX, tx_num, nout) - if utxo_value: - utxo_append((hashX, utxo_value.amount)) - return utxos - return await asyncio.get_event_loop().run_in_executor(self._executor, lookup_utxos) diff --git a/lbry/wallet/server/db/elasticsearch/__init__.py b/lbry/wallet/server/db/elasticsearch/__init__.py deleted file mode 100644 index 385e96219..000000000 --- a/lbry/wallet/server/db/elasticsearch/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .search import SearchIndex \ No newline at end of file diff --git a/lbry/wallet/server/db/elasticsearch/common.py b/lbry/wallet/server/db/elasticsearch/common.py deleted file mode 100644 index 4441aad34..000000000 --- a/lbry/wallet/server/db/elasticsearch/common.py +++ /dev/null @@ -1,150 +0,0 @@ -from decimal import Decimal -from typing import Iterable - -from lbry.error import TooManyClaimSearchParametersError -from lbry.schema.tags import clean_tags -from lbry.schema.url import normalize_name -from lbry.wallet.server.db.common import CLAIM_TYPES, STREAM_TYPES -from lbry.wallet.server.db.elasticsearch.constants import REPLACEMENTS, FIELDS, TEXT_FIELDS, RANGE_FIELDS - - -def expand_query(**kwargs): - if "amount_order" in kwargs: - kwargs["limit"] = 1 - kwargs["order_by"] = "effective_amount" - kwargs["offset"] = int(kwargs["amount_order"]) - 1 - if 'name' in kwargs: - kwargs['name'] = normalize_name(kwargs.pop('name')) - if kwargs.get('is_controlling') is False: - kwargs.pop('is_controlling') - query = {'must': [], 'must_not': []} - collapse = None - if 'fee_currency' in kwargs and kwargs['fee_currency'] is not None: - kwargs['fee_currency'] = kwargs['fee_currency'].upper() - for key, value in kwargs.items(): - key = key.replace('claim.', '') - many = key.endswith('__in') or isinstance(value, list) - if many and len(value) > 2048: - raise TooManyClaimSearchParametersError(key, 2048) - if many: - key = key.replace('__in', '') - value = list(filter(None, value)) - if value is None or isinstance(value, list) and len(value) == 0: - continue - key = REPLACEMENTS.get(key, key) - if key in FIELDS: - partial_id = False - if key == 'claim_type': - if isinstance(value, str): - value = CLAIM_TYPES[value] - else: - value = [CLAIM_TYPES[claim_type] for claim_type in value] - elif key == 'stream_type': - value = [STREAM_TYPES[value]] if isinstance(value, str) else list(map(STREAM_TYPES.get, value)) - if key == '_id': - if isinstance(value, Iterable): - value = [item[::-1].hex() for item in value] - else: - value = value[::-1].hex() - if not many and key in ('_id', 'claim_id') and len(value) < 20: - partial_id = True - if key in ('signature_valid', 'has_source'): - continue # handled later - if key in TEXT_FIELDS: - key += '.keyword' - ops = {'<=': 'lte', '>=': 'gte', '<': 'lt', '>': 'gt'} - if partial_id: - query['must'].append({"prefix": {"claim_id": value}}) - elif key in RANGE_FIELDS and isinstance(value, str) and value[0] in ops: - operator_length = 2 if value[:2] in ops else 1 - operator, value = value[:operator_length], value[operator_length:] - if key == 'fee_amount': - value = str(Decimal(value)*1000) - query['must'].append({"range": {key: {ops[operator]: value}}}) - elif many: - query['must'].append({"terms": {key: value}}) - else: - if key == 'fee_amount': - value = str(Decimal(value)*1000) - query['must'].append({"term": {key: {"value": value}}}) - elif key == 'not_channel_ids': - for channel_id in value: - query['must_not'].append({"term": {'channel_id.keyword': channel_id}}) - query['must_not'].append({"term": {'_id': channel_id}}) - elif key == 'channel_ids': - query['must'].append({"terms": {'channel_id.keyword': value}}) - elif key == 'claim_ids': - query['must'].append({"terms": {'claim_id.keyword': value}}) - elif key == 'media_types': - query['must'].append({"terms": {'media_type.keyword': value}}) - elif key == 'any_languages': - query['must'].append({"terms": {'languages': clean_tags(value)}}) - elif key == 'any_languages': - query['must'].append({"terms": {'languages': value}}) - elif key == 'all_languages': - query['must'].extend([{"term": {'languages': tag}} for tag in value]) - elif key == 'any_tags': - query['must'].append({"terms": {'tags.keyword': clean_tags(value)}}) - elif key == 'all_tags': - query['must'].extend([{"term": {'tags.keyword': tag}} for tag in clean_tags(value)]) - elif key == 'not_tags': - query['must_not'].extend([{"term": {'tags.keyword': tag}} for tag in clean_tags(value)]) - elif key == 'not_claim_id': - query['must_not'].extend([{"term": {'claim_id.keyword': cid}} for cid in value]) - elif key == 'limit_claims_per_channel': - collapse = ('channel_id.keyword', value) - if kwargs.get('has_channel_signature'): - query['must'].append({"exists": {"field": "signature"}}) - if 'signature_valid' in kwargs: - query['must'].append({"term": {"is_signature_valid": bool(kwargs["signature_valid"])}}) - elif 'signature_valid' in kwargs: - query.setdefault('should', []) - query["minimum_should_match"] = 1 - query['should'].append({"bool": {"must_not": {"exists": {"field": "signature"}}}}) - query['should'].append({"term": {"is_signature_valid": bool(kwargs["signature_valid"])}}) - if 'has_source' in kwargs: - query.setdefault('should', []) - query["minimum_should_match"] = 1 - is_stream_or_repost = {"terms": {"claim_type": [CLAIM_TYPES['stream'], CLAIM_TYPES['repost']]}} - query['should'].append( - {"bool": {"must": [{"match": {"has_source": kwargs['has_source']}}, is_stream_or_repost]}}) - query['should'].append({"bool": {"must_not": [is_stream_or_repost]}}) - query['should'].append({"bool": {"must": [{"term": {"reposted_claim_type": CLAIM_TYPES['channel']}}]}}) - if kwargs.get('text'): - query['must'].append( - {"simple_query_string": - {"query": kwargs["text"], "fields": [ - "claim_name^4", "channel_name^8", "title^1", "description^.5", "author^1", "tags^.5" - ]}}) - query = { - "_source": {"excludes": ["description", "title"]}, - 'query': {'bool': query}, - "sort": [], - } - if "limit" in kwargs: - query["size"] = kwargs["limit"] - if 'offset' in kwargs: - query["from"] = kwargs["offset"] - if 'order_by' in kwargs: - if isinstance(kwargs["order_by"], str): - kwargs["order_by"] = [kwargs["order_by"]] - for value in kwargs['order_by']: - if 'trending_group' in value: - # fixme: trending_mixed is 0 for all records on variable decay, making sort slow. - continue - is_asc = value.startswith('^') - value = value[1:] if is_asc else value - value = REPLACEMENTS.get(value, value) - if value in TEXT_FIELDS: - value += '.keyword' - query['sort'].append({value: "asc" if is_asc else "desc"}) - if collapse: - query["collapse"] = { - "field": collapse[0], - "inner_hits": { - "name": collapse[0], - "size": collapse[1], - "sort": query["sort"] - } - } - return query diff --git a/lbry/wallet/server/db/elasticsearch/constants.py b/lbry/wallet/server/db/elasticsearch/constants.py deleted file mode 100644 index afdfd6fe1..000000000 --- a/lbry/wallet/server/db/elasticsearch/constants.py +++ /dev/null @@ -1,100 +0,0 @@ -INDEX_DEFAULT_SETTINGS = { - "settings": - {"analysis": - {"analyzer": { - "default": {"tokenizer": "whitespace", "filter": ["lowercase", "porter_stem"]}}}, - "index": - {"refresh_interval": -1, - "number_of_shards": 1, - "number_of_replicas": 0, - "sort": { - "field": ["trending_score", "release_time"], - "order": ["desc", "desc"] - }} - }, - "mappings": { - "properties": { - "claim_id": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text", - "index_prefixes": { - "min_chars": 1, - "max_chars": 10 - } - }, - "sd_hash": { - "fields": { - "keyword": { - "ignore_above": 96, - "type": "keyword" - } - }, - "type": "text", - "index_prefixes": { - "min_chars": 1, - "max_chars": 4 - } - }, - "height": {"type": "integer"}, - "claim_type": {"type": "byte"}, - "censor_type": {"type": "byte"}, - "trending_score": {"type": "double"}, - "release_time": {"type": "long"} - } - } -} - -FIELDS = { - '_id', - 'claim_id', 'claim_type', 'claim_name', 'normalized_name', - 'tx_id', 'tx_nout', 'tx_position', - 'short_url', 'canonical_url', - 'is_controlling', 'last_take_over_height', - 'public_key_bytes', 'public_key_id', 'claims_in_channel', - 'channel_id', 'signature', 'signature_digest', 'is_signature_valid', - 'amount', 'effective_amount', 'support_amount', - 'fee_amount', 'fee_currency', - 'height', 'creation_height', 'activation_height', 'expiration_height', - 'stream_type', 'media_type', 'censor_type', - 'title', 'author', 'description', - 'timestamp', 'creation_timestamp', - 'duration', 'release_time', - 'tags', 'languages', 'has_source', 'reposted_claim_type', - 'reposted_claim_id', 'repost_count', 'sd_hash', - 'trending_score', 'tx_num' -} - -TEXT_FIELDS = {'author', 'canonical_url', 'channel_id', 'description', 'claim_id', 'censoring_channel_id', - 'media_type', 'normalized_name', 'public_key_bytes', 'public_key_id', 'short_url', 'signature', - 'claim_name', 'signature_digest', 'title', 'tx_id', 'fee_currency', 'reposted_claim_id', - 'tags', 'sd_hash'} - -RANGE_FIELDS = { - 'height', 'creation_height', 'activation_height', 'expiration_height', - 'timestamp', 'creation_timestamp', 'duration', 'release_time', 'fee_amount', - 'tx_position', 'repost_count', 'limit_claims_per_channel', - 'amount', 'effective_amount', 'support_amount', - 'trending_score', 'censor_type', 'tx_num' -} - -ALL_FIELDS = RANGE_FIELDS | TEXT_FIELDS | FIELDS - -REPLACEMENTS = { - 'claim_name': 'normalized_name', - 'name': 'normalized_name', - 'txid': 'tx_id', - 'nout': 'tx_nout', - 'trending_group': 'trending_score', - 'trending_mixed': 'trending_score', - 'trending_global': 'trending_score', - 'trending_local': 'trending_score', - 'reposted': 'repost_count', - 'stream_types': 'stream_type', - 'media_types': 'media_type', - 'valid_channel_signature': 'is_signature_valid' -} diff --git a/lbry/wallet/server/db/elasticsearch/fast_ar_trending.py b/lbry/wallet/server/db/elasticsearch/fast_ar_trending.py deleted file mode 100644 index 2e5836b2c..000000000 --- a/lbry/wallet/server/db/elasticsearch/fast_ar_trending.py +++ /dev/null @@ -1,117 +0,0 @@ -FAST_AR_TRENDING_SCRIPT = """ -double softenLBC(double lbc) { return (Math.pow(lbc, 1.0 / 3.0)); } - -double logsumexp(double x, double y) -{ - double top; - if(x > y) - top = x; - else - top = y; - double result = top + Math.log(Math.exp(x-top) + Math.exp(y-top)); - return(result); -} - -double logdiffexp(double big, double small) -{ - return big + Math.log(1.0 - Math.exp(small - big)); -} - -double squash(double x) -{ - if(x < 0.0) -return -Math.log(1.0 - x); - else -return Math.log(x + 1.0); -} - -double unsquash(double x) -{ - if(x < 0.0) - return 1.0 - Math.exp(-x); - else - return Math.exp(x) - 1.0; -} - -double log_to_squash(double x) -{ - return logsumexp(x, 0.0); -} - -double squash_to_log(double x) -{ - //assert x > 0.0; - return logdiffexp(x, 0.0); -} - -double squashed_add(double x, double y) -{ - // squash(unsquash(x) + unsquash(y)) but avoiding overflow. - // Cases where the signs are the same - if (x < 0.0 && y < 0.0) - return -logsumexp(-x, logdiffexp(-y, 0.0)); - if (x >= 0.0 && y >= 0.0) - return logsumexp(x, logdiffexp(y, 0.0)); - // Where the signs differ - if (x >= 0.0 && y < 0.0) - if (Math.abs(x) >= Math.abs(y)) - return logsumexp(0.0, logdiffexp(x, -y)); - else - return -logsumexp(0.0, logdiffexp(-y, x)); - if (x < 0.0 && y >= 0.0) - { - // Addition is commutative, hooray for new math - return squashed_add(y, x); - } - return 0.0; -} - -double squashed_multiply(double x, double y) -{ - // squash(unsquash(x)*unsquash(y)) but avoiding overflow. - int sign; - if(x*y >= 0.0) -sign = 1; - else -sign = -1; - return sign*logsumexp(squash_to_log(Math.abs(x)) - + squash_to_log(Math.abs(y)), 0.0); -} - -// Squashed inflated units -double inflateUnits(int height) { - double timescale = 576.0; // Half life of 400 = e-folding time of a day - // by coincidence, so may as well go with it - return log_to_squash(height / timescale); -} - -double spikePower(double newAmount) { - if (newAmount < 50.0) { - return(0.5); - } else if (newAmount < 85.0) { - return(newAmount / 100.0); - } else { - return(0.85); - } -} - -double spikeMass(double oldAmount, double newAmount) { - double softenedChange = softenLBC(Math.abs(newAmount - oldAmount)); - double changeInSoftened = Math.abs(softenLBC(newAmount) - softenLBC(oldAmount)); - double power = spikePower(newAmount); - if (oldAmount > newAmount) { - -1.0 * Math.pow(changeInSoftened, power) * Math.pow(softenedChange, 1.0 - power) - } else { - Math.pow(changeInSoftened, power) * Math.pow(softenedChange, 1.0 - power) - } -} - -for (i in params.src.changes) { - double units = inflateUnits(i.height); - if (ctx._source.trending_score == null) { - ctx._source.trending_score = 0.0; - } - double bigSpike = squashed_multiply(units, squash(spikeMass(i.prev_amount, i.new_amount))); - ctx._source.trending_score = squashed_add(ctx._source.trending_score, bigSpike); -} -""" diff --git a/lbry/wallet/server/db/elasticsearch/notifier.py b/lbry/wallet/server/db/elasticsearch/notifier.py deleted file mode 100644 index 8964f0671..000000000 --- a/lbry/wallet/server/db/elasticsearch/notifier.py +++ /dev/null @@ -1,55 +0,0 @@ -import struct -import typing -import asyncio -import logging - - -log = logging.getLogger(__name__) - - -class ElasticNotifierProtocol(asyncio.Protocol): - """notifies the reader when ES has written updates""" - - def __init__(self, listeners): - self._listeners = listeners - self.transport: typing.Optional[asyncio.Transport] = None - - def connection_made(self, transport): - self.transport = transport - self._listeners.append(self) - log.info("got es notifier connection") - - def connection_lost(self, exc) -> None: - self._listeners.remove(self) - self.transport = None - - def send_height(self, height: int, block_hash: bytes): - log.info("notify es update '%s'", height) - self.transport.write(struct.pack(b'>Q32s', height, block_hash)) - - -class ElasticNotifierClientProtocol(asyncio.Protocol): - """notifies the reader when ES has written updates""" - - def __init__(self, notifications: asyncio.Queue): - self.notifications = notifications - self.transport: typing.Optional[asyncio.Transport] = None - - def close(self): - if self.transport and not self.transport.is_closing(): - self.transport.close() - - def connection_made(self, transport): - self.transport = transport - log.info("connected to es notifier") - - def connection_lost(self, exc) -> None: - self.transport = None - - def data_received(self, data: bytes) -> None: - try: - height, block_hash = struct.unpack(b'>Q32s', data) - except: - log.exception("failed to decode %s", (data or b'').hex()) - raise - self.notifications.put_nowait((height, block_hash)) diff --git a/lbry/wallet/server/db/elasticsearch/search.py b/lbry/wallet/server/db/elasticsearch/search.py deleted file mode 100644 index 859ee056c..000000000 --- a/lbry/wallet/server/db/elasticsearch/search.py +++ /dev/null @@ -1,727 +0,0 @@ -import time -import asyncio -import struct -from binascii import unhexlify -from collections import Counter, deque -from decimal import Decimal -from operator import itemgetter -from typing import Optional, List, Iterable, Union - -from elasticsearch import AsyncElasticsearch, NotFoundError, ConnectionError -from elasticsearch.helpers import async_streaming_bulk -from lbry.error import ResolveCensoredError, TooManyClaimSearchParametersError -from lbry.schema.result import Outputs, Censor -from lbry.schema.tags import clean_tags -from lbry.schema.url import URL, normalize_name -from lbry.utils import LRUCache -from lbry.wallet.server.db.common import CLAIM_TYPES, STREAM_TYPES -from lbry.wallet.server.db.elasticsearch.constants import INDEX_DEFAULT_SETTINGS, REPLACEMENTS, FIELDS, TEXT_FIELDS, \ - RANGE_FIELDS, ALL_FIELDS -from lbry.wallet.server.util import class_logger -from lbry.wallet.server.db.common import ResolveResult - - -class ChannelResolution(str): - @classmethod - def lookup_error(cls, url): - return LookupError(f'Could not find channel in "{url}".') - - -class StreamResolution(str): - @classmethod - def lookup_error(cls, url): - return LookupError(f'Could not find claim at "{url}".') - - -class IndexVersionMismatch(Exception): - def __init__(self, got_version, expected_version): - self.got_version = got_version - self.expected_version = expected_version - - -class SearchIndex: - VERSION = 1 - - def __init__(self, index_prefix: str, search_timeout=3.0, elastic_host='localhost', elastic_port=9200): - self.search_timeout = search_timeout - self.sync_timeout = 600 # wont hit that 99% of the time, but can hit on a fresh import - self.search_client: Optional[AsyncElasticsearch] = None - self.sync_client: Optional[AsyncElasticsearch] = None - self.index = index_prefix + 'claims' - self.logger = class_logger(__name__, self.__class__.__name__) - self.claim_cache = LRUCache(2 ** 15) - self.search_cache = LRUCache(2 ** 17) - self._elastic_host = elastic_host - self._elastic_port = elastic_port - - async def get_index_version(self) -> int: - try: - template = await self.sync_client.indices.get_template(self.index) - return template[self.index]['version'] - except NotFoundError: - return 0 - - async def set_index_version(self, version): - await self.sync_client.indices.put_template( - self.index, body={'version': version, 'index_patterns': ['ignored']}, ignore=400 - ) - - async def start(self) -> bool: - if self.sync_client: - return False - hosts = [{'host': self._elastic_host, 'port': self._elastic_port}] - self.sync_client = AsyncElasticsearch(hosts, timeout=self.sync_timeout) - self.search_client = AsyncElasticsearch(hosts, timeout=self.search_timeout) - while True: - try: - await self.sync_client.cluster.health(wait_for_status='yellow') - break - except ConnectionError: - self.logger.warning("Failed to connect to Elasticsearch. Waiting for it!") - await asyncio.sleep(1) - - res = await self.sync_client.indices.create(self.index, INDEX_DEFAULT_SETTINGS, ignore=400) - acked = res.get('acknowledged', False) - if acked: - await self.set_index_version(self.VERSION) - return acked - index_version = await self.get_index_version() - if index_version != self.VERSION: - self.logger.error("es search index has an incompatible version: %s vs %s", index_version, self.VERSION) - raise IndexVersionMismatch(index_version, self.VERSION) - await self.sync_client.indices.refresh(self.index) - return acked - - async def stop(self): - clients = [c for c in (self.sync_client, self.search_client) if c is not None] - self.sync_client, self.search_client = None, None - if clients: - await asyncio.gather(*(client.close() for client in clients)) - - def delete_index(self): - return self.sync_client.indices.delete(self.index, ignore_unavailable=True) - - async def _consume_claim_producer(self, claim_producer): - count = 0 - async for op, doc in claim_producer: - if op == 'delete': - yield { - '_index': self.index, - '_op_type': 'delete', - '_id': doc - } - else: - yield { - 'doc': {key: value for key, value in doc.items() if key in ALL_FIELDS}, - '_id': doc['claim_id'], - '_index': self.index, - '_op_type': 'update', - 'doc_as_upsert': True - } - count += 1 - if count % 100 == 0: - self.logger.info("Indexing in progress, %d claims.", count) - if count: - self.logger.info("Indexing done for %d claims.", count) - else: - self.logger.debug("Indexing done for %d claims.", count) - - async def claim_consumer(self, claim_producer): - touched = set() - async for ok, item in async_streaming_bulk(self.sync_client, self._consume_claim_producer(claim_producer), - raise_on_error=False): - if not ok: - self.logger.warning("indexing failed for an item: %s", item) - else: - item = item.popitem()[1] - touched.add(item['_id']) - await self.sync_client.indices.refresh(self.index) - self.logger.debug("Indexing done.") - - def update_filter_query(self, censor_type, blockdict, channels=False): - blockdict = {blocked.hex(): blocker.hex() for blocked, blocker in blockdict.items()} - if channels: - update = expand_query(channel_id__in=list(blockdict.keys()), censor_type=f"<{censor_type}") - else: - update = expand_query(claim_id__in=list(blockdict.keys()), censor_type=f"<{censor_type}") - key = 'channel_id' if channels else 'claim_id' - update['script'] = { - "source": f"ctx._source.censor_type={censor_type}; " - f"ctx._source.censoring_channel_id=params[ctx._source.{key}];", - "lang": "painless", - "params": blockdict - } - return update - - async def update_trending_score(self, params): - update_trending_score_script = """ - double softenLBC(double lbc) { return (Math.pow(lbc, 1.0 / 3.0)); } - - double logsumexp(double x, double y) - { - double top; - if(x > y) - top = x; - else - top = y; - double result = top + Math.log(Math.exp(x-top) + Math.exp(y-top)); - return(result); - } - - double logdiffexp(double big, double small) - { - return big + Math.log(1.0 - Math.exp(small - big)); - } - - double squash(double x) - { - if(x < 0.0) - return -Math.log(1.0 - x); - else - return Math.log(x + 1.0); - } - - double unsquash(double x) - { - if(x < 0.0) - return 1.0 - Math.exp(-x); - else - return Math.exp(x) - 1.0; - } - - double log_to_squash(double x) - { - return logsumexp(x, 0.0); - } - - double squash_to_log(double x) - { - //assert x > 0.0; - return logdiffexp(x, 0.0); - } - - double squashed_add(double x, double y) - { - // squash(unsquash(x) + unsquash(y)) but avoiding overflow. - // Cases where the signs are the same - if (x < 0.0 && y < 0.0) - return -logsumexp(-x, logdiffexp(-y, 0.0)); - if (x >= 0.0 && y >= 0.0) - return logsumexp(x, logdiffexp(y, 0.0)); - // Where the signs differ - if (x >= 0.0 && y < 0.0) - if (Math.abs(x) >= Math.abs(y)) - return logsumexp(0.0, logdiffexp(x, -y)); - else - return -logsumexp(0.0, logdiffexp(-y, x)); - if (x < 0.0 && y >= 0.0) - { - // Addition is commutative, hooray for new math - return squashed_add(y, x); - } - return 0.0; - } - - double squashed_multiply(double x, double y) - { - // squash(unsquash(x)*unsquash(y)) but avoiding overflow. - int sign; - if(x*y >= 0.0) - sign = 1; - else - sign = -1; - return sign*logsumexp(squash_to_log(Math.abs(x)) - + squash_to_log(Math.abs(y)), 0.0); - } - - // Squashed inflated units - double inflateUnits(int height) { - double timescale = 576.0; // Half life of 400 = e-folding time of a day - // by coincidence, so may as well go with it - return log_to_squash(height / timescale); - } - - double spikePower(double newAmount) { - if (newAmount < 50.0) { - return(0.5); - } else if (newAmount < 85.0) { - return(newAmount / 100.0); - } else { - return(0.85); - } - } - - double spikeMass(double oldAmount, double newAmount) { - double softenedChange = softenLBC(Math.abs(newAmount - oldAmount)); - double changeInSoftened = Math.abs(softenLBC(newAmount) - softenLBC(oldAmount)); - double power = spikePower(newAmount); - if (oldAmount > newAmount) { - -1.0 * Math.pow(changeInSoftened, power) * Math.pow(softenedChange, 1.0 - power) - } else { - Math.pow(changeInSoftened, power) * Math.pow(softenedChange, 1.0 - power) - } - } - for (i in params.src.changes) { - double units = inflateUnits(i.height); - if (ctx._source.trending_score == null) { - ctx._source.trending_score = 0.0; - } - double bigSpike = squashed_multiply(units, squash(spikeMass(i.prev_amount, i.new_amount))); - ctx._source.trending_score = squashed_add(ctx._source.trending_score, bigSpike); - } - """ - start = time.perf_counter() - - def producer(): - for claim_id, claim_updates in params.items(): - yield { - '_id': claim_id, - '_index': self.index, - '_op_type': 'update', - 'script': { - 'lang': 'painless', - 'source': update_trending_score_script, - 'params': {'src': { - 'changes': [ - { - 'height': p.height, - 'prev_amount': p.prev_amount / 1E8, - 'new_amount': p.new_amount / 1E8, - } for p in claim_updates - ] - }} - }, - } - if not params: - return - async for ok, item in async_streaming_bulk(self.sync_client, producer(), raise_on_error=False): - if not ok: - self.logger.warning("updating trending failed for an item: %s", item) - await self.sync_client.indices.refresh(self.index) - self.logger.info("updated trending scores in %ims", int((time.perf_counter() - start) * 1000)) - - async def apply_filters(self, blocked_streams, blocked_channels, filtered_streams, filtered_channels): - if filtered_streams: - await self.sync_client.update_by_query( - self.index, body=self.update_filter_query(Censor.SEARCH, filtered_streams), slices=4) - await self.sync_client.indices.refresh(self.index) - if filtered_channels: - await self.sync_client.update_by_query( - self.index, body=self.update_filter_query(Censor.SEARCH, filtered_channels), slices=4) - await self.sync_client.indices.refresh(self.index) - await self.sync_client.update_by_query( - self.index, body=self.update_filter_query(Censor.SEARCH, filtered_channels, True), slices=4) - await self.sync_client.indices.refresh(self.index) - if blocked_streams: - await self.sync_client.update_by_query( - self.index, body=self.update_filter_query(Censor.RESOLVE, blocked_streams), slices=4) - await self.sync_client.indices.refresh(self.index) - if blocked_channels: - await self.sync_client.update_by_query( - self.index, body=self.update_filter_query(Censor.RESOLVE, blocked_channels), slices=4) - await self.sync_client.indices.refresh(self.index) - await self.sync_client.update_by_query( - self.index, body=self.update_filter_query(Censor.RESOLVE, blocked_channels, True), slices=4) - await self.sync_client.indices.refresh(self.index) - self.clear_caches() - - def clear_caches(self): - self.search_cache.clear() - self.claim_cache.clear() - - async def cached_search(self, kwargs): - total_referenced = [] - cache_item = ResultCacheItem.from_cache(str(kwargs), self.search_cache) - if cache_item.result is not None: - return cache_item.result - async with cache_item.lock: - if cache_item.result: - return cache_item.result - censor = Censor(Censor.SEARCH) - if kwargs.get('no_totals'): - response, offset, total = await self.search(**kwargs, censor_type=Censor.NOT_CENSORED) - else: - response, offset, total = await self.search(**kwargs) - censor.apply(response) - total_referenced.extend(response) - - if censor.censored: - response, _, _ = await self.search(**kwargs, censor_type=Censor.NOT_CENSORED) - total_referenced.extend(response) - response = [ - ResolveResult( - name=r['claim_name'], - normalized_name=r['normalized_name'], - claim_hash=r['claim_hash'], - tx_num=r['tx_num'], - position=r['tx_nout'], - tx_hash=r['tx_hash'], - height=r['height'], - amount=r['amount'], - short_url=r['short_url'], - is_controlling=r['is_controlling'], - canonical_url=r['canonical_url'], - creation_height=r['creation_height'], - activation_height=r['activation_height'], - expiration_height=r['expiration_height'], - effective_amount=r['effective_amount'], - support_amount=r['support_amount'], - last_takeover_height=r['last_take_over_height'], - claims_in_channel=r['claims_in_channel'], - channel_hash=r['channel_hash'], - reposted_claim_hash=r['reposted_claim_hash'], - reposted=r['reposted'], - signature_valid=r['signature_valid'] - ) for r in response - ] - extra = [ - ResolveResult( - name=r['claim_name'], - normalized_name=r['normalized_name'], - claim_hash=r['claim_hash'], - tx_num=r['tx_num'], - position=r['tx_nout'], - tx_hash=r['tx_hash'], - height=r['height'], - amount=r['amount'], - short_url=r['short_url'], - is_controlling=r['is_controlling'], - canonical_url=r['canonical_url'], - creation_height=r['creation_height'], - activation_height=r['activation_height'], - expiration_height=r['expiration_height'], - effective_amount=r['effective_amount'], - support_amount=r['support_amount'], - last_takeover_height=r['last_take_over_height'], - claims_in_channel=r['claims_in_channel'], - channel_hash=r['channel_hash'], - reposted_claim_hash=r['reposted_claim_hash'], - reposted=r['reposted'], - signature_valid=r['signature_valid'] - ) for r in await self._get_referenced_rows(total_referenced) - ] - result = Outputs.to_base64( - response, extra, offset, total, censor - ) - cache_item.result = result - return result - - async def get_many(self, *claim_ids): - await self.populate_claim_cache(*claim_ids) - return filter(None, map(self.claim_cache.get, claim_ids)) - - async def populate_claim_cache(self, *claim_ids): - missing = [claim_id for claim_id in claim_ids if self.claim_cache.get(claim_id) is None] - if missing: - results = await self.search_client.mget( - index=self.index, body={"ids": missing} - ) - for result in expand_result(filter(lambda doc: doc['found'], results["docs"])): - self.claim_cache.set(result['claim_id'], result) - - - async def search(self, **kwargs): - try: - return await self.search_ahead(**kwargs) - except NotFoundError: - return [], 0, 0 - # return expand_result(result['hits']), 0, result.get('total', {}).get('value', 0) - - async def search_ahead(self, **kwargs): - # 'limit_claims_per_channel' case. Fetch 1000 results, reorder, slice, inflate and return - per_channel_per_page = kwargs.pop('limit_claims_per_channel', 0) or 0 - remove_duplicates = kwargs.pop('remove_duplicates', False) - page_size = kwargs.pop('limit', 10) - offset = kwargs.pop('offset', 0) - kwargs['limit'] = 1000 - cache_item = ResultCacheItem.from_cache(f"ahead{per_channel_per_page}{kwargs}", self.search_cache) - if cache_item.result is not None: - reordered_hits = cache_item.result - else: - async with cache_item.lock: - if cache_item.result: - reordered_hits = cache_item.result - else: - query = expand_query(**kwargs) - search_hits = deque((await self.search_client.search( - query, index=self.index, track_total_hits=False, - _source_includes=['_id', 'channel_id', 'reposted_claim_id', 'creation_height'] - ))['hits']['hits']) - if remove_duplicates: - search_hits = self.__remove_duplicates(search_hits) - if per_channel_per_page > 0: - reordered_hits = self.__search_ahead(search_hits, page_size, per_channel_per_page) - else: - reordered_hits = [(hit['_id'], hit['_source']['channel_id']) for hit in search_hits] - cache_item.result = reordered_hits - result = list(await self.get_many(*(claim_id for claim_id, _ in reordered_hits[offset:(offset + page_size)]))) - return result, 0, len(reordered_hits) - - def __remove_duplicates(self, search_hits: deque) -> deque: - known_ids = {} # claim_id -> (creation_height, hit_id), where hit_id is either reposted claim id or original - dropped = set() - for hit in search_hits: - hit_height, hit_id = hit['_source']['creation_height'], hit['_source']['reposted_claim_id'] or hit['_id'] - if hit_id not in known_ids: - known_ids[hit_id] = (hit_height, hit['_id']) - else: - previous_height, previous_id = known_ids[hit_id] - if hit_height < previous_height: - known_ids[hit_id] = (hit_height, hit['_id']) - dropped.add(previous_id) - else: - dropped.add(hit['_id']) - return deque(hit for hit in search_hits if hit['_id'] not in dropped) - - def __search_ahead(self, search_hits: list, page_size: int, per_channel_per_page: int): - reordered_hits = [] - channel_counters = Counter() - next_page_hits_maybe_check_later = deque() - while search_hits or next_page_hits_maybe_check_later: - if reordered_hits and len(reordered_hits) % page_size == 0: - channel_counters.clear() - elif not reordered_hits: - pass - else: - break # means last page was incomplete and we are left with bad replacements - for _ in range(len(next_page_hits_maybe_check_later)): - claim_id, channel_id = next_page_hits_maybe_check_later.popleft() - if per_channel_per_page > 0 and channel_counters[channel_id] < per_channel_per_page: - reordered_hits.append((claim_id, channel_id)) - channel_counters[channel_id] += 1 - else: - next_page_hits_maybe_check_later.append((claim_id, channel_id)) - while search_hits: - hit = search_hits.popleft() - hit_id, hit_channel_id = hit['_id'], hit['_source']['channel_id'] - if hit_channel_id is None or per_channel_per_page <= 0: - reordered_hits.append((hit_id, hit_channel_id)) - elif channel_counters[hit_channel_id] < per_channel_per_page: - reordered_hits.append((hit_id, hit_channel_id)) - channel_counters[hit_channel_id] += 1 - if len(reordered_hits) % page_size == 0: - break - else: - next_page_hits_maybe_check_later.append((hit_id, hit_channel_id)) - return reordered_hits - - async def _get_referenced_rows(self, txo_rows: List[dict]): - txo_rows = [row for row in txo_rows if isinstance(row, dict)] - referenced_ids = set(filter(None, map(itemgetter('reposted_claim_id'), txo_rows))) - referenced_ids |= set(filter(None, (row['channel_id'] for row in txo_rows))) - referenced_ids |= set(filter(None, (row['censoring_channel_id'] for row in txo_rows))) - - referenced_txos = [] - if referenced_ids: - referenced_txos.extend(await self.get_many(*referenced_ids)) - referenced_ids = set(filter(None, (row['channel_id'] for row in referenced_txos))) - - if referenced_ids: - referenced_txos.extend(await self.get_many(*referenced_ids)) - - return referenced_txos - - -def expand_query(**kwargs): - if "amount_order" in kwargs: - kwargs["limit"] = 1 - kwargs["order_by"] = "effective_amount" - kwargs["offset"] = int(kwargs["amount_order"]) - 1 - if 'name' in kwargs: - kwargs['name'] = normalize_name(kwargs.pop('name')) - if kwargs.get('is_controlling') is False: - kwargs.pop('is_controlling') - query = {'must': [], 'must_not': []} - collapse = None - if 'fee_currency' in kwargs and kwargs['fee_currency'] is not None: - kwargs['fee_currency'] = kwargs['fee_currency'].upper() - for key, value in kwargs.items(): - key = key.replace('claim.', '') - many = key.endswith('__in') or isinstance(value, list) - if many and len(value) > 2048: - raise TooManyClaimSearchParametersError(key, 2048) - if many: - key = key.replace('__in', '') - value = list(filter(None, value)) - if value is None or isinstance(value, list) and len(value) == 0: - continue - key = REPLACEMENTS.get(key, key) - if key in FIELDS: - partial_id = False - if key == 'claim_type': - if isinstance(value, str): - value = CLAIM_TYPES[value] - else: - value = [CLAIM_TYPES[claim_type] for claim_type in value] - elif key == 'stream_type': - value = [STREAM_TYPES[value]] if isinstance(value, str) else list(map(STREAM_TYPES.get, value)) - if key == '_id': - if isinstance(value, Iterable): - value = [item[::-1].hex() for item in value] - else: - value = value[::-1].hex() - if not many and key in ('_id', 'claim_id', 'sd_hash') and len(value) < 20: - partial_id = True - if key in ('signature_valid', 'has_source'): - continue # handled later - if key in TEXT_FIELDS: - key += '.keyword' - ops = {'<=': 'lte', '>=': 'gte', '<': 'lt', '>': 'gt'} - if partial_id: - query['must'].append({"prefix": {key: value}}) - elif key in RANGE_FIELDS and isinstance(value, str) and value[0] in ops: - operator_length = 2 if value[:2] in ops else 1 - operator, value = value[:operator_length], value[operator_length:] - if key == 'fee_amount': - value = str(Decimal(value)*1000) - query['must'].append({"range": {key: {ops[operator]: value}}}) - elif key in RANGE_FIELDS and isinstance(value, list) and all(v[0] in ops for v in value): - range_constraints = [] - for v in value: - operator_length = 2 if v[:2] in ops else 1 - operator, stripped_op_v = v[:operator_length], v[operator_length:] - if key == 'fee_amount': - stripped_op_v = str(Decimal(stripped_op_v)*1000) - range_constraints.append((operator, stripped_op_v)) - query['must'].append({"range": {key: {ops[operator]: v for operator, v in range_constraints}}}) - elif many: - query['must'].append({"terms": {key: value}}) - else: - if key == 'fee_amount': - value = str(Decimal(value)*1000) - query['must'].append({"term": {key: {"value": value}}}) - elif key == 'not_channel_ids': - for channel_id in value: - query['must_not'].append({"term": {'channel_id.keyword': channel_id}}) - query['must_not'].append({"term": {'_id': channel_id}}) - elif key == 'channel_ids': - query['must'].append({"terms": {'channel_id.keyword': value}}) - elif key == 'claim_ids': - query['must'].append({"terms": {'claim_id.keyword': value}}) - elif key == 'media_types': - query['must'].append({"terms": {'media_type.keyword': value}}) - elif key == 'any_languages': - query['must'].append({"terms": {'languages': clean_tags(value)}}) - elif key == 'any_languages': - query['must'].append({"terms": {'languages': value}}) - elif key == 'all_languages': - query['must'].extend([{"term": {'languages': tag}} for tag in value]) - elif key == 'any_tags': - query['must'].append({"terms": {'tags.keyword': clean_tags(value)}}) - elif key == 'all_tags': - query['must'].extend([{"term": {'tags.keyword': tag}} for tag in clean_tags(value)]) - elif key == 'not_tags': - query['must_not'].extend([{"term": {'tags.keyword': tag}} for tag in clean_tags(value)]) - elif key == 'not_claim_id': - query['must_not'].extend([{"term": {'claim_id.keyword': cid}} for cid in value]) - elif key == 'limit_claims_per_channel': - collapse = ('channel_id.keyword', value) - if kwargs.get('has_channel_signature'): - query['must'].append({"exists": {"field": "signature"}}) - if 'signature_valid' in kwargs: - query['must'].append({"term": {"is_signature_valid": bool(kwargs["signature_valid"])}}) - elif 'signature_valid' in kwargs: - query.setdefault('should', []) - query["minimum_should_match"] = 1 - query['should'].append({"bool": {"must_not": {"exists": {"field": "signature"}}}}) - query['should'].append({"term": {"is_signature_valid": bool(kwargs["signature_valid"])}}) - if 'has_source' in kwargs: - query.setdefault('should', []) - query["minimum_should_match"] = 1 - is_stream_or_repost = {"terms": {"claim_type": [CLAIM_TYPES['stream'], CLAIM_TYPES['repost']]}} - query['should'].append( - {"bool": {"must": [{"match": {"has_source": kwargs['has_source']}}, is_stream_or_repost]}}) - query['should'].append({"bool": {"must_not": [is_stream_or_repost]}}) - query['should'].append({"bool": {"must": [{"term": {"reposted_claim_type": CLAIM_TYPES['channel']}}]}}) - if kwargs.get('text'): - query['must'].append( - {"simple_query_string": - {"query": kwargs["text"], "fields": [ - "claim_name^4", "channel_name^8", "title^1", "description^.5", "author^1", "tags^.5" - ]}}) - query = { - "_source": {"excludes": ["description", "title"]}, - 'query': {'bool': query}, - "sort": [], - } - if "limit" in kwargs: - query["size"] = kwargs["limit"] - if 'offset' in kwargs: - query["from"] = kwargs["offset"] - if 'order_by' in kwargs: - if isinstance(kwargs["order_by"], str): - kwargs["order_by"] = [kwargs["order_by"]] - for value in kwargs['order_by']: - if 'trending_group' in value: - # fixme: trending_mixed is 0 for all records on variable decay, making sort slow. - continue - is_asc = value.startswith('^') - value = value[1:] if is_asc else value - value = REPLACEMENTS.get(value, value) - if value in TEXT_FIELDS: - value += '.keyword' - query['sort'].append({value: "asc" if is_asc else "desc"}) - if collapse: - query["collapse"] = { - "field": collapse[0], - "inner_hits": { - "name": collapse[0], - "size": collapse[1], - "sort": query["sort"] - } - } - return query - - -def expand_result(results): - inner_hits = [] - expanded = [] - for result in results: - if result.get("inner_hits"): - for _, inner_hit in result["inner_hits"].items(): - inner_hits.extend(inner_hit["hits"]["hits"]) - continue - result = result['_source'] - result['claim_hash'] = unhexlify(result['claim_id'])[::-1] - if result['reposted_claim_id']: - result['reposted_claim_hash'] = unhexlify(result['reposted_claim_id'])[::-1] - else: - result['reposted_claim_hash'] = None - result['channel_hash'] = unhexlify(result['channel_id'])[::-1] if result['channel_id'] else None - result['txo_hash'] = unhexlify(result['tx_id'])[::-1] + struct.pack(' str: - return self._result - - @result.setter - def result(self, result: str): - self._result = result - if result is not None: - self.has_result.set() - - @classmethod - def from_cache(cls, cache_key, cache): - cache_item = cache.get(cache_key) - if cache_item is None: - cache_item = cache[cache_key] = ResultCacheItem() - return cache_item diff --git a/lbry/wallet/server/db/elasticsearch/sync.py b/lbry/wallet/server/db/elasticsearch/sync.py deleted file mode 100644 index 070db8270..000000000 --- a/lbry/wallet/server/db/elasticsearch/sync.py +++ /dev/null @@ -1,416 +0,0 @@ -import os -import time -import signal -import json -import typing -from collections import defaultdict -import asyncio -import logging -from elasticsearch import AsyncElasticsearch, NotFoundError -from elasticsearch.helpers import async_streaming_bulk -from prometheus_client import Gauge, Histogram - -from lbry.schema.result import Censor -from lbry.wallet.server.db.elasticsearch.search import IndexVersionMismatch -from lbry.wallet.server.db.elasticsearch.constants import ALL_FIELDS, INDEX_DEFAULT_SETTINGS -from lbry.wallet.server.db.elasticsearch.common import expand_query -from lbry.wallet.server.db.elasticsearch.notifier import ElasticNotifierProtocol -from lbry.wallet.server.db.elasticsearch.fast_ar_trending import FAST_AR_TRENDING_SCRIPT -from lbry.wallet.server.chain_reader import BlockchainReader, HISTOGRAM_BUCKETS -from lbry.wallet.server.db.revertable import RevertableOp -from lbry.wallet.server.db.common import TrendingNotification -from lbry.wallet.server.db import DB_PREFIXES - - -log = logging.getLogger() - - -class ElasticWriter(BlockchainReader): - VERSION = 1 - prometheus_namespace = "" - block_count_metric = Gauge( - "block_count", "Number of processed blocks", namespace="elastic_sync" - ) - block_update_time_metric = Histogram( - "block_time", "Block update times", namespace="elastic_sync", buckets=HISTOGRAM_BUCKETS - ) - reorg_count_metric = Gauge( - "reorg_count", "Number of reorgs", namespace="elastic_sync" - ) - - def __init__(self, env): - super().__init__(env, 'lbry-elastic-writer', thread_workers=1, thread_prefix='lbry-elastic-writer') - # self._refresh_interval = 0.1 - self._task = None - self.index = self.env.es_index_prefix + 'claims' - self._elastic_host = env.elastic_host - self._elastic_port = env.elastic_port - self.sync_timeout = 1800 - self.sync_client = None - self._es_info_path = os.path.join(env.db_dir, 'es_info') - self._last_wrote_height = 0 - self._last_wrote_block_hash = None - - self._touched_claims = set() - self._deleted_claims = set() - - self._removed_during_undo = set() - - self._trending = defaultdict(list) - self._advanced = True - self.synchronized = asyncio.Event() - self._listeners: typing.List[ElasticNotifierProtocol] = [] - - async def run_es_notifier(self, synchronized: asyncio.Event): - server = await asyncio.get_event_loop().create_server( - lambda: ElasticNotifierProtocol(self._listeners), '127.0.0.1', self.env.elastic_notifier_port - ) - self.log.info("ES notifier server listening on TCP localhost:%i", self.env.elastic_notifier_port) - synchronized.set() - async with server: - await server.serve_forever() - - def notify_es_notification_listeners(self, height: int, block_hash: bytes): - for p in self._listeners: - p.send_height(height, block_hash) - self.log.info("notify listener %i", height) - - def _read_es_height(self): - info = {} - if os.path.exists(self._es_info_path): - with open(self._es_info_path, 'r') as f: - info.update(json.loads(f.read())) - self._last_wrote_height = int(info.get('height', 0)) - self._last_wrote_block_hash = info.get('block_hash', None) - - async def read_es_height(self): - await asyncio.get_event_loop().run_in_executor(self._executor, self._read_es_height) - - def write_es_height(self, height: int, block_hash: str): - with open(self._es_info_path, 'w') as f: - f.write(json.dumps({'height': height, 'block_hash': block_hash}, indent=2)) - self._last_wrote_height = height - self._last_wrote_block_hash = block_hash - - async def get_index_version(self) -> int: - try: - template = await self.sync_client.indices.get_template(self.index) - return template[self.index]['version'] - except NotFoundError: - return 0 - - async def set_index_version(self, version): - await self.sync_client.indices.put_template( - self.index, body={'version': version, 'index_patterns': ['ignored']}, ignore=400 - ) - - async def start_index(self) -> bool: - if self.sync_client: - return False - hosts = [{'host': self._elastic_host, 'port': self._elastic_port}] - self.sync_client = AsyncElasticsearch(hosts, timeout=self.sync_timeout) - while True: - try: - await self.sync_client.cluster.health(wait_for_status='yellow') - self.log.info("ES is ready to connect to") - break - except ConnectionError: - self.log.warning("Failed to connect to Elasticsearch. Waiting for it!") - await asyncio.sleep(1) - - index_version = await self.get_index_version() - - res = await self.sync_client.indices.create(self.index, INDEX_DEFAULT_SETTINGS, ignore=400) - acked = res.get('acknowledged', False) - - if acked: - await self.set_index_version(self.VERSION) - return True - elif index_version != self.VERSION: - self.log.error("es search index has an incompatible version: %s vs %s", index_version, self.VERSION) - raise IndexVersionMismatch(index_version, self.VERSION) - else: - await self.sync_client.indices.refresh(self.index) - return False - - async def stop_index(self): - if self.sync_client: - await self.sync_client.close() - self.sync_client = None - - async def delete_index(self): - if self.sync_client: - return await self.sync_client.indices.delete(self.index, ignore_unavailable=True) - - def update_filter_query(self, censor_type, blockdict, channels=False): - blockdict = {blocked.hex(): blocker.hex() for blocked, blocker in blockdict.items()} - if channels: - update = expand_query(channel_id__in=list(blockdict.keys()), censor_type=f"<{censor_type}") - else: - update = expand_query(claim_id__in=list(blockdict.keys()), censor_type=f"<{censor_type}") - key = 'channel_id' if channels else 'claim_id' - update['script'] = { - "source": f"ctx._source.censor_type={censor_type}; " - f"ctx._source.censoring_channel_id=params[ctx._source.{key}];", - "lang": "painless", - "params": blockdict - } - return update - - async def apply_filters(self, blocked_streams, blocked_channels, filtered_streams, filtered_channels): - if filtered_streams: - await self.sync_client.update_by_query( - self.index, body=self.update_filter_query(Censor.SEARCH, filtered_streams), slices=4) - await self.sync_client.indices.refresh(self.index) - if filtered_channels: - await self.sync_client.update_by_query( - self.index, body=self.update_filter_query(Censor.SEARCH, filtered_channels), slices=4) - await self.sync_client.indices.refresh(self.index) - await self.sync_client.update_by_query( - self.index, body=self.update_filter_query(Censor.SEARCH, filtered_channels, True), slices=4) - await self.sync_client.indices.refresh(self.index) - if blocked_streams: - await self.sync_client.update_by_query( - self.index, body=self.update_filter_query(Censor.RESOLVE, blocked_streams), slices=4) - await self.sync_client.indices.refresh(self.index) - if blocked_channels: - await self.sync_client.update_by_query( - self.index, body=self.update_filter_query(Censor.RESOLVE, blocked_channels), slices=4) - await self.sync_client.indices.refresh(self.index) - await self.sync_client.update_by_query( - self.index, body=self.update_filter_query(Censor.RESOLVE, blocked_channels, True), slices=4) - await self.sync_client.indices.refresh(self.index) - - @staticmethod - def _upsert_claim_query(index, claim): - return { - 'doc': {key: value for key, value in claim.items() if key in ALL_FIELDS}, - '_id': claim['claim_id'], - '_index': index, - '_op_type': 'update', - 'doc_as_upsert': True - } - - @staticmethod - def _delete_claim_query(index, claim_hash: bytes): - return { - '_index': index, - '_op_type': 'delete', - '_id': claim_hash.hex() - } - - @staticmethod - def _update_trending_query(index, claim_hash, notifications): - return { - '_id': claim_hash.hex(), - '_index': index, - '_op_type': 'update', - 'script': { - 'lang': 'painless', - 'source': FAST_AR_TRENDING_SCRIPT, - 'params': {'src': { - 'changes': [ - { - 'height': notification.height, - 'prev_amount': notification.prev_amount / 1E8, - 'new_amount': notification.new_amount / 1E8, - } for notification in notifications - ] - }} - }, - } - - async def _claim_producer(self): - for deleted in self._deleted_claims: - yield self._delete_claim_query(self.index, deleted) - for touched in self._touched_claims: - claim = self.db.claim_producer(touched) - if claim: - yield self._upsert_claim_query(self.index, claim) - for claim_hash, notifications in self._trending.items(): - yield self._update_trending_query(self.index, claim_hash, notifications) - - def advance(self, height: int): - super().advance(height) - - touched_or_deleted = self.db.prefix_db.touched_or_deleted.get(height) - for k, v in self.db.prefix_db.trending_notification.iterate((height,)): - self._trending[k.claim_hash].append(TrendingNotification(k.height, v.previous_amount, v.new_amount)) - if touched_or_deleted: - readded_after_reorg = self._removed_during_undo.intersection(touched_or_deleted.touched_claims) - self._deleted_claims.difference_update(readded_after_reorg) - self._touched_claims.update(touched_or_deleted.touched_claims) - self._deleted_claims.update(touched_or_deleted.deleted_claims) - self._touched_claims.difference_update(self._deleted_claims) - for to_del in touched_or_deleted.deleted_claims: - if to_del in self._trending: - self._trending.pop(to_del) - self._advanced = True - - def unwind(self): - self.db.tx_counts.pop() - reverted_block_hash = self.db.coin.header_hash(self.db.headers.pop()) - packed = self.db.prefix_db.undo.get(len(self.db.tx_counts), reverted_block_hash) - touched_or_deleted = None - claims_to_delete = [] - # find and apply the touched_or_deleted items in the undos for the reverted blocks - assert packed, f'missing undo information for block {len(self.db.tx_counts)}' - while packed: - op, packed = RevertableOp.unpack(packed) - if op.is_delete and op.key.startswith(DB_PREFIXES.touched_or_deleted.value): - assert touched_or_deleted is None, 'only should have one match' - touched_or_deleted = self.db.prefix_db.touched_or_deleted.unpack_value(op.value) - elif op.is_delete and op.key.startswith(DB_PREFIXES.claim_to_txo.value): - v = self.db.prefix_db.claim_to_txo.unpack_value(op.value) - if v.root_tx_num == v.tx_num and v.root_tx_num > self.db.tx_counts[-1]: - claims_to_delete.append(self.db.prefix_db.claim_to_txo.unpack_key(op.key).claim_hash) - if touched_or_deleted: - self._touched_claims.update(set(touched_or_deleted.deleted_claims).union( - touched_or_deleted.touched_claims.difference(set(claims_to_delete)))) - self._deleted_claims.update(claims_to_delete) - self._removed_during_undo.update(claims_to_delete) - self._advanced = True - self.log.warning("delete %i claim and upsert %i from reorg", len(self._deleted_claims), len(self._touched_claims)) - - async def poll_for_changes(self): - await super().poll_for_changes() - cnt = 0 - success = 0 - if self._advanced: - if self._touched_claims or self._deleted_claims or self._trending: - async for ok, item in async_streaming_bulk( - self.sync_client, self._claim_producer(), - raise_on_error=False): - cnt += 1 - if not ok: - self.log.warning("indexing failed for an item: %s", item) - else: - success += 1 - await self.sync_client.indices.refresh(self.index) - await self.db.reload_blocking_filtering_streams() - await self.apply_filters( - self.db.blocked_streams, self.db.blocked_channels, self.db.filtered_streams, - self.db.filtered_channels - ) - self.write_es_height(self.db.db_height, self.db.db_tip[::-1].hex()) - self.log.info("Indexing block %i done. %i/%i successful", self._last_wrote_height, success, cnt) - self._touched_claims.clear() - self._deleted_claims.clear() - self._removed_during_undo.clear() - self._trending.clear() - self._advanced = False - self.synchronized.set() - self.notify_es_notification_listeners(self._last_wrote_height, self.db.db_tip) - - @property - def last_synced_height(self) -> int: - return self._last_wrote_height - - async def start(self, reindex=False): - await super().start() - - def _start_cancellable(run, *args): - _flag = asyncio.Event() - self.cancellable_tasks.append(asyncio.ensure_future(run(*args, _flag))) - return _flag.wait() - - self.db.open_db() - await self.db.initialize_caches() - await self.read_es_height() - await self.start_index() - self.last_state = self.db.read_db_state() - - await _start_cancellable(self.run_es_notifier) - - if reindex or self._last_wrote_height == 0 and self.db.db_height > 0: - if self._last_wrote_height == 0: - self.log.info("running initial ES indexing of rocksdb at block height %i", self.db.db_height) - else: - self.log.info("reindex (last wrote: %i, db height: %i)", self._last_wrote_height, self.db.db_height) - await self.reindex() - await _start_cancellable(self.refresh_blocks_forever) - - async def stop(self, delete_index=False): - async with self._lock: - while self.cancellable_tasks: - t = self.cancellable_tasks.pop() - if not t.done(): - t.cancel() - if delete_index: - await self.delete_index() - await self.stop_index() - self._executor.shutdown(wait=True) - self._executor = None - self.shutdown_event.set() - - def run(self, reindex=False): - loop = asyncio.get_event_loop() - loop.set_default_executor(self._executor) - - def __exit(): - raise SystemExit() - try: - loop.add_signal_handler(signal.SIGINT, __exit) - loop.add_signal_handler(signal.SIGTERM, __exit) - loop.run_until_complete(self.start(reindex=reindex)) - loop.run_until_complete(self.shutdown_event.wait()) - except (SystemExit, KeyboardInterrupt): - pass - finally: - loop.run_until_complete(self.stop()) - - async def reindex(self): - async with self._lock: - self.log.info("reindexing %i claims (estimate)", self.db.prefix_db.claim_to_txo.estimate_num_keys()) - await self.delete_index() - res = await self.sync_client.indices.create(self.index, INDEX_DEFAULT_SETTINGS, ignore=400) - acked = res.get('acknowledged', False) - if acked: - await self.set_index_version(self.VERSION) - await self.sync_client.indices.refresh(self.index) - self.write_es_height(0, self.env.coin.GENESIS_HASH) - await self._sync_all_claims() - await self.sync_client.indices.refresh(self.index) - self.write_es_height(self.db.db_height, self.db.db_tip[::-1].hex()) - self.notify_es_notification_listeners(self.db.db_height, self.db.db_tip) - self.log.info("finished reindexing") - - async def _sync_all_claims(self, batch_size=100000): - def load_historic_trending(): - notifications = self._trending - for k, v in self.db.prefix_db.trending_notification.iterate(): - notifications[k.claim_hash].append(TrendingNotification(k.height, v.previous_amount, v.new_amount)) - - async def all_claims_producer(): - async for claim in self.db.all_claims_producer(batch_size=batch_size): - yield self._upsert_claim_query(self.index, claim) - claim_hash = bytes.fromhex(claim['claim_id']) - if claim_hash in self._trending: - yield self._update_trending_query(self.index, claim_hash, self._trending.pop(claim_hash)) - self._trending.clear() - - self.log.info("loading about %i historic trending updates", self.db.prefix_db.trending_notification.estimate_num_keys()) - await asyncio.get_event_loop().run_in_executor(self._executor, load_historic_trending) - self.log.info("loaded historic trending updates for %i claims", len(self._trending)) - - cnt = 0 - success = 0 - producer = all_claims_producer() - - finished = False - try: - async for ok, item in async_streaming_bulk(self.sync_client, producer, raise_on_error=False): - cnt += 1 - if not ok: - self.log.warning("indexing failed for an item: %s", item) - else: - success += 1 - if cnt % batch_size == 0: - self.log.info(f"indexed {success} claims") - finished = True - await self.sync_client.indices.refresh(self.index) - self.log.info("indexed %i/%i claims", success, cnt) - finally: - if not finished: - await producer.aclose() - self.shutdown_event.set() diff --git a/lbry/wallet/server/db/interface.py b/lbry/wallet/server/db/interface.py deleted file mode 100644 index 53fec1639..000000000 --- a/lbry/wallet/server/db/interface.py +++ /dev/null @@ -1,145 +0,0 @@ -import struct -import typing - -import rocksdb -from typing import Optional -from lbry.wallet.server.db import DB_PREFIXES, COLUMN_SETTINGS -from lbry.wallet.server.db.revertable import RevertableOpStack, RevertablePut, RevertableDelete - - -class BasePrefixDB: - """ - Base class for a revertable rocksdb database (a rocksdb db where each set of applied changes can be undone) - """ - UNDO_KEY_STRUCT = struct.Struct(b'>Q32s') - PARTIAL_UNDO_KEY_STRUCT = struct.Struct(b'>Q') - - def __init__(self, path, max_open_files=64, secondary_path='', max_undo_depth: int = 200, unsafe_prefixes=None): - column_family_options = {} - for prefix in DB_PREFIXES: - settings = COLUMN_SETTINGS[prefix.value] - column_family_options[prefix.value] = rocksdb.ColumnFamilyOptions() - column_family_options[prefix.value].table_factory = rocksdb.BlockBasedTableFactory( - block_cache=rocksdb.LRUCache(settings['cache_size']), - ) - self.column_families: typing.Dict[bytes, 'rocksdb.ColumnFamilyHandle'] = {} - options = rocksdb.Options( - create_if_missing=True, use_fsync=False, target_file_size_base=33554432, - max_open_files=max_open_files if not secondary_path else -1, create_missing_column_families=True - ) - self._db = rocksdb.DB( - path, options, secondary_name=secondary_path, column_families=column_family_options - ) - for prefix in DB_PREFIXES: - cf = self._db.get_column_family(prefix.value) - if cf is None and not secondary_path: - self._db.create_column_family(prefix.value, column_family_options[prefix.value]) - cf = self._db.get_column_family(prefix.value) - self.column_families[prefix.value] = cf - - self._op_stack = RevertableOpStack(self.get, unsafe_prefixes=unsafe_prefixes) - self._max_undo_depth = max_undo_depth - - def unsafe_commit(self): - """ - Write staged changes to the database without keeping undo information - Changes written cannot be undone - """ - try: - if not len(self._op_stack): - return - with self._db.write_batch(sync=True) as batch: - batch_put = batch.put - batch_delete = batch.delete - get_column_family = self.column_families.__getitem__ - for staged_change in self._op_stack: - column_family = get_column_family(DB_PREFIXES(staged_change.key[:1]).value) - if staged_change.is_put: - batch_put((column_family, staged_change.key), staged_change.value) - else: - batch_delete((column_family, staged_change.key)) - finally: - self._op_stack.clear() - - def commit(self, height: int, block_hash: bytes): - """ - Write changes for a block height to the database and keep undo information so that the changes can be reverted - """ - undo_ops = self._op_stack.get_undo_ops() - delete_undos = [] - if height > self._max_undo_depth: - delete_undos.extend(self._db.iterator( - start=DB_PREFIXES.undo.value + self.PARTIAL_UNDO_KEY_STRUCT.pack(0), - iterate_upper_bound=DB_PREFIXES.undo.value + self.PARTIAL_UNDO_KEY_STRUCT.pack(height - self._max_undo_depth), - include_value=False - )) - try: - undo_c_f = self.column_families[DB_PREFIXES.undo.value] - with self._db.write_batch(sync=True) as batch: - batch_put = batch.put - batch_delete = batch.delete - get_column_family = self.column_families.__getitem__ - for staged_change in self._op_stack: - column_family = get_column_family(DB_PREFIXES(staged_change.key[:1]).value) - if staged_change.is_put: - batch_put((column_family, staged_change.key), staged_change.value) - else: - batch_delete((column_family, staged_change.key)) - for undo_to_delete in delete_undos: - batch_delete((undo_c_f, undo_to_delete)) - batch_put((undo_c_f, DB_PREFIXES.undo.value + self.UNDO_KEY_STRUCT.pack(height, block_hash)), undo_ops) - finally: - self._op_stack.clear() - - def rollback(self, height: int, block_hash: bytes): - """ - Revert changes for a block height - """ - undo_key = DB_PREFIXES.undo.value + self.UNDO_KEY_STRUCT.pack(height, block_hash) - undo_c_f = self.column_families[DB_PREFIXES.undo.value] - undo_info = self._db.get((undo_c_f, undo_key)) - self._op_stack.apply_packed_undo_ops(undo_info) - try: - with self._db.write_batch(sync=True) as batch: - batch_put = batch.put - batch_delete = batch.delete - get_column_family = self.column_families.__getitem__ - for staged_change in self._op_stack: - column_family = get_column_family(DB_PREFIXES(staged_change.key[:1]).value) - if staged_change.is_put: - batch_put((column_family, staged_change.key), staged_change.value) - else: - batch_delete((column_family, staged_change.key)) - # batch_delete(undo_key) - finally: - self._op_stack.clear() - - def get(self, key: bytes, fill_cache: bool = True) -> Optional[bytes]: - cf = self.column_families[key[:1]] - return self._db.get((cf, key), fill_cache=fill_cache) - - def iterator(self, start: bytes, column_family: 'rocksdb.ColumnFamilyHandle' = None, - iterate_lower_bound: bytes = None, iterate_upper_bound: bytes = None, - reverse: bool = False, include_key: bool = True, include_value: bool = True, - fill_cache: bool = True, prefix_same_as_start: bool = False, auto_prefix_mode: bool = True): - return self._db.iterator( - start=start, column_family=column_family, iterate_lower_bound=iterate_lower_bound, - iterate_upper_bound=iterate_upper_bound, reverse=reverse, include_key=include_key, - include_value=include_value, fill_cache=fill_cache, prefix_same_as_start=prefix_same_as_start, - auto_prefix_mode=auto_prefix_mode - ) - - def close(self): - self._db.close() - - def try_catch_up_with_primary(self): - self._db.try_catch_up_with_primary() - - def stage_raw_put(self, key: bytes, value: bytes): - self._op_stack.append_op(RevertablePut(key, value)) - - def stage_raw_delete(self, key: bytes, value: bytes): - self._op_stack.append_op(RevertableDelete(key, value)) - - def estimate_num_keys(self, column_family: 'rocksdb.ColumnFamilyHandle' = None): - return int(self._db.get_property(b'rocksdb.estimate-num-keys', column_family).decode()) diff --git a/lbry/wallet/server/db/prefixes.py b/lbry/wallet/server/db/prefixes.py deleted file mode 100644 index 3b091fa77..000000000 --- a/lbry/wallet/server/db/prefixes.py +++ /dev/null @@ -1,1800 +0,0 @@ -import typing -import struct -import array -import base64 -import rocksdb -import rocksdb.interfaces -from typing import Union, Tuple, NamedTuple, Optional -from lbry.wallet.server.db import DB_PREFIXES, COLUMN_SETTINGS -from lbry.wallet.server.db.interface import BasePrefixDB -from lbry.wallet.server.db.revertable import RevertableOpStack, RevertablePut, RevertableDelete -from lbry.schema.url import normalize_name - -ACTIVATED_CLAIM_TXO_TYPE = 1 -ACTIVATED_SUPPORT_TXO_TYPE = 2 - - -def length_encoded_name(name: str) -> bytes: - encoded = name.encode('utf-8') - return len(encoded).to_bytes(2, byteorder='big') + encoded - - -def length_prefix(key: str) -> bytes: - return len(key).to_bytes(1, byteorder='big') + key.encode() - - -ROW_TYPES = {} - - -class PrefixRowType(type): - def __new__(cls, name, bases, kwargs): - klass = super().__new__(cls, name, bases, kwargs) - if name != "PrefixRow": - ROW_TYPES[klass.prefix] = klass - cache_size = klass.cache_size - COLUMN_SETTINGS[klass.prefix] = { - 'cache_size': cache_size, - } - return klass - - -class PrefixRow(metaclass=PrefixRowType): - prefix: bytes - key_struct: struct.Struct - value_struct: struct.Struct - key_part_lambdas = [] - cache_size: int = 1024 * 1024 * 64 - - def __init__(self, db: 'rocksdb.DB', op_stack: RevertableOpStack): - self._db = db - self._op_stack = op_stack - self._column_family = self._db.get_column_family(self.prefix) - if not self._column_family.is_valid: - raise RuntimeError('column family is not valid') - - def iterate(self, prefix=None, start=None, stop=None, reverse: bool = False, include_key: bool = True, - include_value: bool = True, fill_cache: bool = True, deserialize_key: bool = True, - deserialize_value: bool = True): - if not prefix and not start and not stop: - prefix = () - if prefix is not None: - prefix = self.pack_partial_key(*prefix) - if stop is None: - try: - stop = (int.from_bytes(prefix, byteorder='big') + 1).to_bytes(len(prefix), byteorder='big') - except OverflowError: - stop = (int.from_bytes(prefix, byteorder='big') + 1).to_bytes(len(prefix) + 1, byteorder='big') - else: - stop = self.pack_partial_key(*stop) - else: - if start is not None: - start = self.pack_partial_key(*start) - if stop is not None: - stop = self.pack_partial_key(*stop) - - if deserialize_key: - key_getter = lambda k: self.unpack_key(k) - else: - key_getter = lambda k: k - if deserialize_value: - value_getter = lambda v: self.unpack_value(v) - else: - value_getter = lambda v: v - - it = self._db.iterator( - start or prefix, self._column_family, iterate_lower_bound=(start or prefix), - iterate_upper_bound=stop, reverse=reverse, include_key=include_key, - include_value=include_value, fill_cache=fill_cache, prefix_same_as_start=False - ) - - if include_key and include_value: - for k, v in it: - yield key_getter(k[1]), value_getter(v) - elif include_key: - for k in it: - yield key_getter(k[1]) - elif include_value: - for v in it: - yield value_getter(v) - else: - for _ in it: - yield None - - def get(self, *key_args, fill_cache=True, deserialize_value=True): - v = self._db.get((self._column_family, self.pack_key(*key_args)), fill_cache=fill_cache) - if v: - return v if not deserialize_value else self.unpack_value(v) - - def get_pending(self, *key_args, fill_cache=True, deserialize_value=True): - packed_key = self.pack_key(*key_args) - last_op = self._op_stack.get_last_op_for_key(packed_key) - if last_op: - if last_op.is_put: - return last_op.value if not deserialize_value else self.unpack_value(last_op.value) - else: # it's a delete - return - v = self._db.get((self._column_family, packed_key), fill_cache=fill_cache) - if v: - return v if not deserialize_value else self.unpack_value(v) - - def stage_put(self, key_args=(), value_args=()): - self._op_stack.append_op(RevertablePut(self.pack_key(*key_args), self.pack_value(*value_args))) - - def stage_delete(self, key_args=(), value_args=()): - self._op_stack.append_op(RevertableDelete(self.pack_key(*key_args), self.pack_value(*value_args))) - - @classmethod - def pack_partial_key(cls, *args) -> bytes: - return cls.prefix + cls.key_part_lambdas[len(args)](*args) - - @classmethod - def pack_key(cls, *args) -> bytes: - return cls.prefix + cls.key_struct.pack(*args) - - @classmethod - def pack_value(cls, *args) -> bytes: - return cls.value_struct.pack(*args) - - @classmethod - def unpack_key(cls, key: bytes): - assert key[:1] == cls.prefix, f"prefix should be {cls.prefix}, got {key[:1]}" - return cls.key_struct.unpack(key[1:]) - - @classmethod - def unpack_value(cls, data: bytes): - return cls.value_struct.unpack(data) - - @classmethod - def unpack_item(cls, key: bytes, value: bytes): - return cls.unpack_key(key), cls.unpack_value(value) - - def estimate_num_keys(self) -> int: - return int(self._db.get_property(b'rocksdb.estimate-num-keys', self._column_family).decode()) - - -class UTXOKey(NamedTuple): - hashX: bytes - tx_num: int - nout: int - - def __str__(self): - return f"{self.__class__.__name__}(hashX={self.hashX.hex()}, tx_num={self.tx_num}, nout={self.nout})" - - -class UTXOValue(NamedTuple): - amount: int - - -class HashXUTXOKey(NamedTuple): - short_tx_hash: bytes - tx_num: int - nout: int - - def __str__(self): - return f"{self.__class__.__name__}(short_tx_hash={self.short_tx_hash.hex()}, tx_num={self.tx_num}, nout={self.nout})" - - -class HashXUTXOValue(NamedTuple): - hashX: bytes - - def __str__(self): - return f"{self.__class__.__name__}(hashX={self.hashX.hex()})" - - -class HashXHistoryKey(NamedTuple): - hashX: bytes - height: int - - def __str__(self): - return f"{self.__class__.__name__}(hashX={self.hashX.hex()}, height={self.height})" - - -class HashXHistoryValue(NamedTuple): - hashXes: typing.List[int] - - -class BlockHashKey(NamedTuple): - height: int - - -class BlockHashValue(NamedTuple): - block_hash: bytes - - def __str__(self): - return f"{self.__class__.__name__}(block_hash={self.block_hash.hex()})" - - -class BlockTxsKey(NamedTuple): - height: int - - -class BlockTxsValue(NamedTuple): - tx_hashes: typing.List[bytes] - - -class TxCountKey(NamedTuple): - height: int - - -class TxCountValue(NamedTuple): - tx_count: int - - -class TxHashKey(NamedTuple): - tx_num: int - - -class TxHashValue(NamedTuple): - tx_hash: bytes - - def __str__(self): - return f"{self.__class__.__name__}(tx_hash={self.tx_hash[::-1].hex()})" - - -class TxNumKey(NamedTuple): - tx_hash: bytes - - def __str__(self): - return f"{self.__class__.__name__}(tx_hash={self.tx_hash[::-1].hex()})" - - -class TxNumValue(NamedTuple): - tx_num: int - - -class TxKey(NamedTuple): - tx_hash: bytes - - def __str__(self): - return f"{self.__class__.__name__}(tx_hash={self.tx_hash[::-1].hex()})" - - -class TxValue(NamedTuple): - raw_tx: bytes - - def __str__(self): - return f"{self.__class__.__name__}(raw_tx={base64.b64encode(self.raw_tx).decode()})" - - -class BlockHeaderKey(NamedTuple): - height: int - - -class BlockHeaderValue(NamedTuple): - header: bytes - - def __str__(self): - return f"{self.__class__.__name__}(header={base64.b64encode(self.header)})" - - -class ClaimToTXOKey(typing.NamedTuple): - claim_hash: bytes - - def __str__(self): - return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()})" - - -class ClaimToTXOValue(typing.NamedTuple): - tx_num: int - position: int - root_tx_num: int - root_position: int - amount: int - # activation: int - channel_signature_is_valid: bool - name: str - - @property - def normalized_name(self) -> str: - try: - return normalize_name(self.name) - except UnicodeDecodeError: - return self.name - - -class TXOToClaimKey(typing.NamedTuple): - tx_num: int - position: int - - -class TXOToClaimValue(typing.NamedTuple): - claim_hash: bytes - name: str - - def __str__(self): - return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()}, name={self.name})" - - -class ClaimShortIDKey(typing.NamedTuple): - normalized_name: str - partial_claim_id: str - root_tx_num: int - root_position: int - - def __str__(self): - return f"{self.__class__.__name__}(normalized_name={self.normalized_name}, " \ - f"partial_claim_id={self.partial_claim_id}, " \ - f"root_tx_num={self.root_tx_num}, root_position={self.root_position})" - - -class ClaimShortIDValue(typing.NamedTuple): - tx_num: int - position: int - - -class ClaimToChannelKey(typing.NamedTuple): - claim_hash: bytes - tx_num: int - position: int - - def __str__(self): - return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()}, " \ - f"tx_num={self.tx_num}, position={self.position})" - - -class ClaimToChannelValue(typing.NamedTuple): - signing_hash: bytes - - def __str__(self): - return f"{self.__class__.__name__}(signing_hash={self.signing_hash.hex()})" - - -class ChannelToClaimKey(typing.NamedTuple): - signing_hash: bytes - name: str - tx_num: int - position: int - - def __str__(self): - return f"{self.__class__.__name__}(signing_hash={self.signing_hash.hex()}, name={self.name}, " \ - f"tx_num={self.tx_num}, position={self.position})" - - -class ChannelToClaimValue(typing.NamedTuple): - claim_hash: bytes - - def __str__(self): - return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()})" - - -class ChannelCountKey(typing.NamedTuple): - channel_hash: bytes - - def __str__(self): - return f"{self.__class__.__name__}(channel_hash={self.channel_hash.hex()})" - - -class ChannelCountValue(typing.NamedTuple): - count: int - - -class SupportAmountKey(typing.NamedTuple): - claim_hash: bytes - - def __str__(self): - return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()})" - - -class SupportAmountValue(typing.NamedTuple): - amount: int - - -class ClaimToSupportKey(typing.NamedTuple): - claim_hash: bytes - tx_num: int - position: int - - def __str__(self): - return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()}, tx_num={self.tx_num}, " \ - f"position={self.position})" - - -class ClaimToSupportValue(typing.NamedTuple): - amount: int - - -class SupportToClaimKey(typing.NamedTuple): - tx_num: int - position: int - - -class SupportToClaimValue(typing.NamedTuple): - claim_hash: bytes - - def __str__(self): - return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()})" - - -class ClaimExpirationKey(typing.NamedTuple): - expiration: int - tx_num: int - position: int - - -class ClaimExpirationValue(typing.NamedTuple): - claim_hash: bytes - normalized_name: str - - def __str__(self): - return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()}, normalized_name={self.normalized_name})" - - -class ClaimTakeoverKey(typing.NamedTuple): - normalized_name: str - - -class ClaimTakeoverValue(typing.NamedTuple): - claim_hash: bytes - height: int - - def __str__(self): - return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()}, height={self.height})" - - -class PendingActivationKey(typing.NamedTuple): - height: int - txo_type: int - tx_num: int - position: int - - @property - def is_support(self) -> bool: - return self.txo_type == ACTIVATED_SUPPORT_TXO_TYPE - - @property - def is_claim(self) -> bool: - return self.txo_type == ACTIVATED_CLAIM_TXO_TYPE - - -class PendingActivationValue(typing.NamedTuple): - claim_hash: bytes - normalized_name: str - - def __str__(self): - return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()}, normalized_name={self.normalized_name})" - - -class ActivationKey(typing.NamedTuple): - txo_type: int - tx_num: int - position: int - - -class ActivationValue(typing.NamedTuple): - height: int - claim_hash: bytes - normalized_name: str - - def __str__(self): - return f"{self.__class__.__name__}(height={self.height}, claim_hash={self.claim_hash.hex()}, " \ - f"normalized_name={self.normalized_name})" - - -class ActiveAmountKey(typing.NamedTuple): - claim_hash: bytes - txo_type: int - activation_height: int - tx_num: int - position: int - - def __str__(self): - return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()}, txo_type={self.txo_type}, " \ - f"activation_height={self.activation_height}, tx_num={self.tx_num}, position={self.position})" - - -class ActiveAmountValue(typing.NamedTuple): - amount: int - - -class EffectiveAmountKey(typing.NamedTuple): - normalized_name: str - effective_amount: int - tx_num: int - position: int - - -class EffectiveAmountValue(typing.NamedTuple): - claim_hash: bytes - - def __str__(self): - return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()})" - - -class RepostKey(typing.NamedTuple): - claim_hash: bytes - - def __str__(self): - return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()})" - - -class RepostValue(typing.NamedTuple): - reposted_claim_hash: bytes - - def __str__(self): - return f"{self.__class__.__name__}(reposted_claim_hash={self.reposted_claim_hash.hex()})" - - -class RepostedKey(typing.NamedTuple): - reposted_claim_hash: bytes - tx_num: int - position: int - - def __str__(self): - return f"{self.__class__.__name__}(reposted_claim_hash={self.reposted_claim_hash.hex()}, " \ - f"tx_num={self.tx_num}, position={self.position})" - - -class RepostedValue(typing.NamedTuple): - claim_hash: bytes - - def __str__(self): - return f"{self.__class__.__name__}(claim_hash={self.claim_hash.hex()})" - - -class TouchedOrDeletedClaimKey(typing.NamedTuple): - height: int - - -class TouchedOrDeletedClaimValue(typing.NamedTuple): - touched_claims: typing.Set[bytes] - deleted_claims: typing.Set[bytes] - - def __str__(self): - return f"{self.__class__.__name__}(" \ - f"touched_claims={','.join(map(lambda x: x.hex(), self.touched_claims))}," \ - f"deleted_claims={','.join(map(lambda x: x.hex(), self.deleted_claims))})" - - -class DBState(typing.NamedTuple): - genesis: bytes - height: int - tx_count: int - tip: bytes - utxo_flush_count: int - wall_time: int - first_sync: bool - db_version: int - hist_flush_count: int - comp_flush_count: int - comp_cursor: int - es_sync_height: int - - -class ActiveAmountPrefixRow(PrefixRow): - prefix = DB_PREFIXES.active_amount.value - key_struct = struct.Struct(b'>20sBLLH') - value_struct = struct.Struct(b'>Q') - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>20s').pack, - struct.Struct(b'>20sB').pack, - struct.Struct(b'>20sBL').pack, - struct.Struct(b'>20sBLL').pack, - struct.Struct(b'>20sBLLH').pack - ] - cache_size = 1024 * 1024 * 128 - - @classmethod - def pack_key(cls, claim_hash: bytes, txo_type: int, activation_height: int, tx_num: int, position: int): - return super().pack_key(claim_hash, txo_type, activation_height, tx_num, position) - - @classmethod - def unpack_key(cls, key: bytes) -> ActiveAmountKey: - return ActiveAmountKey(*super().unpack_key(key)) - - @classmethod - def unpack_value(cls, data: bytes) -> ActiveAmountValue: - return ActiveAmountValue(*super().unpack_value(data)) - - @classmethod - def pack_value(cls, amount: int) -> bytes: - return cls.value_struct.pack(amount) - - @classmethod - def pack_item(cls, claim_hash: bytes, txo_type: int, activation_height: int, tx_num: int, position: int, amount: int): - return cls.pack_key(claim_hash, txo_type, activation_height, tx_num, position), cls.pack_value(amount) - - -class ClaimToTXOPrefixRow(PrefixRow): - prefix = DB_PREFIXES.claim_to_txo.value - key_struct = struct.Struct(b'>20s') - value_struct = struct.Struct(b'>LHLHQB') - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>20s').pack - ] - cache_size = 1024 * 1024 * 128 - - @classmethod - def pack_key(cls, claim_hash: bytes): - return super().pack_key(claim_hash) - - @classmethod - def unpack_key(cls, key: bytes) -> ClaimToTXOKey: - assert key[:1] == cls.prefix and len(key) == 21 - return ClaimToTXOKey(key[1:]) - - @classmethod - def unpack_value(cls, data: bytes) -> ClaimToTXOValue: - tx_num, position, root_tx_num, root_position, amount, channel_signature_is_valid = cls.value_struct.unpack( - data[:21] - ) - name_len = int.from_bytes(data[21:23], byteorder='big') - name = data[23:23 + name_len].decode() - return ClaimToTXOValue( - tx_num, position, root_tx_num, root_position, amount, bool(channel_signature_is_valid), name - ) - - @classmethod - def pack_value(cls, tx_num: int, position: int, root_tx_num: int, root_position: int, amount: int, - channel_signature_is_valid: bool, name: str) -> bytes: - return cls.value_struct.pack( - tx_num, position, root_tx_num, root_position, amount, int(channel_signature_is_valid) - ) + length_encoded_name(name) - - @classmethod - def pack_item(cls, claim_hash: bytes, tx_num: int, position: int, root_tx_num: int, root_position: int, - amount: int, channel_signature_is_valid: bool, name: str): - return cls.pack_key(claim_hash), \ - cls.pack_value(tx_num, position, root_tx_num, root_position, amount, channel_signature_is_valid, name) - - -class TXOToClaimPrefixRow(PrefixRow): - prefix = DB_PREFIXES.txo_to_claim.value - key_struct = struct.Struct(b'>LH') - value_struct = struct.Struct(b'>20s') - cache_size = 1024 * 1024 * 128 - - @classmethod - def pack_key(cls, tx_num: int, position: int): - return super().pack_key(tx_num, position) - - @classmethod - def unpack_key(cls, key: bytes) -> TXOToClaimKey: - return TXOToClaimKey(*super().unpack_key(key)) - - @classmethod - def unpack_value(cls, data: bytes) -> TXOToClaimValue: - claim_hash, = cls.value_struct.unpack(data[:20]) - name_len = int.from_bytes(data[20:22], byteorder='big') - name = data[22:22 + name_len].decode() - return TXOToClaimValue(claim_hash, name) - - @classmethod - def pack_value(cls, claim_hash: bytes, name: str) -> bytes: - return cls.value_struct.pack(claim_hash) + length_encoded_name(name) - - @classmethod - def pack_item(cls, tx_num: int, position: int, claim_hash: bytes, name: str): - return cls.pack_key(tx_num, position), \ - cls.pack_value(claim_hash, name) - - -def shortid_key_helper(struct_fmt): - packer = struct.Struct(struct_fmt).pack - def wrapper(name, *args): - return length_encoded_name(name) + packer(*args) - return wrapper - - -def shortid_key_partial_claim_helper(name: str, partial_claim_id: str): - assert len(partial_claim_id) < 40 - return length_encoded_name(name) + length_prefix(partial_claim_id) - - -class ClaimShortIDPrefixRow(PrefixRow): - prefix = DB_PREFIXES.claim_short_id_prefix.value - key_struct = struct.Struct(b'>LH') - value_struct = struct.Struct(b'>LH') - key_part_lambdas = [ - lambda: b'', - length_encoded_name, - shortid_key_partial_claim_helper - ] - - @classmethod - def pack_key(cls, name: str, short_claim_id: str, root_tx_num: int, root_position: int): - return cls.prefix + length_encoded_name(name) + length_prefix(short_claim_id) +\ - cls.key_struct.pack(root_tx_num, root_position) - - @classmethod - def pack_value(cls, tx_num: int, position: int): - return super().pack_value(tx_num, position) - - @classmethod - def unpack_key(cls, key: bytes) -> ClaimShortIDKey: - assert key[:1] == cls.prefix - name_len = int.from_bytes(key[1:3], byteorder='big') - name = key[3:3 + name_len].decode() - claim_id_len = int.from_bytes(key[3+name_len:4+name_len], byteorder='big') - partial_claim_id = key[4+name_len:4+name_len+claim_id_len].decode() - return ClaimShortIDKey(name, partial_claim_id, *cls.key_struct.unpack(key[4 + name_len + claim_id_len:])) - - @classmethod - def unpack_value(cls, data: bytes) -> ClaimShortIDValue: - return ClaimShortIDValue(*super().unpack_value(data)) - - @classmethod - def pack_item(cls, name: str, partial_claim_id: str, root_tx_num: int, root_position: int, - tx_num: int, position: int): - return cls.pack_key(name, partial_claim_id, root_tx_num, root_position), \ - cls.pack_value(tx_num, position) - - -class ClaimToChannelPrefixRow(PrefixRow): - prefix = DB_PREFIXES.claim_to_channel.value - key_struct = struct.Struct(b'>20sLH') - value_struct = struct.Struct(b'>20s') - - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>20s').pack, - struct.Struct(b'>20sL').pack, - struct.Struct(b'>20sLH').pack - ] - - @classmethod - def pack_key(cls, claim_hash: bytes, tx_num: int, position: int): - return super().pack_key(claim_hash, tx_num, position) - - @classmethod - def pack_value(cls, signing_hash: bytes): - return super().pack_value(signing_hash) - - @classmethod - def unpack_key(cls, key: bytes) -> ClaimToChannelKey: - return ClaimToChannelKey(*super().unpack_key(key)) - - @classmethod - def unpack_value(cls, data: bytes) -> ClaimToChannelValue: - return ClaimToChannelValue(*super().unpack_value(data)) - - @classmethod - def pack_item(cls, claim_hash: bytes, tx_num: int, position: int, signing_hash: bytes): - return cls.pack_key(claim_hash, tx_num, position), cls.pack_value(signing_hash) - - -def channel_to_claim_helper(struct_fmt): - packer = struct.Struct(struct_fmt).pack - - def wrapper(signing_hash: bytes, name: str, *args): - return signing_hash + length_encoded_name(name) + packer(*args) - - return wrapper - - -class ChannelToClaimPrefixRow(PrefixRow): - prefix = DB_PREFIXES.channel_to_claim.value - key_struct = struct.Struct(b'>LH') - value_struct = struct.Struct(b'>20s') - - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>20s').pack, - channel_to_claim_helper(b''), - channel_to_claim_helper(b'>s'), - channel_to_claim_helper(b'>L'), - channel_to_claim_helper(b'>LH'), - ] - - @classmethod - def pack_key(cls, signing_hash: bytes, name: str, tx_num: int, position: int): - return cls.prefix + signing_hash + length_encoded_name(name) + cls.key_struct.pack( - tx_num, position - ) - - @classmethod - def unpack_key(cls, key: bytes) -> ChannelToClaimKey: - assert key[:1] == cls.prefix - signing_hash = key[1:21] - name_len = int.from_bytes(key[21:23], byteorder='big') - name = key[23:23 + name_len].decode() - tx_num, position = cls.key_struct.unpack(key[23 + name_len:]) - return ChannelToClaimKey( - signing_hash, name, tx_num, position - ) - - @classmethod - def pack_value(cls, claim_hash: bytes) -> bytes: - return super().pack_value(claim_hash) - - @classmethod - def unpack_value(cls, data: bytes) -> ChannelToClaimValue: - return ChannelToClaimValue(*cls.value_struct.unpack(data)) - - @classmethod - def pack_item(cls, signing_hash: bytes, name: str, tx_num: int, position: int, - claim_hash: bytes): - return cls.pack_key(signing_hash, name, tx_num, position), \ - cls.pack_value(claim_hash) - - -class ClaimToSupportPrefixRow(PrefixRow): - prefix = DB_PREFIXES.claim_to_support.value - key_struct = struct.Struct(b'>20sLH') - value_struct = struct.Struct(b'>Q') - - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>20s').pack, - struct.Struct(b'>20sL').pack, - struct.Struct(b'>20sLH').pack - ] - - @classmethod - def pack_key(cls, claim_hash: bytes, tx_num: int, position: int): - return super().pack_key(claim_hash, tx_num, position) - - @classmethod - def unpack_key(cls, key: bytes) -> ClaimToSupportKey: - return ClaimToSupportKey(*super().unpack_key(key)) - - @classmethod - def pack_value(cls, amount: int) -> bytes: - return super().pack_value(amount) - - @classmethod - def unpack_value(cls, data: bytes) -> ClaimToSupportValue: - return ClaimToSupportValue(*super().unpack_value(data)) - - @classmethod - def pack_item(cls, claim_hash: bytes, tx_num: int, position: int, amount: int): - return cls.pack_key(claim_hash, tx_num, position), \ - cls.pack_value(amount) - - -class SupportToClaimPrefixRow(PrefixRow): - prefix = DB_PREFIXES.support_to_claim.value - key_struct = struct.Struct(b'>LH') - value_struct = struct.Struct(b'>20s') - - @classmethod - def pack_key(cls, tx_num: int, position: int): - return super().pack_key(tx_num, position) - - @classmethod - def unpack_key(cls, key: bytes) -> SupportToClaimKey: - return SupportToClaimKey(*super().unpack_key(key)) - - @classmethod - def pack_value(cls, claim_hash: bytes) -> bytes: - return super().pack_value(claim_hash) - - @classmethod - def unpack_value(cls, data: bytes) -> SupportToClaimValue: - return SupportToClaimValue(*super().unpack_value(data)) - - @classmethod - def pack_item(cls, tx_num: int, position: int, claim_hash: bytes): - return cls.pack_key(tx_num, position), \ - cls.pack_value(claim_hash) - - -class ClaimExpirationPrefixRow(PrefixRow): - prefix = DB_PREFIXES.claim_expiration.value - key_struct = struct.Struct(b'>LLH') - value_struct = struct.Struct(b'>20s') - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>L').pack, - struct.Struct(b'>LL').pack, - struct.Struct(b'>LLH').pack, - ] - - @classmethod - def pack_key(cls, expiration: int, tx_num: int, position: int) -> bytes: - return super().pack_key(expiration, tx_num, position) - - @classmethod - def pack_value(cls, claim_hash: bytes, name: str) -> bytes: - return cls.value_struct.pack(claim_hash) + length_encoded_name(name) - - @classmethod - def pack_item(cls, expiration: int, tx_num: int, position: int, claim_hash: bytes, name: str) -> typing.Tuple[bytes, bytes]: - return cls.pack_key(expiration, tx_num, position), cls.pack_value(claim_hash, name) - - @classmethod - def unpack_key(cls, key: bytes) -> ClaimExpirationKey: - return ClaimExpirationKey(*super().unpack_key(key)) - - @classmethod - def unpack_value(cls, data: bytes) -> ClaimExpirationValue: - name_len = int.from_bytes(data[20:22], byteorder='big') - name = data[22:22 + name_len].decode() - claim_id, = cls.value_struct.unpack(data[:20]) - return ClaimExpirationValue(claim_id, name) - - @classmethod - def unpack_item(cls, key: bytes, value: bytes) -> typing.Tuple[ClaimExpirationKey, ClaimExpirationValue]: - return cls.unpack_key(key), cls.unpack_value(value) - - -class ClaimTakeoverPrefixRow(PrefixRow): - prefix = DB_PREFIXES.claim_takeover.value - value_struct = struct.Struct(b'>20sL') - - key_part_lambdas = [ - lambda: b'', - length_encoded_name - ] - - @classmethod - def pack_key(cls, name: str): - return cls.prefix + length_encoded_name(name) - - @classmethod - def pack_value(cls, claim_hash: bytes, takeover_height: int): - return super().pack_value(claim_hash, takeover_height) - - @classmethod - def unpack_key(cls, key: bytes) -> ClaimTakeoverKey: - assert key[:1] == cls.prefix - name_len = int.from_bytes(key[1:3], byteorder='big') - name = key[3:3 + name_len].decode() - return ClaimTakeoverKey(name) - - @classmethod - def unpack_value(cls, data: bytes) -> ClaimTakeoverValue: - return ClaimTakeoverValue(*super().unpack_value(data)) - - @classmethod - def pack_item(cls, name: str, claim_hash: bytes, takeover_height: int): - return cls.pack_key(name), cls.pack_value(claim_hash, takeover_height) - - -class PendingActivationPrefixRow(PrefixRow): - prefix = DB_PREFIXES.pending_activation.value - key_struct = struct.Struct(b'>LBLH') - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>L').pack, - struct.Struct(b'>LB').pack, - struct.Struct(b'>LBL').pack, - struct.Struct(b'>LBLH').pack - ] - - @classmethod - def pack_key(cls, height: int, txo_type: int, tx_num: int, position: int): - return super().pack_key(height, txo_type, tx_num, position) - - @classmethod - def unpack_key(cls, key: bytes) -> PendingActivationKey: - return PendingActivationKey(*super().unpack_key(key)) - - @classmethod - def pack_value(cls, claim_hash: bytes, name: str) -> bytes: - return claim_hash + length_encoded_name(name) - - @classmethod - def unpack_value(cls, data: bytes) -> PendingActivationValue: - claim_hash = data[:20] - name_len = int.from_bytes(data[20:22], byteorder='big') - name = data[22:22 + name_len].decode() - return PendingActivationValue(claim_hash, name) - - @classmethod - def pack_item(cls, height: int, txo_type: int, tx_num: int, position: int, claim_hash: bytes, name: str): - return cls.pack_key(height, txo_type, tx_num, position), \ - cls.pack_value(claim_hash, name) - - -class ActivatedPrefixRow(PrefixRow): - prefix = DB_PREFIXES.activated_claim_and_support.value - key_struct = struct.Struct(b'>BLH') - value_struct = struct.Struct(b'>L20s') - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>B').pack, - struct.Struct(b'>BL').pack, - struct.Struct(b'>BLH').pack - ] - - @classmethod - def pack_key(cls, txo_type: int, tx_num: int, position: int): - return super().pack_key(txo_type, tx_num, position) - - @classmethod - def unpack_key(cls, key: bytes) -> ActivationKey: - return ActivationKey(*super().unpack_key(key)) - - @classmethod - def pack_value(cls, height: int, claim_hash: bytes, name: str) -> bytes: - return cls.value_struct.pack(height, claim_hash) + length_encoded_name(name) - - @classmethod - def unpack_value(cls, data: bytes) -> ActivationValue: - height, claim_hash = cls.value_struct.unpack(data[:24]) - name_len = int.from_bytes(data[24:26], byteorder='big') - name = data[26:26 + name_len].decode() - return ActivationValue(height, claim_hash, name) - - @classmethod - def pack_item(cls, txo_type: int, tx_num: int, position: int, height: int, claim_hash: bytes, name: str): - return cls.pack_key(txo_type, tx_num, position), \ - cls.pack_value(height, claim_hash, name) - - -def effective_amount_helper(struct_fmt): - packer = struct.Struct(struct_fmt).pack - - def wrapper(name, *args): - if not args: - return length_encoded_name(name) - if len(args) == 1: - return length_encoded_name(name) + packer(0xffffffffffffffff - args[0]) - return length_encoded_name(name) + packer(0xffffffffffffffff - args[0], *args[1:]) - - return wrapper - - -class EffectiveAmountPrefixRow(PrefixRow): - prefix = DB_PREFIXES.effective_amount.value - key_struct = struct.Struct(b'>QLH') - value_struct = struct.Struct(b'>20s') - key_part_lambdas = [ - lambda: b'', - length_encoded_name, - shortid_key_helper(b'>Q'), - shortid_key_helper(b'>QL'), - shortid_key_helper(b'>QLH'), - ] - cache_size = 1024 * 1024 * 128 - - @classmethod - def pack_key(cls, name: str, effective_amount: int, tx_num: int, position: int): - return cls.prefix + length_encoded_name(name) + cls.key_struct.pack( - 0xffffffffffffffff - effective_amount, tx_num, position - ) - - @classmethod - def unpack_key(cls, key: bytes) -> EffectiveAmountKey: - assert key[:1] == cls.prefix - name_len = int.from_bytes(key[1:3], byteorder='big') - name = key[3:3 + name_len].decode() - ones_comp_effective_amount, tx_num, position = cls.key_struct.unpack(key[3 + name_len:]) - return EffectiveAmountKey(name, 0xffffffffffffffff - ones_comp_effective_amount, tx_num, position) - - @classmethod - def unpack_value(cls, data: bytes) -> EffectiveAmountValue: - return EffectiveAmountValue(*super().unpack_value(data)) - - @classmethod - def pack_value(cls, claim_hash: bytes) -> bytes: - return super().pack_value(claim_hash) - - @classmethod - def pack_item(cls, name: str, effective_amount: int, tx_num: int, position: int, claim_hash: bytes): - return cls.pack_key(name, effective_amount, tx_num, position), cls.pack_value(claim_hash) - - -class RepostPrefixRow(PrefixRow): - prefix = DB_PREFIXES.repost.value - key_struct = struct.Struct(b'>20s') - - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>20s').pack - ] - - @classmethod - def pack_key(cls, claim_hash: bytes): - return super().pack_key(claim_hash) - - @classmethod - def unpack_key(cls, key: bytes) -> RepostKey: - return RepostKey(*super().unpack_key(key)) - - @classmethod - def pack_value(cls, reposted_claim_hash: bytes) -> bytes: - return reposted_claim_hash - - @classmethod - def unpack_value(cls, data: bytes) -> RepostValue: - return RepostValue(data) - - @classmethod - def pack_item(cls, claim_hash: bytes, reposted_claim_hash: bytes): - return cls.pack_key(claim_hash), cls.pack_value(reposted_claim_hash) - - -class RepostedPrefixRow(PrefixRow): - prefix = DB_PREFIXES.reposted_claim.value - key_struct = struct.Struct(b'>20sLH') - value_struct = struct.Struct(b'>20s') - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>20s').pack, - struct.Struct(b'>20sL').pack, - struct.Struct(b'>20sLH').pack - ] - - @classmethod - def pack_key(cls, reposted_claim_hash: bytes, tx_num: int, position: int): - return super().pack_key(reposted_claim_hash, tx_num, position) - - @classmethod - def unpack_key(cls, key: bytes) -> RepostedKey: - return RepostedKey(*super().unpack_key(key)) - - @classmethod - def pack_value(cls, claim_hash: bytes) -> bytes: - return super().pack_value(claim_hash) - - @classmethod - def unpack_value(cls, data: bytes) -> RepostedValue: - return RepostedValue(*super().unpack_value(data)) - - @classmethod - def pack_item(cls, reposted_claim_hash: bytes, tx_num: int, position: int, claim_hash: bytes): - return cls.pack_key(reposted_claim_hash, tx_num, position), cls.pack_value(claim_hash) - - -class UndoKey(NamedTuple): - height: int - block_hash: bytes - - -class UndoPrefixRow(PrefixRow): - prefix = DB_PREFIXES.undo.value - key_struct = struct.Struct(b'>Q32s') - - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>Q').pack, - struct.Struct(b'>Q32s').pack - ] - - @classmethod - def pack_key(cls, height: int, block_hash: bytes): - return super().pack_key(height, block_hash) - - @classmethod - def unpack_key(cls, key: bytes) -> UndoKey: - return UndoKey(*super().unpack_key(key)) - - @classmethod - def pack_value(cls, undo_ops: bytes) -> bytes: - return undo_ops - - @classmethod - def unpack_value(cls, data: bytes) -> bytes: - return data - - @classmethod - def pack_item(cls, height: int, block_hash: bytes, undo_ops: bytes): - return cls.pack_key(height, block_hash), cls.pack_value(undo_ops) - - -class BlockHashPrefixRow(PrefixRow): - prefix = DB_PREFIXES.block_hash.value - key_struct = struct.Struct(b'>L') - value_struct = struct.Struct(b'>32s') - - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>L').pack - ] - - @classmethod - def pack_key(cls, height: int) -> bytes: - return super().pack_key(height) - - @classmethod - def unpack_key(cls, key: bytes) -> BlockHashKey: - return BlockHashKey(*super().unpack_key(key)) - - @classmethod - def pack_value(cls, block_hash: bytes) -> bytes: - return super().pack_value(block_hash) - - @classmethod - def unpack_value(cls, data: bytes) -> BlockHashValue: - return BlockHashValue(*super().unpack_value(data)) - - @classmethod - def pack_item(cls, height: int, block_hash: bytes): - return cls.pack_key(height), cls.pack_value(block_hash) - - -class BlockHeaderPrefixRow(PrefixRow): - prefix = DB_PREFIXES.header.value - key_struct = struct.Struct(b'>L') - value_struct = struct.Struct(b'>112s') - - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>L').pack - ] - - @classmethod - def pack_key(cls, height: int) -> bytes: - return super().pack_key(height) - - @classmethod - def unpack_key(cls, key: bytes) -> BlockHeaderKey: - return BlockHeaderKey(*super().unpack_key(key)) - - @classmethod - def pack_value(cls, header: bytes) -> bytes: - return super().pack_value(header) - - @classmethod - def unpack_value(cls, data: bytes) -> BlockHeaderValue: - return BlockHeaderValue(*super().unpack_value(data)) - - @classmethod - def pack_item(cls, height: int, header: bytes): - return cls.pack_key(height), cls.pack_value(header) - - -class TXNumPrefixRow(PrefixRow): - prefix = DB_PREFIXES.tx_num.value - key_struct = struct.Struct(b'>32s') - value_struct = struct.Struct(b'>L') - - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>32s').pack - ] - - @classmethod - def pack_key(cls, tx_hash: bytes) -> bytes: - return super().pack_key(tx_hash) - - @classmethod - def unpack_key(cls, tx_hash: bytes) -> TxNumKey: - return TxNumKey(*super().unpack_key(tx_hash)) - - @classmethod - def pack_value(cls, tx_num: int) -> bytes: - return super().pack_value(tx_num) - - @classmethod - def unpack_value(cls, data: bytes) -> TxNumValue: - return TxNumValue(*super().unpack_value(data)) - - @classmethod - def pack_item(cls, tx_hash: bytes, tx_num: int): - return cls.pack_key(tx_hash), cls.pack_value(tx_num) - - -class TxCountPrefixRow(PrefixRow): - prefix = DB_PREFIXES.tx_count.value - key_struct = struct.Struct(b'>L') - value_struct = struct.Struct(b'>L') - - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>L').pack - ] - - @classmethod - def pack_key(cls, height: int) -> bytes: - return super().pack_key(height) - - @classmethod - def unpack_key(cls, key: bytes) -> TxCountKey: - return TxCountKey(*super().unpack_key(key)) - - @classmethod - def pack_value(cls, tx_count: int) -> bytes: - return super().pack_value(tx_count) - - @classmethod - def unpack_value(cls, data: bytes) -> TxCountValue: - return TxCountValue(*super().unpack_value(data)) - - @classmethod - def pack_item(cls, height: int, tx_count: int): - return cls.pack_key(height), cls.pack_value(tx_count) - - -class TXHashPrefixRow(PrefixRow): - prefix = DB_PREFIXES.tx_hash.value - key_struct = struct.Struct(b'>L') - value_struct = struct.Struct(b'>32s') - - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>L').pack - ] - - @classmethod - def pack_key(cls, tx_num: int) -> bytes: - return super().pack_key(tx_num) - - @classmethod - def unpack_key(cls, key: bytes) -> TxHashKey: - return TxHashKey(*super().unpack_key(key)) - - @classmethod - def pack_value(cls, tx_hash: bytes) -> bytes: - return super().pack_value(tx_hash) - - @classmethod - def unpack_value(cls, data: bytes) -> TxHashValue: - return TxHashValue(*super().unpack_value(data)) - - @classmethod - def pack_item(cls, tx_num: int, tx_hash: bytes): - return cls.pack_key(tx_num), cls.pack_value(tx_hash) - - -class TXPrefixRow(PrefixRow): - prefix = DB_PREFIXES.tx.value - key_struct = struct.Struct(b'>32s') - - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>32s').pack - ] - - @classmethod - def pack_key(cls, tx_hash: bytes) -> bytes: - return super().pack_key(tx_hash) - - @classmethod - def unpack_key(cls, tx_hash: bytes) -> TxKey: - return TxKey(*super().unpack_key(tx_hash)) - - @classmethod - def pack_value(cls, tx: bytes) -> bytes: - return tx - - @classmethod - def unpack_value(cls, data: bytes) -> TxValue: - return TxValue(data) - - @classmethod - def pack_item(cls, tx_hash: bytes, raw_tx: bytes): - return cls.pack_key(tx_hash), cls.pack_value(raw_tx) - - -class UTXOPrefixRow(PrefixRow): - prefix = DB_PREFIXES.utxo.value - key_struct = struct.Struct(b'>11sLH') - value_struct = struct.Struct(b'>Q') - - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>11s').pack, - struct.Struct(b'>11sL').pack, - struct.Struct(b'>11sLH').pack - ] - - @classmethod - def pack_key(cls, hashX: bytes, tx_num, nout: int): - return super().pack_key(hashX, tx_num, nout) - - @classmethod - def unpack_key(cls, key: bytes) -> UTXOKey: - return UTXOKey(*super().unpack_key(key)) - - @classmethod - def pack_value(cls, amount: int) -> bytes: - return super().pack_value(amount) - - @classmethod - def unpack_value(cls, data: bytes) -> UTXOValue: - return UTXOValue(*cls.value_struct.unpack(data)) - - @classmethod - def pack_item(cls, hashX: bytes, tx_num: int, nout: int, amount: int): - return cls.pack_key(hashX, tx_num, nout), cls.pack_value(amount) - - -class HashXUTXOPrefixRow(PrefixRow): - prefix = DB_PREFIXES.hashx_utxo.value - key_struct = struct.Struct(b'>4sLH') - value_struct = struct.Struct(b'>11s') - - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>4s').pack, - struct.Struct(b'>4sL').pack, - struct.Struct(b'>4sLH').pack - ] - - @classmethod - def pack_key(cls, short_tx_hash: bytes, tx_num, nout: int): - return super().pack_key(short_tx_hash, tx_num, nout) - - @classmethod - def unpack_key(cls, key: bytes) -> HashXUTXOKey: - return HashXUTXOKey(*super().unpack_key(key)) - - @classmethod - def pack_value(cls, hashX: bytes) -> bytes: - return super().pack_value(hashX) - - @classmethod - def unpack_value(cls, data: bytes) -> HashXUTXOValue: - return HashXUTXOValue(*cls.value_struct.unpack(data)) - - @classmethod - def pack_item(cls, short_tx_hash: bytes, tx_num: int, nout: int, hashX: bytes): - return cls.pack_key(short_tx_hash, tx_num, nout), cls.pack_value(hashX) - - -class HashXHistoryPrefixRow(PrefixRow): - prefix = DB_PREFIXES.hashx_history.value - key_struct = struct.Struct(b'>11sL') - - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>11s').pack, - struct.Struct(b'>11sL').pack - ] - - @classmethod - def pack_key(cls, hashX: bytes, height: int): - return super().pack_key(hashX, height) - - @classmethod - def unpack_key(cls, key: bytes) -> HashXHistoryKey: - return HashXHistoryKey(*super().unpack_key(key)) - - @classmethod - def pack_value(cls, history: typing.List[int]) -> bytes: - a = array.array('I') - a.fromlist(history) - return a.tobytes() - - @classmethod - def unpack_value(cls, data: bytes) -> array.array: - a = array.array('I') - a.frombytes(data) - return a - - @classmethod - def pack_item(cls, hashX: bytes, height: int, history: typing.List[int]): - return cls.pack_key(hashX, height), cls.pack_value(history) - - -class TouchedOrDeletedPrefixRow(PrefixRow): - prefix = DB_PREFIXES.touched_or_deleted.value - key_struct = struct.Struct(b'>L') - value_struct = struct.Struct(b'>LL') - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>L').pack - ] - - @classmethod - def pack_key(cls, height: int): - return super().pack_key(height) - - @classmethod - def unpack_key(cls, key: bytes) -> TouchedOrDeletedClaimKey: - return TouchedOrDeletedClaimKey(*super().unpack_key(key)) - - @classmethod - def pack_value(cls, touched: typing.Set[bytes], deleted: typing.Set[bytes]) -> bytes: - assert True if not touched else all(len(item) == 20 for item in touched) - assert True if not deleted else all(len(item) == 20 for item in deleted) - return cls.value_struct.pack(len(touched), len(deleted)) + b''.join(sorted(touched)) + b''.join(sorted(deleted)) - - @classmethod - def unpack_value(cls, data: bytes) -> TouchedOrDeletedClaimValue: - touched_len, deleted_len = cls.value_struct.unpack(data[:8]) - data = data[8:] - assert len(data) == 20 * (touched_len + deleted_len) - touched_bytes, deleted_bytes = data[:touched_len*20], data[touched_len*20:] - return TouchedOrDeletedClaimValue( - {touched_bytes[20*i:20*(i+1)] for i in range(touched_len)}, - {deleted_bytes[20*i:20*(i+1)] for i in range(deleted_len)} - ) - - @classmethod - def pack_item(cls, height, touched, deleted): - return cls.pack_key(height), cls.pack_value(touched, deleted) - - -class ChannelCountPrefixRow(PrefixRow): - prefix = DB_PREFIXES.channel_count.value - key_struct = struct.Struct(b'>20s') - value_struct = struct.Struct(b'>L') - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>20s').pack - ] - - @classmethod - def pack_key(cls, channel_hash: bytes): - return super().pack_key(channel_hash) - - @classmethod - def unpack_key(cls, key: bytes) -> ChannelCountKey: - return ChannelCountKey(*super().unpack_key(key)) - - @classmethod - def pack_value(cls, count: int) -> bytes: - return super().pack_value(count) - - @classmethod - def unpack_value(cls, data: bytes) -> ChannelCountValue: - return ChannelCountValue(*super().unpack_value(data)) - - @classmethod - def pack_item(cls, channel_hash, count): - return cls.pack_key(channel_hash), cls.pack_value(count) - - -class SupportAmountPrefixRow(PrefixRow): - prefix = DB_PREFIXES.support_amount.value - key_struct = struct.Struct(b'>20s') - value_struct = struct.Struct(b'>Q') - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>20s').pack - ] - - @classmethod - def pack_key(cls, claim_hash: bytes): - return super().pack_key(claim_hash) - - @classmethod - def unpack_key(cls, key: bytes) -> SupportAmountKey: - return SupportAmountKey(*super().unpack_key(key)) - - @classmethod - def pack_value(cls, amount: int) -> bytes: - return super().pack_value(amount) - - @classmethod - def unpack_value(cls, data: bytes) -> SupportAmountValue: - return SupportAmountValue(*super().unpack_value(data)) - - @classmethod - def pack_item(cls, claim_hash, amount): - return cls.pack_key(claim_hash), cls.pack_value(amount) - - -class DBStatePrefixRow(PrefixRow): - prefix = DB_PREFIXES.db_state.value - value_struct = struct.Struct(b'>32sLL32sLLBBlllL') - key_struct = struct.Struct(b'') - - key_part_lambdas = [ - lambda: b'' - ] - - @classmethod - def pack_key(cls) -> bytes: - return cls.prefix - - @classmethod - def unpack_key(cls, key: bytes): - return - - @classmethod - def pack_value(cls, genesis: bytes, height: int, tx_count: int, tip: bytes, utxo_flush_count: int, wall_time: int, - first_sync: bool, db_version: int, hist_flush_count: int, comp_flush_count: int, - comp_cursor: int, es_sync_height: int) -> bytes: - return super().pack_value( - genesis, height, tx_count, tip, utxo_flush_count, - wall_time, 1 if first_sync else 0, db_version, hist_flush_count, - comp_flush_count, comp_cursor, es_sync_height - ) - - @classmethod - def unpack_value(cls, data: bytes) -> DBState: - if len(data) == 94: - # TODO: delete this after making a new snapshot - 10/20/21 - # migrate in the es_sync_height if it doesnt exist - data += data[32:36] - return DBState(*super().unpack_value(data)) - - @classmethod - def pack_item(cls, genesis: bytes, height: int, tx_count: int, tip: bytes, utxo_flush_count: int, wall_time: int, - first_sync: bool, db_version: int, hist_flush_count: int, comp_flush_count: int, - comp_cursor: int, es_sync_height: int): - return cls.pack_key(), cls.pack_value( - genesis, height, tx_count, tip, utxo_flush_count, wall_time, first_sync, db_version, hist_flush_count, - comp_flush_count, comp_cursor, es_sync_height - ) - - -class BlockTxsPrefixRow(PrefixRow): - prefix = DB_PREFIXES.block_tx.value - key_struct = struct.Struct(b'>L') - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>L').pack - ] - - @classmethod - def pack_key(cls, height: int): - return super().pack_key(height) - - @classmethod - def unpack_key(cls, key: bytes) -> BlockTxsKey: - return BlockTxsKey(*super().unpack_key(key)) - - @classmethod - def pack_value(cls, tx_hashes: typing.List[bytes]) -> bytes: - assert all(len(tx_hash) == 32 for tx_hash in tx_hashes) - return b''.join(tx_hashes) - - @classmethod - def unpack_value(cls, data: bytes) -> BlockTxsValue: - return BlockTxsValue([data[i*32:(i+1)*32] for i in range(len(data) // 32)]) - - @classmethod - def pack_item(cls, height, tx_hashes): - return cls.pack_key(height), cls.pack_value(tx_hashes) - - -class MempoolTxKey(NamedTuple): - tx_hash: bytes - - def __str__(self): - return f"{self.__class__.__name__}(tx_hash={self.tx_hash[::-1].hex()})" - - -class MempoolTxValue(NamedTuple): - raw_tx: bytes - - def __str__(self): - return f"{self.__class__.__name__}(raw_tx={base64.b64encode(self.raw_tx).decode()})" - - -class MempoolTXPrefixRow(PrefixRow): - prefix = DB_PREFIXES.mempool_tx.value - key_struct = struct.Struct(b'>32s') - - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>32s').pack - ] - - @classmethod - def pack_key(cls, tx_hash: bytes) -> bytes: - return super().pack_key(tx_hash) - - @classmethod - def unpack_key(cls, tx_hash: bytes) -> MempoolTxKey: - return MempoolTxKey(*super().unpack_key(tx_hash)) - - @classmethod - def pack_value(cls, tx: bytes) -> bytes: - return tx - - @classmethod - def unpack_value(cls, data: bytes) -> MempoolTxValue: - return MempoolTxValue(data) - - @classmethod - def pack_item(cls, tx_hash: bytes, raw_tx: bytes): - return cls.pack_key(tx_hash), cls.pack_value(raw_tx) - - -class TrendingNotificationKey(typing.NamedTuple): - height: int - claim_hash: bytes - - -class TrendingNotificationValue(typing.NamedTuple): - previous_amount: int - new_amount: int - - -class TrendingNotificationPrefixRow(PrefixRow): - prefix = DB_PREFIXES.trending_notifications.value - key_struct = struct.Struct(b'>L20s') - value_struct = struct.Struct(b'>QQ') - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>L').pack, - struct.Struct(b'>L20s').pack - ] - - @classmethod - def pack_key(cls, height: int, claim_hash: bytes): - return super().pack_key(height, claim_hash) - - @classmethod - def unpack_key(cls, key: bytes) -> TrendingNotificationKey: - return TrendingNotificationKey(*super().unpack_key(key)) - - @classmethod - def pack_value(cls, previous_amount: int, new_amount: int) -> bytes: - return super().pack_value(previous_amount, new_amount) - - @classmethod - def unpack_value(cls, data: bytes) -> TrendingNotificationValue: - return TrendingNotificationValue(*super().unpack_value(data)) - - @classmethod - def pack_item(cls, height, claim_hash, previous_amount, new_amount): - return cls.pack_key(height, claim_hash), cls.pack_value(previous_amount, new_amount) - - -class TouchedHashXKey(NamedTuple): - height: int - - -class TouchedHashXValue(NamedTuple): - touched_hashXs: typing.List[bytes] - - def __str__(self): - return f"{self.__class__.__name__}(touched_hashXs=[{', '.join(map(lambda x: x.hex(), self.touched_hashXs))}])" - - -class TouchedHashXPrefixRow(PrefixRow): - prefix = DB_PREFIXES.touched_hashX.value - key_struct = struct.Struct(b'>L') - - key_part_lambdas = [ - lambda: b'', - struct.Struct(b'>L').pack - ] - - @classmethod - def pack_key(cls, height: int): - return super().pack_key(height) - - @classmethod - def unpack_key(cls, key: bytes) -> TouchedHashXKey: - return TouchedHashXKey(*super().unpack_key(key)) - - @classmethod - def pack_value(cls, touched: typing.List[bytes]) -> bytes: - assert all(map(lambda item: len(item) == 11, touched)) - return b''.join(touched) - - @classmethod - def unpack_value(cls, data: bytes) -> TouchedHashXValue: - return TouchedHashXValue([data[idx*11:(idx*11)+11] for idx in range(len(data) // 11)]) - - @classmethod - def pack_item(cls, height: int, touched: typing.List[bytes]): - return cls.pack_key(height), cls.pack_value(touched) - - -class PrefixDB(BasePrefixDB): - def __init__(self, path: str, cache_mb: int = 128, reorg_limit: int = 200, max_open_files: int = 64, - secondary_path: str = '', unsafe_prefixes: Optional[typing.Set[bytes]] = None): - super().__init__(path, max_open_files=max_open_files, secondary_path=secondary_path, - max_undo_depth=reorg_limit, unsafe_prefixes=unsafe_prefixes) - db = self._db - self.claim_to_support = ClaimToSupportPrefixRow(db, self._op_stack) - self.support_to_claim = SupportToClaimPrefixRow(db, self._op_stack) - self.claim_to_txo = ClaimToTXOPrefixRow(db, self._op_stack) - self.txo_to_claim = TXOToClaimPrefixRow(db, self._op_stack) - self.claim_to_channel = ClaimToChannelPrefixRow(db, self._op_stack) - self.channel_to_claim = ChannelToClaimPrefixRow(db, self._op_stack) - self.claim_short_id = ClaimShortIDPrefixRow(db, self._op_stack) - self.claim_expiration = ClaimExpirationPrefixRow(db, self._op_stack) - self.claim_takeover = ClaimTakeoverPrefixRow(db, self._op_stack) - self.pending_activation = PendingActivationPrefixRow(db, self._op_stack) - self.activated = ActivatedPrefixRow(db, self._op_stack) - self.active_amount = ActiveAmountPrefixRow(db, self._op_stack) - self.effective_amount = EffectiveAmountPrefixRow(db, self._op_stack) - self.repost = RepostPrefixRow(db, self._op_stack) - self.reposted_claim = RepostedPrefixRow(db, self._op_stack) - self.undo = UndoPrefixRow(db, self._op_stack) - self.utxo = UTXOPrefixRow(db, self._op_stack) - self.hashX_utxo = HashXUTXOPrefixRow(db, self._op_stack) - self.hashX_history = HashXHistoryPrefixRow(db, self._op_stack) - self.block_hash = BlockHashPrefixRow(db, self._op_stack) - self.tx_count = TxCountPrefixRow(db, self._op_stack) - self.tx_hash = TXHashPrefixRow(db, self._op_stack) - self.tx_num = TXNumPrefixRow(db, self._op_stack) - self.tx = TXPrefixRow(db, self._op_stack) - self.header = BlockHeaderPrefixRow(db, self._op_stack) - self.touched_or_deleted = TouchedOrDeletedPrefixRow(db, self._op_stack) - self.channel_count = ChannelCountPrefixRow(db, self._op_stack) - self.db_state = DBStatePrefixRow(db, self._op_stack) - self.support_amount = SupportAmountPrefixRow(db, self._op_stack) - self.block_txs = BlockTxsPrefixRow(db, self._op_stack) - self.mempool_tx = MempoolTXPrefixRow(db, self._op_stack) - self.trending_notification = TrendingNotificationPrefixRow(db, self._op_stack) - self.touched_hashX = TouchedHashXPrefixRow(db, self._op_stack) - - -def auto_decode_item(key: bytes, value: bytes) -> Union[Tuple[NamedTuple, NamedTuple], Tuple[bytes, bytes]]: - try: - return ROW_TYPES[key[:1]].unpack_item(key, value) - except KeyError: - return key, value diff --git a/lbry/wallet/server/db/revertable.py b/lbry/wallet/server/db/revertable.py deleted file mode 100644 index e59bbcdf3..000000000 --- a/lbry/wallet/server/db/revertable.py +++ /dev/null @@ -1,175 +0,0 @@ -import struct -import logging -from string import printable -from collections import defaultdict -from typing import Tuple, Iterable, Callable, Optional -from lbry.wallet.server.db import DB_PREFIXES - -_OP_STRUCT = struct.Struct('>BLL') -log = logging.getLogger() - - -class RevertableOp: - __slots__ = [ - 'key', - 'value', - ] - is_put = 0 - - def __init__(self, key: bytes, value: bytes): - self.key = key - self.value = value - - @property - def is_delete(self) -> bool: - return not self.is_put - - def invert(self) -> 'RevertableOp': - raise NotImplementedError() - - def pack(self) -> bytes: - """ - Serialize to bytes - """ - return struct.pack( - f'>BLL{len(self.key)}s{len(self.value)}s', int(self.is_put), len(self.key), len(self.value), self.key, - self.value - ) - - @classmethod - def unpack(cls, packed: bytes) -> Tuple['RevertableOp', bytes]: - """ - Deserialize from bytes - - :param packed: bytes containing at least one packed revertable op - :return: tuple of the deserialized op (a put or a delete) and the remaining serialized bytes - """ - is_put, key_len, val_len = _OP_STRUCT.unpack(packed[:9]) - key = packed[9:9 + key_len] - value = packed[9 + key_len:9 + key_len + val_len] - if is_put == 1: - return RevertablePut(key, value), packed[9 + key_len + val_len:] - return RevertableDelete(key, value), packed[9 + key_len + val_len:] - - def __eq__(self, other: 'RevertableOp') -> bool: - return (self.is_put, self.key, self.value) == (other.is_put, other.key, other.value) - - def __repr__(self) -> str: - return str(self) - - def __str__(self) -> str: - from lbry.wallet.server.db.prefixes import auto_decode_item - k, v = auto_decode_item(self.key, self.value) - key = ''.join(c if c in printable else '.' for c in str(k)) - val = ''.join(c if c in printable else '.' for c in str(v)) - return f"{'PUT' if self.is_put else 'DELETE'} {DB_PREFIXES(self.key[:1]).name}: {key} | {val}" - - -class RevertableDelete(RevertableOp): - def invert(self): - return RevertablePut(self.key, self.value) - - -class RevertablePut(RevertableOp): - is_put = True - - def invert(self): - return RevertableDelete(self.key, self.value) - - -class OpStackIntegrity(Exception): - pass - - -class RevertableOpStack: - def __init__(self, get_fn: Callable[[bytes], Optional[bytes]], unsafe_prefixes=None): - """ - This represents a sequence of revertable puts and deletes to a key-value database that checks for integrity - violations when applying the puts and deletes. The integrity checks assure that keys that do not exist - are not deleted, and that when keys are deleted the current value is correctly known so that the delete - may be undone. When putting values, the integrity checks assure that existing values are not overwritten - without first being deleted. Updates are performed by applying a delete op for the old value and a put op - for the new value. - - :param get_fn: getter function from an object implementing `KeyValueStorage` - :param unsafe_prefixes: optional set of prefixes to ignore integrity errors for, violations are still logged - """ - self._get = get_fn - self._items = defaultdict(list) - self._unsafe_prefixes = unsafe_prefixes or set() - - def append_op(self, op: RevertableOp): - """ - Apply a put or delete op, checking that it introduces no integrity errors - """ - - inverted = op.invert() - if self._items[op.key] and inverted == self._items[op.key][-1]: - self._items[op.key].pop() # if the new op is the inverse of the last op, we can safely null both - return - elif self._items[op.key] and self._items[op.key][-1] == op: # duplicate of last op - return # raise an error? - stored_val = self._get(op.key) - has_stored_val = stored_val is not None - delete_stored_op = None if not has_stored_val else RevertableDelete(op.key, stored_val) - will_delete_existing_stored = False if delete_stored_op is None else (delete_stored_op in self._items[op.key]) - try: - if op.is_put and has_stored_val and not will_delete_existing_stored: - raise OpStackIntegrity( - f"db op tries to add on top of existing key without deleting first: {op}" - ) - elif op.is_delete and has_stored_val and stored_val != op.value and not will_delete_existing_stored: - # there is a value and we're not deleting it in this op - # check that a delete for the stored value is in the stack - raise OpStackIntegrity(f"db op tries to delete with incorrect existing value {op}") - elif op.is_delete and not has_stored_val: - raise OpStackIntegrity(f"db op tries to delete nonexistent key: {op}") - elif op.is_delete and stored_val != op.value: - raise OpStackIntegrity(f"db op tries to delete with incorrect value: {op}") - except OpStackIntegrity as err: - if op.key[:1] in self._unsafe_prefixes: - log.debug(f"skipping over integrity error: {err}") - else: - raise err - self._items[op.key].append(op) - - def extend_ops(self, ops: Iterable[RevertableOp]): - """ - Apply a sequence of put or delete ops, checking that they introduce no integrity errors - """ - for op in ops: - self.append_op(op) - - def clear(self): - self._items.clear() - - def __len__(self): - return sum(map(len, self._items.values())) - - def __iter__(self): - for key, ops in self._items.items(): - for op in ops: - yield op - - def __reversed__(self): - for key, ops in self._items.items(): - for op in reversed(ops): - yield op - - def get_undo_ops(self) -> bytes: - """ - Get the serialized bytes to undo all of the changes made by the pending ops - """ - return b''.join(op.invert().pack() for op in reversed(self)) - - def apply_packed_undo_ops(self, packed: bytes): - """ - Unpack and apply a sequence of undo ops from serialized undo bytes - """ - while packed: - op, packed = RevertableOp.unpack(packed) - self.append_op(op) - - def get_last_op_for_key(self, key: bytes) -> Optional[RevertableOp]: - if key in self._items and self._items[key]: - return self._items[key][-1] diff --git a/lbry/wallet/server/env.py b/lbry/wallet/server/env.py deleted file mode 100644 index e872b8262..000000000 --- a/lbry/wallet/server/env.py +++ /dev/null @@ -1,397 +0,0 @@ -# Copyright (c) 2016, Neil Booth -# -# All rights reserved. -# -# See the file "LICENCE" for information about the copyright -# and warranty status of this software. - -import math -import re -import resource -from os import environ -from collections import namedtuple -from ipaddress import ip_address - -from lbry.wallet.server.util import class_logger -from lbry.wallet.server.coin import Coin, LBC, LBCTestNet, LBCRegTest -import lbry.wallet.server.util as lib_util - - -NetIdentity = namedtuple('NetIdentity', 'host tcp_port ssl_port nick_suffix') - - -class Env: - - # Peer discovery - PD_OFF, PD_SELF, PD_ON = range(3) - - class Error(Exception): - pass - - def __init__(self, coin=None, db_dir=None, daemon_url=None, host=None, rpc_host=None, elastic_host=None, - elastic_port=None, loop_policy=None, max_query_workers=None, websocket_host=None, websocket_port=None, - chain=None, es_index_prefix=None, cache_MB=None, reorg_limit=None, tcp_port=None, - udp_port=None, ssl_port=None, ssl_certfile=None, ssl_keyfile=None, rpc_port=None, - prometheus_port=None, max_subscriptions=None, banner_file=None, anon_logs=None, log_sessions=None, - allow_lan_udp=None, cache_all_tx_hashes=None, cache_all_claim_txos=None, country=None, - payment_address=None, donation_address=None, max_send=None, max_receive=None, max_sessions=None, - session_timeout=None, drop_client=None, description=None, daily_fee=None, - database_query_timeout=None, db_max_open_files=64, elastic_notifier_port=None, - blocking_channel_ids=None, filtering_channel_ids=None, peer_hubs=None, peer_announce=None): - self.logger = class_logger(__name__, self.__class__.__name__) - - self.db_dir = db_dir if db_dir is not None else self.required('DB_DIRECTORY') - self.daemon_url = daemon_url if daemon_url is not None else self.required('DAEMON_URL') - self.db_max_open_files = db_max_open_files - - self.host = host if host is not None else self.default('HOST', 'localhost') - self.rpc_host = rpc_host if rpc_host is not None else self.default('RPC_HOST', 'localhost') - self.elastic_host = elastic_host if elastic_host is not None else self.default('ELASTIC_HOST', 'localhost') - self.elastic_port = elastic_port if elastic_port is not None else self.integer('ELASTIC_PORT', 9200) - self.elastic_notifier_port = elastic_notifier_port if elastic_notifier_port is not None else self.integer('ELASTIC_NOTIFIER_PORT', 19080) - - self.loop_policy = self.set_event_loop_policy( - loop_policy if loop_policy is not None else self.default('EVENT_LOOP_POLICY', None) - ) - self.obsolete(['UTXO_MB', 'HIST_MB', 'NETWORK']) - self.max_query_workers = max_query_workers if max_query_workers is not None else self.integer('MAX_QUERY_WORKERS', 4) - self.websocket_host = websocket_host if websocket_host is not None else self.default('WEBSOCKET_HOST', self.host) - self.websocket_port = websocket_port if websocket_port is not None else self.integer('WEBSOCKET_PORT', None) - if coin is not None: - assert issubclass(coin, Coin) - self.coin = coin - else: - chain = chain if chain is not None else self.default('NET', 'mainnet').strip().lower() - if chain == 'mainnet': - self.coin = LBC - elif chain == 'testnet': - self.coin = LBCTestNet - else: - self.coin = LBCRegTest - self.es_index_prefix = es_index_prefix if es_index_prefix is not None else self.default('ES_INDEX_PREFIX', '') - self.cache_MB = cache_MB if cache_MB is not None else self.integer('CACHE_MB', 1024) - self.reorg_limit = reorg_limit if reorg_limit is not None else self.integer('REORG_LIMIT', self.coin.REORG_LIMIT) - # Server stuff - self.tcp_port = tcp_port if tcp_port is not None else self.integer('TCP_PORT', None) - self.udp_port = udp_port if udp_port is not None else self.integer('UDP_PORT', self.tcp_port) - self.ssl_port = ssl_port if ssl_port is not None else self.integer('SSL_PORT', None) - if self.ssl_port: - self.ssl_certfile = ssl_certfile if ssl_certfile is not None else self.required('SSL_CERTFILE') - self.ssl_keyfile = ssl_keyfile if ssl_keyfile is not None else self.required('SSL_KEYFILE') - self.rpc_port = rpc_port if rpc_port is not None else self.integer('RPC_PORT', 8000) - self.prometheus_port = prometheus_port if prometheus_port is not None else self.integer('PROMETHEUS_PORT', 0) - self.max_subscriptions = max_subscriptions if max_subscriptions is not None else self.integer('MAX_SUBSCRIPTIONS', 10000) - self.banner_file = banner_file if banner_file is not None else self.default('BANNER_FILE', None) - # self.tor_banner_file = self.default('TOR_BANNER_FILE', self.banner_file) - self.anon_logs = anon_logs if anon_logs is not None else self.boolean('ANON_LOGS', False) - self.log_sessions = log_sessions if log_sessions is not None else self.integer('LOG_SESSIONS', 3600) - self.allow_lan_udp = allow_lan_udp if allow_lan_udp is not None else self.boolean('ALLOW_LAN_UDP', False) - self.cache_all_tx_hashes = cache_all_tx_hashes if cache_all_tx_hashes is not None else self.boolean('CACHE_ALL_TX_HASHES', False) - self.cache_all_claim_txos = cache_all_claim_txos if cache_all_claim_txos is not None else self.boolean('CACHE_ALL_CLAIM_TXOS', False) - self.country = country if country is not None else self.default('COUNTRY', 'US') - # Peer discovery - self.peer_discovery = self.peer_discovery_enum() - self.peer_announce = peer_announce if peer_announce is not None else self.boolean('PEER_ANNOUNCE', True) - if peer_hubs is not None: - self.peer_hubs = [p.strip("") for p in peer_hubs.split(",")] - else: - self.peer_hubs = self.extract_peer_hubs() - # self.tor_proxy_host = self.default('TOR_PROXY_HOST', 'localhost') - # self.tor_proxy_port = self.integer('TOR_PROXY_PORT', None) - # The electrum client takes the empty string as unspecified - self.payment_address = payment_address if payment_address is not None else self.default('PAYMENT_ADDRESS', '') - self.donation_address = donation_address if donation_address is not None else self.default('DONATION_ADDRESS', '') - # Server limits to help prevent DoS - self.max_send = max_send if max_send is not None else self.integer('MAX_SEND', 1000000) - self.max_receive = max_receive if max_receive is not None else self.integer('MAX_RECEIVE', 1000000) - # self.max_subs = self.integer('MAX_SUBS', 250000) - self.max_sessions = max_sessions if max_sessions is not None else self.sane_max_sessions() - # self.max_session_subs = self.integer('MAX_SESSION_SUBS', 50000) - self.session_timeout = session_timeout if session_timeout is not None else self.integer('SESSION_TIMEOUT', 600) - self.drop_client = drop_client if drop_client is not None else self.custom("DROP_CLIENT", None, re.compile) - self.description = description if description is not None else self.default('DESCRIPTION', '') - self.daily_fee = daily_fee if daily_fee is not None else self.string_amount('DAILY_FEE', '0') - - # Identities - clearnet_identity = self.clearnet_identity() - tor_identity = self.tor_identity(clearnet_identity) - self.identities = [identity - for identity in (clearnet_identity, tor_identity) - if identity is not None] - self.database_query_timeout = database_query_timeout if database_query_timeout is not None else \ - (float(self.integer('QUERY_TIMEOUT_MS', 10000)) / 1000.0) - - # Filtering / Blocking - self.blocking_channel_ids = blocking_channel_ids if blocking_channel_ids is not None else self.default('BLOCKING_CHANNEL_IDS', '').split(' ') - self.filtering_channel_ids = filtering_channel_ids if filtering_channel_ids is not None else self.default('FILTERING_CHANNEL_IDS', '').split(' ') - - @classmethod - def default(cls, envvar, default): - return environ.get(envvar, default) - - @classmethod - def boolean(cls, envvar, default): - default = 'Yes' if default else '' - return bool(cls.default(envvar, default).strip()) - - @classmethod - def required(cls, envvar): - value = environ.get(envvar) - if value is None: - raise cls.Error(f'required envvar {envvar} not set') - return value - - @classmethod - def string_amount(cls, envvar, default): - value = environ.get(envvar, default) - amount_pattern = re.compile("[0-9]{0,10}(\.[0-9]{1,8})?") - if len(value) > 0 and not amount_pattern.fullmatch(value): - raise cls.Error(f'{value} is not a valid amount for {envvar}') - return value - - @classmethod - def integer(cls, envvar, default): - value = environ.get(envvar) - if value is None: - return default - try: - return int(value) - except Exception: - raise cls.Error(f'cannot convert envvar {envvar} value {value} to an integer') - - @classmethod - def custom(cls, envvar, default, parse): - value = environ.get(envvar) - if value is None: - return default - try: - return parse(value) - except Exception as e: - raise cls.Error(f'cannot parse envvar {envvar} value {value}') from e - - @classmethod - def obsolete(cls, envvars): - bad = [envvar for envvar in envvars if environ.get(envvar)] - if bad: - raise cls.Error(f'remove obsolete environment variables {bad}') - - @classmethod - def set_event_loop_policy(cls, policy_name: str = None): - if not policy_name or policy_name == 'default': - import asyncio - return asyncio.get_event_loop_policy() - elif policy_name == 'uvloop': - import uvloop - import asyncio - loop_policy = uvloop.EventLoopPolicy() - asyncio.set_event_loop_policy(loop_policy) - return loop_policy - raise cls.Error(f'unknown event loop policy "{policy_name}"') - - def cs_host(self, *, for_rpc): - """Returns the 'host' argument to pass to asyncio's create_server - call. The result can be a single host name string, a list of - host name strings, or an empty string to bind to all interfaces. - - If rpc is True the host to use for the RPC server is returned. - Otherwise the host to use for SSL/TCP servers is returned. - """ - host = self.rpc_host if for_rpc else self.host - result = [part.strip() for part in host.split(',')] - if len(result) == 1: - result = result[0] - # An empty result indicates all interfaces, which we do not - # permitted for an RPC server. - if for_rpc and not result: - result = 'localhost' - if result == 'localhost': - # 'localhost' resolves to ::1 (ipv6) on many systems, which fails on default setup of - # docker, using 127.0.0.1 instead forces ipv4 - result = '127.0.0.1' - return result - - def sane_max_sessions(self): - """Return the maximum number of sessions to permit. Normally this - is MAX_SESSIONS. However, to prevent open file exhaustion, ajdust - downwards if running with a small open file rlimit.""" - env_value = self.integer('MAX_SESSIONS', 1000) - nofile_limit = resource.getrlimit(resource.RLIMIT_NOFILE)[0] - # We give the DB 250 files; allow ElectrumX 100 for itself - value = max(0, min(env_value, nofile_limit - 350)) - if value < env_value: - self.logger.warning(f'lowered maximum sessions from {env_value:,d} to {value:,d} ' - f'because your open file limit is {nofile_limit:,d}') - return value - - def clearnet_identity(self): - host = self.default('REPORT_HOST', None) - if host is None: - return None - try: - ip = ip_address(host) - except ValueError: - bad = (not lib_util.is_valid_hostname(host) - or host.lower() == 'localhost') - else: - bad = (ip.is_multicast or ip.is_unspecified - or (ip.is_private and self.peer_announce)) - if bad: - raise self.Error(f'"{host}" is not a valid REPORT_HOST') - tcp_port = self.integer('REPORT_TCP_PORT', self.tcp_port) or None - ssl_port = self.integer('REPORT_SSL_PORT', self.ssl_port) or None - if tcp_port == ssl_port: - raise self.Error('REPORT_TCP_PORT and REPORT_SSL_PORT ' - f'both resolve to {tcp_port}') - return NetIdentity( - host, - tcp_port, - ssl_port, - '' - ) - - def tor_identity(self, clearnet): - host = self.default('REPORT_HOST_TOR', None) - if host is None: - return None - if not host.endswith('.onion'): - raise self.Error(f'tor host "{host}" must end with ".onion"') - - def port(port_kind): - """Returns the clearnet identity port, if any and not zero, - otherwise the listening port.""" - result = 0 - if clearnet: - result = getattr(clearnet, port_kind) - return result or getattr(self, port_kind) - - tcp_port = self.integer('REPORT_TCP_PORT_TOR', - port('tcp_port')) or None - ssl_port = self.integer('REPORT_SSL_PORT_TOR', - port('ssl_port')) or None - if tcp_port == ssl_port: - raise self.Error('REPORT_TCP_PORT_TOR and REPORT_SSL_PORT_TOR ' - f'both resolve to {tcp_port}') - - return NetIdentity( - host, - tcp_port, - ssl_port, - '_tor', - ) - - def hosts_dict(self): - return {identity.host: {'tcp_port': identity.tcp_port, - 'ssl_port': identity.ssl_port} - for identity in self.identities} - - def peer_discovery_enum(self): - pd = self.default('PEER_DISCOVERY', 'on').strip().lower() - if pd in ('off', ''): - return self.PD_OFF - elif pd == 'self': - return self.PD_SELF - else: - return self.PD_ON - - def extract_peer_hubs(self): - return [hub.strip() for hub in self.default('PEER_HUBS', '').split(',') if hub.strip()] - - @classmethod - def contribute_to_arg_parser(cls, parser): - parser.add_argument('--db_dir', type=str, help='path of the directory containing lbry-leveldb', - default=cls.default('DB_DIRECTORY', None)) - parser.add_argument('--daemon_url', - help='URL for rpc from lbrycrd, :@', - default=cls.default('DAEMON_URL', None)) - parser.add_argument('--db_max_open_files', type=int, default=64, - help='number of files rocksdb can have open at a time') - parser.add_argument('--host', type=str, default=cls.default('HOST', 'localhost'), - help='Interface for hub server to listen on') - parser.add_argument('--tcp_port', type=int, default=cls.integer('TCP_PORT', 50001), - help='TCP port to listen on for hub server') - parser.add_argument('--udp_port', type=int, default=cls.integer('UDP_PORT', 50001), - help='UDP port to listen on for hub server') - parser.add_argument('--rpc_host', default=cls.default('RPC_HOST', 'localhost'), type=str, - help='Listening interface for admin rpc') - parser.add_argument('--rpc_port', default=cls.integer('RPC_PORT', 8000), type=int, - help='Listening port for admin rpc') - parser.add_argument('--websocket_host', default=cls.default('WEBSOCKET_HOST', 'localhost'), type=str, - help='Listening interface for websocket') - parser.add_argument('--websocket_port', default=cls.integer('WEBSOCKET_PORT', None), type=int, - help='Listening port for websocket') - - parser.add_argument('--ssl_port', default=cls.integer('SSL_PORT', None), type=int, - help='SSL port to listen on for hub server') - parser.add_argument('--ssl_certfile', default=cls.default('SSL_CERTFILE', None), type=str, - help='Path to SSL cert file') - parser.add_argument('--ssl_keyfile', default=cls.default('SSL_KEYFILE', None), type=str, - help='Path to SSL key file') - parser.add_argument('--reorg_limit', default=cls.integer('REORG_LIMIT', 200), type=int, help='Max reorg depth') - parser.add_argument('--elastic_host', default=cls.default('ELASTIC_HOST', 'localhost'), type=str, - help='elasticsearch host') - parser.add_argument('--elastic_port', default=cls.integer('ELASTIC_PORT', 9200), type=int, - help='elasticsearch port') - parser.add_argument('--es_index_prefix', default=cls.default('ES_INDEX_PREFIX', ''), type=str) - parser.add_argument('--loop_policy', default=cls.default('EVENT_LOOP_POLICY', 'default'), type=str, - choices=['default', 'uvloop']) - parser.add_argument('--max_query_workers', type=int, default=cls.integer('MAX_QUERY_WORKERS', 4), - help='number of threads used by the request handler to read the database') - parser.add_argument('--cache_MB', type=int, default=cls.integer('CACHE_MB', 1024), - help='size of the leveldb lru cache, in megabytes') - parser.add_argument('--cache_all_tx_hashes', type=bool, - help='Load all tx hashes into memory. This will make address subscriptions and sync, ' - 'resolve, transaction fetching, and block sync all faster at the expense of higher ' - 'memory usage') - parser.add_argument('--cache_all_claim_txos', type=bool, - help='Load all claim txos into memory. This will make address subscriptions and sync, ' - 'resolve, transaction fetching, and block sync all faster at the expense of higher ' - 'memory usage') - parser.add_argument('--prometheus_port', type=int, default=cls.integer('PROMETHEUS_PORT', 0), - help='port for hub prometheus metrics to listen on, disabled by default') - parser.add_argument('--max_subscriptions', type=int, default=cls.integer('MAX_SUBSCRIPTIONS', 10000), - help='max subscriptions per connection') - parser.add_argument('--banner_file', type=str, default=cls.default('BANNER_FILE', None), - help='path to file containing banner text') - parser.add_argument('--anon_logs', type=bool, default=cls.boolean('ANON_LOGS', False), - help="don't log ip addresses") - parser.add_argument('--allow_lan_udp', type=bool, default=cls.boolean('ALLOW_LAN_UDP', False), - help='reply to hub UDP ping messages from LAN ip addresses') - parser.add_argument('--country', type=str, default=cls.default('COUNTRY', 'US'), help='') - parser.add_argument('--max_send', type=int, default=cls.default('MAX_SEND', 1000000), help='') - parser.add_argument('--max_receive', type=int, default=cls.default('MAX_RECEIVE', 1000000), help='') - parser.add_argument('--max_sessions', type=int, default=cls.default('MAX_SESSIONS', 1000), help='') - parser.add_argument('--session_timeout', type=int, default=cls.default('SESSION_TIMEOUT', 600), help='') - parser.add_argument('--drop_client', type=str, default=cls.default('DROP_CLIENT', None), help='') - parser.add_argument('--description', type=str, default=cls.default('DESCRIPTION', ''), help='') - parser.add_argument('--daily_fee', type=float, default=cls.default('DAILY_FEE', 0.0), help='') - parser.add_argument('--payment_address', type=str, default=cls.default('PAYMENT_ADDRESS', ''), help='') - parser.add_argument('--donation_address', type=str, default=cls.default('DONATION_ADDRESS', ''), help='') - parser.add_argument('--chain', type=str, default=cls.default('NET', 'mainnet'), - help="Which chain to use, default is mainnet") - parser.add_argument('--query_timeout_ms', type=int, default=cls.integer('QUERY_TIMEOUT_MS', 10000), - help="elasticsearch query timeout") - - parser.add_argument('--blocking_channel_ids', nargs='*', help='', - default=cls.default('BLOCKING_CHANNEL_IDS', '').split(' ')) - parser.add_argument('--filtering_channel_ids', nargs='*', help='', - default=cls.default('FILTERING_CHANNEL_IDS', '').split(' ')) - - @classmethod - def from_arg_parser(cls, args): - return cls( - db_dir=args.db_dir, daemon_url=args.daemon_url, db_max_open_files=args.db_max_open_files, - host=args.host, rpc_host=args.rpc_host, elastic_host=args.elastic_host, elastic_port=args.elastic_port, - loop_policy=args.loop_policy, max_query_workers=args.max_query_workers, websocket_host=args.websocket_host, - websocket_port=args.websocket_port, chain=args.chain, es_index_prefix=args.es_index_prefix, - cache_MB=args.cache_MB, reorg_limit=args.reorg_limit, tcp_port=args.tcp_port, - udp_port=args.udp_port, ssl_port=args.ssl_port, ssl_certfile=args.ssl_certfile, - ssl_keyfile=args.ssl_keyfile, rpc_port=args.rpc_port, prometheus_port=args.prometheus_port, - max_subscriptions=args.max_subscriptions, banner_file=args.banner_file, anon_logs=args.anon_logs, - log_sessions=None, allow_lan_udp=args.allow_lan_udp, - cache_all_tx_hashes=args.cache_all_tx_hashes, cache_all_claim_txos=args.cache_all_claim_txos, - country=args.country, payment_address=args.payment_address, donation_address=args.donation_address, - max_send=args.max_send, max_receive=args.max_receive, max_sessions=args.max_sessions, - session_timeout=args.session_timeout, drop_client=args.drop_client, description=args.description, - daily_fee=args.daily_fee, database_query_timeout=(args.query_timeout_ms / 1000), - blocking_channel_ids=args.blocking_channel_ids, filtering_channel_ids=args.filtering_channel_ids - ) diff --git a/lbry/wallet/server/hash.py b/lbry/wallet/server/hash.py deleted file mode 100644 index e9d088684..000000000 --- a/lbry/wallet/server/hash.py +++ /dev/null @@ -1,160 +0,0 @@ -# Copyright (c) 2016-2017, Neil Booth -# -# All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -"""Cryptograph hash functions and related classes.""" - - -import hashlib -import hmac - -from lbry.wallet.server.util import bytes_to_int, int_to_bytes, hex_to_bytes - -_sha256 = hashlib.sha256 -_sha512 = hashlib.sha512 -_new_hash = hashlib.new -_new_hmac = hmac.new -HASHX_LEN = 11 -CLAIM_HASH_LEN = 20 - - -def sha256(x): - """Simple wrapper of hashlib sha256.""" - return _sha256(x).digest() - - -def ripemd160(x): - """Simple wrapper of hashlib ripemd160.""" - h = _new_hash('ripemd160') - h.update(x) - return h.digest() - - -def double_sha256(x): - """SHA-256 of SHA-256, as used extensively in bitcoin.""" - return sha256(sha256(x)) - - -def hmac_sha512(key, msg): - """Use SHA-512 to provide an HMAC.""" - return _new_hmac(key, msg, _sha512).digest() - - -def hash160(x): - """RIPEMD-160 of SHA-256. - - Used to make bitcoin addresses from pubkeys.""" - return ripemd160(sha256(x)) - - -def hash_to_hex_str(x: bytes) -> str: - """Convert a big-endian binary hash to displayed hex string. - - Display form of a binary hash is reversed and converted to hex. - """ - return x[::-1].hex() - - -def hex_str_to_hash(x: str) -> bytes: - """Convert a displayed hex string to a binary hash.""" - return hex_to_bytes(x)[::-1] - - -class Base58Error(Exception): - """Exception used for Base58 errors.""" - - -class Base58: - """Class providing base 58 functionality.""" - - chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' - assert len(chars) == 58 - cmap = {c: n for n, c in enumerate(chars)} - - @staticmethod - def char_value(c): - val = Base58.cmap.get(c) - if val is None: - raise Base58Error(f'invalid base 58 character "{c}"') - return val - - @staticmethod - def decode(txt): - """Decodes txt into a big-endian bytearray.""" - if not isinstance(txt, str): - raise TypeError('a string is required') - - if not txt: - raise Base58Error('string cannot be empty') - - value = 0 - for c in txt: - value = value * 58 + Base58.char_value(c) - - result = int_to_bytes(value) - - # Prepend leading zero bytes if necessary - count = 0 - for c in txt: - if c != '1': - break - count += 1 - if count: - result = bytes(count) + result - - return result - - @staticmethod - def encode(be_bytes): - """Converts a big-endian bytearray into a base58 string.""" - value = bytes_to_int(be_bytes) - - txt = '' - while value: - value, mod = divmod(value, 58) - txt += Base58.chars[mod] - - for byte in be_bytes: - if byte != 0: - break - txt += '1' - - return txt[::-1] - - @staticmethod - def decode_check(txt, *, hash_fn=double_sha256): - """Decodes a Base58Check-encoded string to a payload. The version - prefixes it.""" - be_bytes = Base58.decode(txt) - result, check = be_bytes[:-4], be_bytes[-4:] - if check != hash_fn(result)[:4]: - raise Base58Error(f'invalid base 58 checksum for {txt}') - return result - - @staticmethod - def encode_check(payload, *, hash_fn=double_sha256): - """Encodes a payload bytearray (which includes the version byte(s)) - into a Base58Check string.""" - be_bytes = payload + hash_fn(payload)[:4] - return Base58.encode(be_bytes) diff --git a/lbry/wallet/server/mempool.py b/lbry/wallet/server/mempool.py deleted file mode 100644 index c5181cfb4..000000000 --- a/lbry/wallet/server/mempool.py +++ /dev/null @@ -1,207 +0,0 @@ -# Copyright (c) 2016-2018, Neil Booth -# -# All rights reserved. -# -# See the file "LICENCE" for information about the copyright -# and warranty status of this software. - -"""Mempool handling.""" -import asyncio -import itertools -import time -import attr -import typing -from collections import defaultdict -from prometheus_client import Histogram -from lbry.wallet.server.util import class_logger - -if typing.TYPE_CHECKING: - from lbry.wallet.server.session import LBRYSessionManager - from wallet.server.db.db import HubDB - - -@attr.s(slots=True) -class MemPoolTx: - prevouts = attr.ib() - # A pair is a (hashX, value) tuple - in_pairs = attr.ib() - out_pairs = attr.ib() - fee = attr.ib() - size = attr.ib() - raw_tx = attr.ib() - - -@attr.s(slots=True) -class MemPoolTxSummary: - hash = attr.ib() - fee = attr.ib() - has_unconfirmed_inputs = attr.ib() - - -NAMESPACE = "wallet_server" -HISTOGRAM_BUCKETS = ( - .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf') -) -mempool_process_time_metric = Histogram( - "processed_mempool", "Time to process mempool and notify touched addresses", - namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS -) - - -class MemPool: - def __init__(self, coin, db: 'HubDB', refresh_secs=1.0): - self.coin = coin - self._db = db - self.logger = class_logger(__name__, self.__class__.__name__) - self.txs = {} - self.raw_mempool = {} - self.touched_hashXs: typing.DefaultDict[bytes, typing.Set[bytes]] = defaultdict(set) # None can be a key - self.refresh_secs = refresh_secs - self.mempool_process_time_metric = mempool_process_time_metric - self.session_manager: typing.Optional['LBRYSessionManager'] = None - - def refresh(self) -> typing.Set[bytes]: # returns list of new touched hashXs - prefix_db = self._db.prefix_db - new_mempool = {k.tx_hash: v.raw_tx for k, v in prefix_db.mempool_tx.iterate()} # TODO: make this more efficient - self.raw_mempool.clear() - self.raw_mempool.update(new_mempool) - - # hashXs = self.hashXs # hashX: [tx_hash, ...] - touched_hashXs = set() - - # Remove txs that aren't in mempool anymore - for tx_hash in set(self.txs).difference(self.raw_mempool.keys()): - tx = self.txs.pop(tx_hash) - tx_hashXs = {hashX for hashX, value in tx.in_pairs}.union({hashX for hashX, value in tx.out_pairs}) - for hashX in tx_hashXs: - if hashX in self.touched_hashXs and tx_hash in self.touched_hashXs[hashX]: - self.touched_hashXs[hashX].remove(tx_hash) - if not self.touched_hashXs[hashX]: - self.touched_hashXs.pop(hashX) - touched_hashXs.update(tx_hashXs) - - # Re-sync with the new set of hashes - tx_map = {} - for tx_hash, raw_tx in self.raw_mempool.items(): - if tx_hash in self.txs: - continue - tx, tx_size = self.coin.DESERIALIZER(raw_tx).read_tx_and_vsize() - # Convert the inputs and outputs into (hashX, value) pairs - # Drop generation-like inputs from MemPoolTx.prevouts - txin_pairs = tuple((txin.prev_hash, txin.prev_idx) - for txin in tx.inputs - if not txin.is_generation()) - txout_pairs = tuple((self.coin.hashX_from_script(txout.pk_script), txout.value) - for txout in tx.outputs) - - tx_map[tx_hash] = MemPoolTx(None, txin_pairs, txout_pairs, 0, tx_size, raw_tx) - - for tx_hash, tx in tx_map.items(): - prevouts = [] - # Look up the prevouts - for prev_hash, prev_index in tx.in_pairs: - if prev_hash in self.txs: # accepted mempool - utxo = self.txs[prev_hash].out_pairs[prev_index] - elif prev_hash in tx_map: # this set of changes - utxo = tx_map[prev_hash].out_pairs[prev_index] - else: # get it from the db - prev_tx_num = prefix_db.tx_num.get(prev_hash) - if not prev_tx_num: - continue - prev_tx_num = prev_tx_num.tx_num - hashX_val = prefix_db.hashX_utxo.get(prev_hash[:4], prev_tx_num, prev_index) - if not hashX_val: - continue - hashX = hashX_val.hashX - utxo_value = prefix_db.utxo.get(hashX, prev_tx_num, prev_index) - utxo = (hashX, utxo_value.amount) - prevouts.append(utxo) - - # Save the prevouts, compute the fee and accept the TX - tx.prevouts = tuple(prevouts) - # Avoid negative fees if dealing with generation-like transactions - # because some in_parts would be missing - tx.fee = max(0, (sum(v for _, v in tx.prevouts) - - sum(v for _, v in tx.out_pairs))) - self.txs[tx_hash] = tx - # print(f"added {tx_hash[::-1].hex()} reader to mempool") - - for hashX, value in itertools.chain(tx.prevouts, tx.out_pairs): - self.touched_hashXs[hashX].add(tx_hash) - touched_hashXs.add(hashX) - return touched_hashXs - - def transaction_summaries(self, hashX): - """Return a list of MemPoolTxSummary objects for the hashX.""" - result = [] - for tx_hash in self.touched_hashXs.get(hashX, ()): - if tx_hash not in self.txs: - continue # the tx hash for the touched address is an input that isn't in mempool anymore - tx = self.txs[tx_hash] - has_ui = any(hash in self.txs for hash, idx in tx.in_pairs) - result.append(MemPoolTxSummary(tx_hash, tx.fee, has_ui)) - return result - - def get_mempool_height(self, tx_hash: bytes) -> int: - # Height Progression - # -2: not broadcast - # -1: in mempool but has unconfirmed inputs - # 0: in mempool and all inputs confirmed - # +num: confirmed in a specific block (height) - if tx_hash not in self.txs: - return -2 - tx = self.txs[tx_hash] - unspent_inputs = any(hash in self.raw_mempool for hash, idx in tx.in_pairs) - if unspent_inputs: - return -1 - return 0 - - async def start(self, height, session_manager: 'LBRYSessionManager'): - self.session_manager = session_manager - await self._notify_sessions(height, set(), set()) - - async def on_mempool(self, touched, new_touched, height): - await self._notify_sessions(height, touched, new_touched) - - async def on_block(self, touched, height): - await self._notify_sessions(height, touched, set()) - - async def _notify_sessions(self, height, touched, new_touched): - """Notify sessions about height changes and touched addresses.""" - height_changed = height != self.session_manager.notified_height - if height_changed: - await self.session_manager._refresh_hsub_results(height) - - if not self.session_manager.sessions: - return - - if height_changed: - header_tasks = [ - session.send_notification('blockchain.headers.subscribe', (self.session_manager.hsub_results[session.subscribe_headers_raw], )) - for session in self.session_manager.sessions.values() if session.subscribe_headers - ] - if header_tasks: - self.logger.info(f'notify {len(header_tasks)} sessions of new header') - asyncio.create_task(asyncio.wait(header_tasks)) - for hashX in touched.intersection(self.session_manager.mempool_statuses.keys()): - self.session_manager.mempool_statuses.pop(hashX, None) - # self.bp._chain_executor - await asyncio.get_event_loop().run_in_executor( - self._db._executor, touched.intersection_update, self.session_manager.hashx_subscriptions_by_session.keys() - ) - - if touched or new_touched or (height_changed and self.session_manager.mempool_statuses): - notified_hashxs = 0 - session_hashxes_to_notify = defaultdict(list) - to_notify = touched if height_changed else new_touched - - for hashX in to_notify: - if hashX not in self.session_manager.hashx_subscriptions_by_session: - continue - for session_id in self.session_manager.hashx_subscriptions_by_session[hashX]: - session_hashxes_to_notify[session_id].append(hashX) - notified_hashxs += 1 - for session_id, hashXes in session_hashxes_to_notify.items(): - asyncio.create_task(self.session_manager.sessions[session_id].send_history_notifications(*hashXes)) - if session_hashxes_to_notify: - self.logger.info(f'notified {len(session_hashxes_to_notify)} sessions/{notified_hashxs:,d} touched addresses') diff --git a/lbry/wallet/server/merkle.py b/lbry/wallet/server/merkle.py deleted file mode 100644 index 8cf1ca08b..000000000 --- a/lbry/wallet/server/merkle.py +++ /dev/null @@ -1,258 +0,0 @@ -# Copyright (c) 2018, Neil Booth -# -# All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# and warranty status of this software. - -"""Merkle trees, branches, proofs and roots.""" - -from asyncio import Event -from math import ceil, log - -from lbry.wallet.server.hash import double_sha256 - - -class Merkle: - """Perform merkle tree calculations on binary hashes using a given hash - function. - - If the hash count is not even, the final hash is repeated when - calculating the next merkle layer up the tree. - """ - - def __init__(self, hash_func=double_sha256): - self.hash_func = hash_func - - @staticmethod - def tree_depth(hash_count): - return Merkle.branch_length(hash_count) + 1 - - @staticmethod - def branch_length(hash_count): - """Return the length of a merkle branch given the number of hashes.""" - if not isinstance(hash_count, int): - raise TypeError('hash_count must be an integer') - if hash_count < 1: - raise ValueError('hash_count must be at least 1') - return ceil(log(hash_count, 2)) - - @staticmethod - def branch_and_root(hashes, index, length=None, hash_func=double_sha256): - """Return a (merkle branch, merkle_root) pair given hashes, and the - index of one of those hashes. - """ - hashes = list(hashes) - if not isinstance(index, int): - raise TypeError('index must be an integer') - # This also asserts hashes is not empty - if not 0 <= index < len(hashes): - raise ValueError(f"index '{index}/{len(hashes)}' out of range") - natural_length = Merkle.branch_length(len(hashes)) - if length is None: - length = natural_length - else: - if not isinstance(length, int): - raise TypeError('length must be an integer') - if length < natural_length: - raise ValueError('length out of range') - - branch = [] - for _ in range(length): - if len(hashes) & 1: - hashes.append(hashes[-1]) - branch.append(hashes[index ^ 1]) - index >>= 1 - hashes = [hash_func(hashes[n] + hashes[n + 1]) - for n in range(0, len(hashes), 2)] - - return branch, hashes[0] - - @staticmethod - def root(hashes, length=None): - """Return the merkle root of a non-empty iterable of binary hashes.""" - branch, root = Merkle.branch_and_root(hashes, 0, length) - return root - - # @staticmethod - # def root_from_proof(hash, branch, index, hash_func=double_sha256): - # """Return the merkle root given a hash, a merkle branch to it, and - # its index in the hashes array. - # - # branch is an iterable sorted deepest to shallowest. If the - # returned root is the expected value then the merkle proof is - # verified. - # - # The caller should have confirmed the length of the branch with - # branch_length(). Unfortunately this is not easily done for - # bitcoin transactions as the number of transactions in a block - # is unknown to an SPV client. - # """ - # for elt in branch: - # if index & 1: - # hash = hash_func(elt + hash) - # else: - # hash = hash_func(hash + elt) - # index >>= 1 - # if index: - # raise ValueError('index out of range for branch') - # return hash - - @staticmethod - def level(hashes, depth_higher): - """Return a level of the merkle tree of hashes the given depth - higher than the bottom row of the original tree.""" - size = 1 << depth_higher - root = Merkle.root - return [root(hashes[n: n + size], depth_higher) - for n in range(0, len(hashes), size)] - - @staticmethod - def branch_and_root_from_level(level, leaf_hashes, index, - depth_higher): - """Return a (merkle branch, merkle_root) pair when a merkle-tree has a - level cached. - - To maximally reduce the amount of data hashed in computing a - markle branch, cache a tree of depth N at level N // 2. - - level is a list of hashes in the middle of the tree (returned - by level()) - - leaf_hashes are the leaves needed to calculate a partial branch - up to level. - - depth_higher is how much higher level is than the leaves of the tree - - index is the index in the full list of hashes of the hash whose - merkle branch we want. - """ - if not isinstance(level, list): - raise TypeError("level must be a list") - if not isinstance(leaf_hashes, list): - raise TypeError("leaf_hashes must be a list") - leaf_index = (index >> depth_higher) << depth_higher - leaf_branch, leaf_root = Merkle.branch_and_root( - leaf_hashes, index - leaf_index, depth_higher) - index >>= depth_higher - level_branch, root = Merkle.branch_and_root(level, index) - # Check last so that we know index is in-range - if leaf_root != level[index]: - raise ValueError('leaf hashes inconsistent with level') - return leaf_branch + level_branch, root - - -class MerkleCache: - """A cache to calculate merkle branches efficiently.""" - - def __init__(self, merkle, source_func): - """Initialise a cache hashes taken from source_func: - - async def source_func(index, count): - ... - """ - self.merkle = merkle - self.source_func = source_func - self.length = 0 - self.depth_higher = 0 - self.initialized = Event() - - def _segment_length(self): - return 1 << self.depth_higher - - def _leaf_start(self, index): - """Given a level's depth higher and a hash index, return the leaf - index and leaf hash count needed to calculate a merkle branch. - """ - depth_higher = self.depth_higher - return (index >> depth_higher) << depth_higher - - def _level(self, hashes): - return self.merkle.level(hashes, self.depth_higher) - - async def _extend_to(self, length): - """Extend the length of the cache if necessary.""" - if length <= self.length: - return - # Start from the beginning of any final partial segment. - # Retain the value of depth_higher; in practice this is fine - start = self._leaf_start(self.length) - hashes = await self.source_func(start, length - start) - self.level[start >> self.depth_higher:] = self._level(hashes) - self.length = length - - async def _level_for(self, length): - """Return a (level_length, final_hash) pair for a truncation - of the hashes to the given length.""" - if length == self.length: - return self.level - level = self.level[:length >> self.depth_higher] - leaf_start = self._leaf_start(length) - count = min(self._segment_length(), length - leaf_start) - hashes = await self.source_func(leaf_start, count) - level += self._level(hashes) - return level - - async def initialize(self, length): - """Call to initialize the cache to a source of given length.""" - self.length = length - self.depth_higher = self.merkle.tree_depth(length) // 2 - self.level = self._level(await self.source_func(0, length)) - self.initialized.set() - - def truncate(self, length): - """Truncate the cache so it covers no more than length underlying - hashes.""" - if not isinstance(length, int): - raise TypeError('length must be an integer') - if length <= 0: - raise ValueError('length must be positive') - if length >= self.length: - return - length = self._leaf_start(length) - self.length = length - self.level[length >> self.depth_higher:] = [] - - async def branch_and_root(self, length, index): - """Return a merkle branch and root. Length is the number of - hashes used to calculate the merkle root, index is the position - of the hash to calculate the branch of. - - index must be less than length, which must be at least 1.""" - if not isinstance(length, int): - raise TypeError('length must be an integer') - if not isinstance(index, int): - raise TypeError('index must be an integer') - if length <= 0: - raise ValueError('length must be positive') - if index >= length: - raise ValueError('index must be less than length') - await self.initialized.wait() - await self._extend_to(length) - leaf_start = self._leaf_start(index) - count = min(self._segment_length(), length - leaf_start) - leaf_hashes = await self.source_func(leaf_start, count) - if length < self._segment_length(): - return self.merkle.branch_and_root(leaf_hashes, index) - level = await self._level_for(length) - return self.merkle.branch_and_root_from_level( - level, leaf_hashes, index, self.depth_higher) diff --git a/lbry/wallet/server/prefetcher.py b/lbry/wallet/server/prefetcher.py deleted file mode 100644 index 46962c1e4..000000000 --- a/lbry/wallet/server/prefetcher.py +++ /dev/null @@ -1,119 +0,0 @@ -import asyncio -import typing - -from lbry.wallet.server.util import chunks, class_logger -if typing.TYPE_CHECKING: - from lbry.wallet.server.daemon import LBCDaemon - from lbry.wallet.server.coin import Coin - - -class Prefetcher: - """Prefetches blocks (in the forward direction only).""" - - def __init__(self, daemon: 'LBCDaemon', coin: 'Coin', blocks_event: asyncio.Event): - self.logger = class_logger(__name__, self.__class__.__name__) - self.daemon = daemon - self.coin = coin - self.blocks_event = blocks_event - self.blocks = [] - self.caught_up = False - # Access to fetched_height should be protected by the semaphore - self.fetched_height = None - self.semaphore = asyncio.Semaphore() - self.refill_event = asyncio.Event() - # The prefetched block cache size. The min cache size has - # little effect on sync time. - self.cache_size = 0 - self.min_cache_size = 10 * 1024 * 1024 - # This makes the first fetch be 10 blocks - self.ave_size = self.min_cache_size // 10 - self.polling_delay = 0.5 - - async def main_loop(self, bp_height): - """Loop forever polling for more blocks.""" - await self.reset_height(bp_height) - try: - while True: - # Sleep a while if there is nothing to prefetch - await self.refill_event.wait() - if not await self._prefetch_blocks(): - await asyncio.sleep(self.polling_delay) - finally: - self.logger.info("block pre-fetcher is shutting down") - - def get_prefetched_blocks(self): - """Called by block processor when it is processing queued blocks.""" - blocks = self.blocks - self.blocks = [] - self.cache_size = 0 - self.refill_event.set() - return blocks - - async def reset_height(self, height): - """Reset to prefetch blocks from the block processor's height. - - Used in blockchain reorganisations. This coroutine can be - called asynchronously to the _prefetch_blocks coroutine so we - must synchronize with a semaphore. - """ - async with self.semaphore: - self.blocks.clear() - self.cache_size = 0 - self.fetched_height = height - self.refill_event.set() - - daemon_height = await self.daemon.height() - behind = daemon_height - height - if behind > 0: - self.logger.info(f'catching up to daemon height {daemon_height:,d} ' - f'({behind:,d} blocks behind)') - else: - self.logger.info(f'caught up to daemon height {daemon_height:,d}') - - async def _prefetch_blocks(self): - """Prefetch some blocks and put them on the queue. - - Repeats until the queue is full or caught up. - """ - daemon = self.daemon - daemon_height = await daemon.height() - async with self.semaphore: - while self.cache_size < self.min_cache_size: - # Try and catch up all blocks but limit to room in cache. - # Constrain fetch count to between 0 and 500 regardless; - # testnet can be lumpy. - cache_room = self.min_cache_size // self.ave_size - count = min(daemon_height - self.fetched_height, cache_room) - count = min(500, max(count, 0)) - if not count: - self.caught_up = True - return False - - first = self.fetched_height + 1 - hex_hashes = await daemon.block_hex_hashes(first, count) - if self.caught_up: - self.logger.info('new block height {:,d} hash {}' - .format(first + count-1, hex_hashes[-1])) - blocks = await daemon.raw_blocks(hex_hashes) - - assert count == len(blocks) - - # Special handling for genesis block - if first == 0: - blocks[0] = self.coin.genesis_block(blocks[0]) - self.logger.info(f'verified genesis block with hash {hex_hashes[0]}') - - # Update our recent average block size estimate - size = sum(len(block) for block in blocks) - if count >= 10: - self.ave_size = size // count - else: - self.ave_size = (size + (10 - count) * self.ave_size) // 10 - - self.blocks.extend(blocks) - self.cache_size += size - self.fetched_height += count - self.blocks_event.set() - - self.refill_event.clear() - return True diff --git a/lbry/wallet/server/script.py b/lbry/wallet/server/script.py deleted file mode 100644 index ce8d5e5a1..000000000 --- a/lbry/wallet/server/script.py +++ /dev/null @@ -1,289 +0,0 @@ -# Copyright (c) 2016-2017, Neil Booth -# -# All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# and warranty status of this software. - -"""Script-related classes and functions.""" - - -from collections import namedtuple - -from lbry.wallet.server.util import unpack_le_uint16_from, unpack_le_uint32_from, \ - pack_le_uint16, pack_le_uint32 - - -class EnumError(Exception): - pass - - -class Enumeration: - - def __init__(self, name, enumList): - self.__doc__ = name - - lookup = {} - reverseLookup = {} - i = 0 - uniqueNames = set() - uniqueValues = set() - for x in enumList: - if isinstance(x, tuple): - x, i = x - if not isinstance(x, str): - raise EnumError(f"enum name {x} not a string") - if not isinstance(i, int): - raise EnumError(f"enum value {i} not an integer") - if x in uniqueNames: - raise EnumError(f"enum name {x} not unique") - if i in uniqueValues: - raise EnumError(f"enum value {i} not unique") - uniqueNames.add(x) - uniqueValues.add(i) - lookup[x] = i - reverseLookup[i] = x - i = i + 1 - self.lookup = lookup - self.reverseLookup = reverseLookup - - def __getattr__(self, attr): - result = self.lookup.get(attr) - if result is None: - raise AttributeError(f'enumeration has no member {attr}') - return result - - def whatis(self, value): - return self.reverseLookup[value] - - -class ScriptError(Exception): - """Exception used for script errors.""" - - -OpCodes = Enumeration("Opcodes", [ - ("OP_0", 0), ("OP_PUSHDATA1", 76), - "OP_PUSHDATA2", "OP_PUSHDATA4", "OP_1NEGATE", - "OP_RESERVED", - "OP_1", "OP_2", "OP_3", "OP_4", "OP_5", "OP_6", "OP_7", "OP_8", - "OP_9", "OP_10", "OP_11", "OP_12", "OP_13", "OP_14", "OP_15", "OP_16", - "OP_NOP", "OP_VER", "OP_IF", "OP_NOTIF", "OP_VERIF", "OP_VERNOTIF", - "OP_ELSE", "OP_ENDIF", "OP_VERIFY", "OP_RETURN", - "OP_TOALTSTACK", "OP_FROMALTSTACK", "OP_2DROP", "OP_2DUP", "OP_3DUP", - "OP_2OVER", "OP_2ROT", "OP_2SWAP", "OP_IFDUP", "OP_DEPTH", "OP_DROP", - "OP_DUP", "OP_NIP", "OP_OVER", "OP_PICK", "OP_ROLL", "OP_ROT", - "OP_SWAP", "OP_TUCK", - "OP_CAT", "OP_SUBSTR", "OP_LEFT", "OP_RIGHT", "OP_SIZE", - "OP_INVERT", "OP_AND", "OP_OR", "OP_XOR", "OP_EQUAL", "OP_EQUALVERIFY", - "OP_RESERVED1", "OP_RESERVED2", - "OP_1ADD", "OP_1SUB", "OP_2MUL", "OP_2DIV", "OP_NEGATE", "OP_ABS", - "OP_NOT", "OP_0NOTEQUAL", "OP_ADD", "OP_SUB", "OP_MUL", "OP_DIV", "OP_MOD", - "OP_LSHIFT", "OP_RSHIFT", "OP_BOOLAND", "OP_BOOLOR", "OP_NUMEQUAL", - "OP_NUMEQUALVERIFY", "OP_NUMNOTEQUAL", "OP_LESSTHAN", "OP_GREATERTHAN", - "OP_LESSTHANOREQUAL", "OP_GREATERTHANOREQUAL", "OP_MIN", "OP_MAX", - "OP_WITHIN", - "OP_RIPEMD160", "OP_SHA1", "OP_SHA256", "OP_HASH160", "OP_HASH256", - "OP_CODESEPARATOR", "OP_CHECKSIG", "OP_CHECKSIGVERIFY", "OP_CHECKMULTISIG", - "OP_CHECKMULTISIGVERIFY", - "OP_NOP1", - "OP_CHECKLOCKTIMEVERIFY", "OP_CHECKSEQUENCEVERIFY" -]) - - -# Paranoia to make it hard to create bad scripts -assert OpCodes.OP_DUP == 0x76 -assert OpCodes.OP_HASH160 == 0xa9 -assert OpCodes.OP_EQUAL == 0x87 -assert OpCodes.OP_EQUALVERIFY == 0x88 -assert OpCodes.OP_CHECKSIG == 0xac -assert OpCodes.OP_CHECKMULTISIG == 0xae - - -def _match_ops(ops, pattern): - if len(ops) != len(pattern): - return False - for op, pop in zip(ops, pattern): - if pop != op: - # -1 means 'data push', whose op is an (op, data) tuple - if pop == -1 and isinstance(op, tuple): - continue - return False - - return True - - -class ScriptPubKey: - """A class for handling a tx output script that gives conditions - necessary for spending. - """ - - TO_ADDRESS_OPS = [OpCodes.OP_DUP, OpCodes.OP_HASH160, -1, - OpCodes.OP_EQUALVERIFY, OpCodes.OP_CHECKSIG] - TO_P2SH_OPS = [OpCodes.OP_HASH160, -1, OpCodes.OP_EQUAL] - TO_PUBKEY_OPS = [-1, OpCodes.OP_CHECKSIG] - - PayToHandlers = namedtuple('PayToHandlers', 'address script_hash pubkey ' - 'unspendable strange') - - @classmethod - def pay_to(cls, handlers, script): - """Parse a script, invoke the appropriate handler and - return the result. - - One of the following handlers is invoked: - handlers.address(hash160) - handlers.script_hash(hash160) - handlers.pubkey(pubkey) - handlers.unspendable() - handlers.strange(script) - """ - try: - ops = Script.get_ops(script) - except ScriptError: - return handlers.unspendable() - - match = _match_ops - - if match(ops, cls.TO_ADDRESS_OPS): - return handlers.address(ops[2][-1]) - if match(ops, cls.TO_P2SH_OPS): - return handlers.script_hash(ops[1][-1]) - if match(ops, cls.TO_PUBKEY_OPS): - return handlers.pubkey(ops[0][-1]) - if ops and ops[0] == OpCodes.OP_RETURN: - return handlers.unspendable() - return handlers.strange(script) - - @classmethod - def P2SH_script(cls, hash160): - return (bytes([OpCodes.OP_HASH160]) - + Script.push_data(hash160) - + bytes([OpCodes.OP_EQUAL])) - - @classmethod - def P2PKH_script(cls, hash160): - return (bytes([OpCodes.OP_DUP, OpCodes.OP_HASH160]) - + Script.push_data(hash160) - + bytes([OpCodes.OP_EQUALVERIFY, OpCodes.OP_CHECKSIG])) - - @classmethod - def validate_pubkey(cls, pubkey, req_compressed=False): - if isinstance(pubkey, (bytes, bytearray)): - if len(pubkey) == 33 and pubkey[0] in (2, 3): - return # Compressed - if len(pubkey) == 65 and pubkey[0] == 4: - if not req_compressed: - return - raise PubKeyError('uncompressed pubkeys are invalid') - raise PubKeyError(f'invalid pubkey {pubkey}') - - @classmethod - def pubkey_script(cls, pubkey): - cls.validate_pubkey(pubkey) - return Script.push_data(pubkey) + bytes([OpCodes.OP_CHECKSIG]) - - @classmethod - def multisig_script(cls, m, pubkeys): - """Returns the script for a pay-to-multisig transaction.""" - n = len(pubkeys) - if not 1 <= m <= n <= 15: - raise ScriptError(f'{m:d} of {n:d} multisig script not possible') - for pubkey in pubkeys: - cls.validate_pubkey(pubkey, req_compressed=True) - # See https://bitcoin.org/en/developer-guide - # 2 of 3 is: OP_2 pubkey1 pubkey2 pubkey3 OP_3 OP_CHECKMULTISIG - return (bytes([OP_1 + m - 1]) - + b''.join(cls.push_data(pubkey) for pubkey in pubkeys) - + bytes([OP_1 + n - 1, OP_CHECK_MULTISIG])) - - -class Script: - - @classmethod - def get_ops(cls, script): - ops = [] - - # The unpacks or script[n] below throw on truncated scripts - try: - n = 0 - while n < len(script): - op = script[n] - n += 1 - - if op <= OpCodes.OP_PUSHDATA4: - # Raw bytes follow - if op < OpCodes.OP_PUSHDATA1: - dlen = op - elif op == OpCodes.OP_PUSHDATA1: - dlen = script[n] - n += 1 - elif op == OpCodes.OP_PUSHDATA2: - dlen, = unpack_le_uint16_from(script[n: n + 2]) - n += 2 - else: - dlen, = unpack_le_uint32_from(script[n: n + 4]) - n += 4 - if n + dlen > len(script): - raise IndexError - op = (op, script[n:n + dlen]) - n += dlen - - ops.append(op) - except Exception: - # Truncated script; e.g. tx_hash - # ebc9fa1196a59e192352d76c0f6e73167046b9d37b8302b6bb6968dfd279b767 - raise ScriptError('truncated script') - - return ops - - @classmethod - def push_data(cls, data): - """Returns the opcodes to push the data on the stack.""" - assert isinstance(data, (bytes, bytearray)) - - n = len(data) - if n < OpCodes.OP_PUSHDATA1: - return bytes([n]) + data - if n < 256: - return bytes([OpCodes.OP_PUSHDATA1, n]) + data - if n < 65536: - return bytes([OpCodes.OP_PUSHDATA2]) + pack_le_uint16(n) + data - return bytes([OpCodes.OP_PUSHDATA4]) + pack_le_uint32(n) + data - - @classmethod - def opcode_name(cls, opcode): - if OpCodes.OP_0 < opcode < OpCodes.OP_PUSHDATA1: - return f'OP_{opcode:d}' - try: - return OpCodes.whatis(opcode) - except KeyError: - return f'OP_UNKNOWN:{opcode:d}' - - @classmethod - def dump(cls, script): - opcodes, datas = cls.get_ops(script) - for opcode, data in zip(opcodes, datas): - name = cls.opcode_name(opcode) - if data is None: - print(name) - else: - print(f'{name} {data.hex()} ({len(data):d} bytes)') diff --git a/lbry/wallet/server/server.py b/lbry/wallet/server/server.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/lbry/wallet/server/session.py b/lbry/wallet/server/session.py deleted file mode 100644 index 5aeba99cc..000000000 --- a/lbry/wallet/server/session.py +++ /dev/null @@ -1,1525 +0,0 @@ -import os -import ssl -import math -import time -import codecs -import typing -import asyncio -import logging -import itertools -import collections -from bisect import bisect_right -from asyncio import Event, sleep -from collections import defaultdict -from functools import partial - -from elasticsearch import ConnectionTimeout -from prometheus_client import Counter, Info, Histogram, Gauge - -import lbry -from lbry.error import ResolveCensoredError, TooManyClaimSearchParametersError -from lbry.build_info import BUILD, COMMIT_HASH, DOCKER_TAG -from lbry.schema.result import Outputs -from lbry.wallet.server.db.db import HubDB -from lbry.wallet.server.websocket import AdminWebSocket -from lbry.wallet.rpc.framing import NewlineFramer - -import lbry.wallet.server.version as VERSION - -from lbry.wallet.rpc import ( - RPCSession, JSONRPCAutoDetect, JSONRPCConnection, - handler_invocation, RPCError, Request, JSONRPC, Notification, Batch -) -from lbry.wallet.server import util -from lbry.wallet.server.hash import sha256, hash_to_hex_str, hex_str_to_hash, HASHX_LEN, Base58Error -from lbry.wallet.server.daemon import DaemonError -from lbry.wallet.server.db.elasticsearch import SearchIndex - -if typing.TYPE_CHECKING: - from lbry.wallet.server.env import Env - from lbry.wallet.server.daemon import Daemon - from lbry.wallet.server.mempool import MemPool - -BAD_REQUEST = 1 -DAEMON_ERROR = 2 - -log = logging.getLogger(__name__) - - -def scripthash_to_hashX(scripthash: str) -> bytes: - try: - bin_hash = hex_str_to_hash(scripthash) - if len(bin_hash) == 32: - return bin_hash[:HASHX_LEN] - except Exception: - pass - raise RPCError(BAD_REQUEST, f'{scripthash} is not a valid script hash') - - -def non_negative_integer(value) -> int: - """Return param value it is or can be converted to a non-negative - integer, otherwise raise an RPCError.""" - try: - value = int(value) - if value >= 0: - return value - except ValueError: - pass - raise RPCError(BAD_REQUEST, - f'{value} should be a non-negative integer') - - -def assert_boolean(value) -> bool: - """Return param value it is boolean otherwise raise an RPCError.""" - if value in (False, True): - return value - raise RPCError(BAD_REQUEST, f'{value} should be a boolean value') - - -def assert_tx_hash(value: str) -> None: - """Raise an RPCError if the value is not a valid transaction - hash.""" - try: - if len(util.hex_to_bytes(value)) == 32: - return - except Exception: - pass - raise RPCError(BAD_REQUEST, f'{value} should be a transaction hash') - - -class Semaphores: - """For aiorpcX's semaphore handling.""" - - def __init__(self, semaphores): - self.semaphores = semaphores - self.acquired = [] - - async def __aenter__(self): - for semaphore in self.semaphores: - await semaphore.acquire() - self.acquired.append(semaphore) - - async def __aexit__(self, exc_type, exc_value, traceback): - for semaphore in self.acquired: - semaphore.release() - - -class SessionGroup: - - def __init__(self, gid: int): - self.gid = gid - # Concurrency per group - self.semaphore = asyncio.Semaphore(20) - - -NAMESPACE = "wallet_server" -HISTOGRAM_BUCKETS = ( - .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf') -) - - -class SessionManager: - """Holds global state about all sessions.""" - - version_info_metric = Info( - 'build', 'Wallet server build info (e.g. version, commit hash)', namespace=NAMESPACE - ) - version_info_metric.info({ - 'build': BUILD, - "commit": COMMIT_HASH, - "docker_tag": DOCKER_TAG, - 'version': lbry.__version__, - "min_version": util.version_string(VERSION.PROTOCOL_MIN), - "cpu_count": str(os.cpu_count()) - }) - session_count_metric = Gauge("session_count", "Number of connected client sessions", namespace=NAMESPACE, - labelnames=("version",)) - request_count_metric = Counter("requests_count", "Number of requests received", namespace=NAMESPACE, - labelnames=("method", "version")) - tx_request_count_metric = Counter("requested_transaction", "Number of transactions requested", namespace=NAMESPACE) - tx_replied_count_metric = Counter("replied_transaction", "Number of transactions responded", namespace=NAMESPACE) - urls_to_resolve_count_metric = Counter("urls_to_resolve", "Number of urls to resolve", namespace=NAMESPACE) - resolved_url_count_metric = Counter("resolved_url", "Number of resolved urls", namespace=NAMESPACE) - - interrupt_count_metric = Counter("interrupt", "Number of interrupted queries", namespace=NAMESPACE) - db_operational_error_metric = Counter( - "operational_error", "Number of queries that raised operational errors", namespace=NAMESPACE - ) - db_error_metric = Counter( - "internal_error", "Number of queries raising unexpected errors", namespace=NAMESPACE - ) - executor_time_metric = Histogram( - "executor_time", "SQLite executor times", namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS - ) - pending_query_metric = Gauge( - "pending_queries_count", "Number of pending and running sqlite queries", namespace=NAMESPACE - ) - - client_version_metric = Counter( - "clients", "Number of connections received per client version", - namespace=NAMESPACE, labelnames=("version",) - ) - address_history_metric = Histogram( - "address_history", "Time to fetch an address history", - namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS - ) - notifications_in_flight_metric = Gauge( - "notifications_in_flight", "Count of notifications in flight", - namespace=NAMESPACE - ) - notifications_sent_metric = Histogram( - "notifications_sent", "Time to send an address notification", - namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS - ) - - def __init__(self, env: 'Env', db: HubDB, mempool: 'MemPool', history_cache, resolve_cache, resolve_outputs_cache, - daemon: 'Daemon', shutdown_event: asyncio.Event, - on_available_callback: typing.Callable[[], None], on_unavailable_callback: typing.Callable[[], None]): - env.max_send = max(350000, env.max_send) - self.env = env - self.db = db - self.on_available_callback = on_available_callback - self.on_unavailable_callback = on_unavailable_callback - self.daemon = daemon - self.mempool = mempool - self.shutdown_event = shutdown_event - self.logger = util.class_logger(__name__, self.__class__.__name__) - self.servers: typing.Dict[str, asyncio.AbstractServer] = {} - self.sessions: typing.Dict[int, 'LBRYElectrumX'] = {} - self.hashx_subscriptions_by_session: typing.DefaultDict[str, typing.Set[int]] = defaultdict(set) - self.mempool_statuses = {} - self.cur_group = SessionGroup(0) - self.txs_sent = 0 - self.start_time = time.time() - self.history_cache = history_cache - self.resolve_cache = resolve_cache - self.resolve_outputs_cache = resolve_outputs_cache - self.notified_height: typing.Optional[int] = None - # Cache some idea of room to avoid recounting on each subscription - self.subs_room = 0 - - self.session_event = Event() - - # Search index - self.search_index = SearchIndex( - self.env.es_index_prefix, self.env.database_query_timeout, - elastic_host=env.elastic_host, elastic_port=env.elastic_port - ) - - async def _start_server(self, kind, *args, **kw_args): - loop = asyncio.get_event_loop() - - if kind == 'RPC': - protocol_class = LocalRPC - else: - protocol_class = self.env.coin.SESSIONCLS - protocol_factory = partial(protocol_class, self, self.db, - self.mempool, kind) - - host, port = args[:2] - try: - self.servers[kind] = await loop.create_server(protocol_factory, *args, **kw_args) - except OSError as e: # don't suppress CancelledError - self.logger.error(f'{kind} server failed to listen on {host}:' - f'{port:d} :{e!r}') - else: - self.logger.info(f'{kind} server listening on {host}:{port:d}') - - async def _start_external_servers(self): - """Start listening on TCP and SSL ports, but only if the respective - port was given in the environment. - """ - env = self.env - host = env.cs_host(for_rpc=False) - if env.tcp_port is not None: - await self._start_server('TCP', host, env.tcp_port) - if env.ssl_port is not None: - sslc = ssl.SSLContext(ssl.PROTOCOL_TLS) - sslc.load_cert_chain(env.ssl_certfile, keyfile=env.ssl_keyfile) - await self._start_server('SSL', host, env.ssl_port, ssl=sslc) - - async def _close_servers(self, kinds): - """Close the servers of the given kinds (TCP etc.).""" - if kinds: - self.logger.info('closing down {} listening servers' - .format(', '.join(kinds))) - for kind in kinds: - server = self.servers.pop(kind, None) - if server: - server.close() - await server.wait_closed() - - async def _manage_servers(self): - paused = False - max_sessions = self.env.max_sessions - low_watermark = int(max_sessions * 0.95) - while True: - await self.session_event.wait() - self.session_event.clear() - if not paused and len(self.sessions) >= max_sessions: - self.on_unavailable_callback() - self.logger.info(f'maximum sessions {max_sessions:,d} ' - f'reached, stopping new connections until ' - f'count drops to {low_watermark:,d}') - await self._close_servers(['TCP', 'SSL']) - paused = True - # Start listening for incoming connections if paused and - # session count has fallen - if paused and len(self.sessions) <= low_watermark: - self.on_available_callback() - self.logger.info('resuming listening for incoming connections') - await self._start_external_servers() - paused = False - - def _group_map(self): - group_map = defaultdict(list) - for session in self.sessions.values(): - group_map[session.group].append(session) - return group_map - - def _sub_count(self) -> int: - return sum(s.sub_count() for s in self.sessions.values()) - - def _lookup_session(self, session_id): - try: - session_id = int(session_id) - except Exception: - pass - else: - for session in self.sessions.values(): - if session.session_id == session_id: - return session - return None - - async def _for_each_session(self, session_ids, operation): - if not isinstance(session_ids, list): - raise RPCError(BAD_REQUEST, 'expected a list of session IDs') - - result = [] - for session_id in session_ids: - session = self._lookup_session(session_id) - if session: - result.append(await operation(session)) - else: - result.append(f'unknown session: {session_id}') - return result - - async def _clear_stale_sessions(self): - """Cut off sessions that haven't done anything for 10 minutes.""" - session_timeout = self.env.session_timeout - while True: - await sleep(session_timeout // 10) - stale_cutoff = time.perf_counter() - session_timeout - stale_sessions = [session for session in self.sessions.values() - if session.last_recv < stale_cutoff] - if stale_sessions: - text = ', '.join(str(session.session_id) - for session in stale_sessions) - self.logger.info(f'closing stale connections {text}') - # Give the sockets some time to close gracefully - if stale_sessions: - await asyncio.wait([ - session.close(force_after=session_timeout // 10) for session in stale_sessions - ]) - - # Consolidate small groups - group_map = self._group_map() - groups = [group for group, sessions in group_map.items() - if len(sessions) <= 5] # fixme: apply session cost here - if len(groups) > 1: - new_group = groups[-1] - for group in groups: - for session in group_map[group]: - session.group = new_group - - def _get_info(self): - """A summary of server state.""" - group_map = self._group_map() - method_counts = collections.defaultdict(int) - error_count = 0 - logged = 0 - paused = 0 - pending_requests = 0 - closing = 0 - - for s in self.sessions.values(): - error_count += s.errors - if s.log_me: - logged += 1 - if not s._can_send.is_set(): - paused += 1 - pending_requests += s.count_pending_items() - if s.is_closing(): - closing += 1 - for request, _ in s.connection._requests.values(): - method_counts[request.method] += 1 - return { - 'closing': closing, - 'daemon': self.daemon.logged_url(), - 'daemon_height': self.daemon.cached_height(), - 'db_height': self.db.db_height, - 'errors': error_count, - 'groups': len(group_map), - 'logged': logged, - 'paused': paused, - 'pid': os.getpid(), - 'peers': [], - 'requests': pending_requests, - 'method_counts': method_counts, - 'sessions': self.session_count(), - 'subs': self._sub_count(), - 'txs_sent': self.txs_sent, - 'uptime': util.formatted_time(time.time() - self.start_time), - 'version': lbry.__version__, - } - - def _group_data(self): - """Returned to the RPC 'groups' call.""" - result = [] - group_map = self._group_map() - for group, sessions in group_map.items(): - result.append([group.gid, - len(sessions), - sum(s.bw_charge for s in sessions), - sum(s.count_pending_items() for s in sessions), - sum(s.txs_sent for s in sessions), - sum(s.sub_count() for s in sessions), - sum(s.recv_count for s in sessions), - sum(s.recv_size for s in sessions), - sum(s.send_count for s in sessions), - sum(s.send_size for s in sessions), - ]) - return result - - async def _electrum_and_raw_headers(self, height): - raw_header = await self.raw_header(height) - electrum_header = self.env.coin.electrum_header(raw_header, height) - return electrum_header, raw_header - - async def _refresh_hsub_results(self, height): - """Refresh the cached header subscription responses to be for height, - and record that as notified_height. - """ - # Paranoia: a reorg could race and leave db_height lower - height = min(height, self.db.db_height) - electrum, raw = await self._electrum_and_raw_headers(height) - self.hsub_results = (electrum, {'hex': raw.hex(), 'height': height}) - self.notified_height = height - - # --- LocalRPC command handlers - - async def rpc_add_peer(self, real_name): - """Add a peer. - - real_name: "bch.electrumx.cash t50001 s50002" for example - """ - await self._notify_peer(real_name) - return f"peer '{real_name}' added" - - async def rpc_disconnect(self, session_ids): - """Disconnect sessions. - - session_ids: array of session IDs - """ - async def close(session): - """Close the session's transport.""" - await session.close(force_after=2) - return f'disconnected {session.session_id}' - - return await self._for_each_session(session_ids, close) - - async def rpc_log(self, session_ids): - """Toggle logging of sessions. - - session_ids: array of session IDs - """ - async def toggle_logging(session): - """Toggle logging of the session.""" - session.toggle_logging() - return f'log {session.session_id}: {session.log_me}' - - return await self._for_each_session(session_ids, toggle_logging) - - async def rpc_daemon_url(self, daemon_url): - """Replace the daemon URL.""" - daemon_url = daemon_url or self.env.daemon_url - try: - self.daemon.set_url(daemon_url) - except Exception as e: - raise RPCError(BAD_REQUEST, f'an error occurred: {e!r}') - return f'now using daemon at {self.daemon.logged_url()}' - - async def rpc_stop(self): - """Shut down the server cleanly.""" - self.shutdown_event.set() - return 'stopping' - - async def rpc_getinfo(self): - """Return summary information about the server process.""" - return self._get_info() - - async def rpc_groups(self): - """Return statistics about the session groups.""" - return self._group_data() - - async def rpc_peers(self): - """Return a list of data about server peers.""" - return self.env.peer_hubs - - async def rpc_query(self, items, limit): - """Return a list of data about server peers.""" - coin = self.env.coin - db = self.db - lines = [] - - def arg_to_hashX(arg): - try: - script = bytes.fromhex(arg) - lines.append(f'Script: {arg}') - return coin.hashX_from_script(script) - except ValueError: - pass - - try: - hashX = coin.address_to_hashX(arg) - except Base58Error as e: - lines.append(e.args[0]) - return None - lines.append(f'Address: {arg}') - return hashX - - for arg in items: - hashX = arg_to_hashX(arg) - if not hashX: - continue - n = None - history = await db.limited_history(hashX, limit=limit) - for n, (tx_hash, height) in enumerate(history): - lines.append(f'History #{n:,d}: height {height:,d} ' - f'tx_hash {hash_to_hex_str(tx_hash)}') - if n is None: - lines.append('No history found') - n = None - utxos = await db.all_utxos(hashX) - for n, utxo in enumerate(utxos, start=1): - lines.append(f'UTXO #{n:,d}: tx_hash ' - f'{hash_to_hex_str(utxo.tx_hash)} ' - f'tx_pos {utxo.tx_pos:,d} height ' - f'{utxo.height:,d} value {utxo.value:,d}') - if n == limit: - break - if n is None: - lines.append('No UTXOs found') - - balance = sum(utxo.value for utxo in utxos) - lines.append(f'Balance: {coin.decimal_value(balance):,f} ' - f'{coin.SHORTNAME}') - - return lines - - # async def rpc_reorg(self, count): - # """Force a reorg of the given number of blocks. - # - # count: number of blocks to reorg - # """ - # count = non_negative_integer(count) - # if not self.bp.force_chain_reorg(count): - # raise RPCError(BAD_REQUEST, 'still catching up with daemon') - # return f'scheduled a reorg of {count:,d} blocks' - - # --- External Interface - - async def serve(self, mempool, server_listening_event): - """Start the RPC server if enabled. When the event is triggered, - start TCP and SSL servers.""" - try: - if self.env.rpc_port is not None: - await self._start_server('RPC', self.env.cs_host(for_rpc=True), - self.env.rpc_port) - self.logger.info(f'max session count: {self.env.max_sessions:,d}') - self.logger.info(f'session timeout: ' - f'{self.env.session_timeout:,d} seconds') - self.logger.info(f'max response size {self.env.max_send:,d} bytes') - if self.env.drop_client is not None: - self.logger.info(f'drop clients matching: {self.env.drop_client.pattern}') - # Start notifications; initialize hsub_results - await mempool.start(self.db.db_height, self) - await self.start_other() - await self._start_external_servers() - server_listening_event.set() - self.on_available_callback() - # Peer discovery should start after the external servers - # because we connect to ourself - await asyncio.wait([ - self._clear_stale_sessions(), - self._manage_servers() - ]) - except Exception as err: - if not isinstance(err, asyncio.CancelledError): - log.exception("hub server died") - raise err - finally: - await self._close_servers(list(self.servers.keys())) - log.info("disconnect %i sessions", len(self.sessions)) - if self.sessions: - await asyncio.wait([ - session.close(force_after=1) for session in self.sessions.values() - ]) - await self.stop_other() - - async def start_other(self): - pass - - async def stop_other(self): - pass - - def session_count(self) -> int: - """The number of connections that we've sent something to.""" - return len(self.sessions) - - async def daemon_request(self, method, *args): - """Catch a DaemonError and convert it to an RPCError.""" - try: - return await getattr(self.daemon, method)(*args) - except DaemonError as e: - raise RPCError(DAEMON_ERROR, f'daemon error: {e!r}') from None - - async def raw_header(self, height): - """Return the binary header at the given height.""" - try: - return await self.db.raw_header(height) - except IndexError: - raise RPCError(BAD_REQUEST, f'height {height:,d} ' - 'out of range') from None - - async def electrum_header(self, height): - """Return the deserialized header at the given height.""" - electrum_header, _ = await self._electrum_and_raw_headers(height) - return electrum_header - - async def broadcast_transaction(self, raw_tx): - hex_hash = await self.daemon.broadcast_transaction(raw_tx) - self.txs_sent += 1 - return hex_hash - - async def limited_history(self, hashX): - """A caching layer.""" - if hashX not in self.history_cache: - # History DoS limit. Each element of history is about 99 - # bytes when encoded as JSON. This limits resource usage - # on bloated history requests, and uses a smaller divisor - # so large requests are logged before refusing them. - limit = self.env.max_send // 97 - self.history_cache[hashX] = await self.db.limited_history(hashX, limit=limit) - return self.history_cache[hashX] - - def _notify_peer(self, peer): - notify_tasks = [ - session.send_notification('blockchain.peers.subscribe', [peer]) - for session in self.sessions.values() if session.subscribe_peers - ] - if notify_tasks: - self.logger.info(f'notify {len(notify_tasks)} sessions of new peers') - asyncio.create_task(asyncio.wait(notify_tasks)) - - def add_session(self, session): - self.sessions[id(session)] = session - self.session_event.set() - gid = int(session.start_time - self.start_time) // 900 - if self.cur_group.gid != gid: - self.cur_group = SessionGroup(gid) - return self.cur_group - - def remove_session(self, session): - """Remove a session from our sessions list if there.""" - session_id = id(session) - for hashX in session.hashX_subs: - sessions = self.hashx_subscriptions_by_session[hashX] - sessions.remove(session_id) - if not sessions: - self.hashx_subscriptions_by_session.pop(hashX) - self.sessions.pop(session_id) - self.session_event.set() - - -class SessionBase(RPCSession): - """Base class of ElectrumX JSON sessions. - - Each session runs its tasks in asynchronous parallelism with other - sessions. - """ - - MAX_CHUNK_SIZE = 40960 - session_counter = itertools.count() - request_handlers: typing.Dict[str, typing.Callable] = {} - version = '0.5.7' - - def __init__(self, session_manager: 'LBRYSessionManager', db: 'HubDB', mempool: 'MemPool', kind: str): - connection = JSONRPCConnection(JSONRPCAutoDetect) - self.env = session_manager.env - super().__init__(connection=connection) - self.logger = util.class_logger(__name__, self.__class__.__name__) - self.session_manager = session_manager - self.db = db - self.mempool = mempool - self.kind = kind # 'RPC', 'TCP' etc. - self.coin = self.env.coin - self.anon_logs = self.env.anon_logs - self.txs_sent = 0 - self.log_me = False - self.daemon_request = self.session_manager.daemon_request - # Hijack the connection so we can log messages - self._receive_message_orig = self.connection.receive_message - self.connection.receive_message = self.receive_message - - - def default_framer(self): - return NewlineFramer(self.env.max_receive) - - def peer_address_str(self, *, for_log=True): - """Returns the peer's IP address and port as a human-readable - string, respecting anon logs if the output is for a log.""" - if for_log and self.anon_logs: - return 'xx.xx.xx.xx:xx' - return super().peer_address_str() - - def receive_message(self, message): - if self.log_me: - self.logger.info(f'processing {message}') - return self._receive_message_orig(message) - - def toggle_logging(self): - self.log_me = not self.log_me - - def connection_made(self, transport): - """Handle an incoming client connection.""" - super().connection_made(transport) - self.session_id = next(self.session_counter) - context = {'conn_id': f'{self.session_id}'} - self.logger = util.ConnectionLogger(self.logger, context) - self.group = self.session_manager.add_session(self) - self.session_manager.session_count_metric.labels(version=self.client_version).inc() - peer_addr_str = self.peer_address_str() - self.logger.info(f'{self.kind} {peer_addr_str}, ' - f'{self.session_manager.session_count():,d} total') - - def connection_lost(self, exc): - """Handle client disconnection.""" - super().connection_lost(exc) - self.session_manager.remove_session(self) - self.session_manager.session_count_metric.labels(version=self.client_version).dec() - msg = '' - if not self._can_send.is_set(): - msg += ' whilst paused' - if self.send_size >= 1024*1024: - msg += ('. Sent {:,d} bytes in {:,d} messages' - .format(self.send_size, self.send_count)) - if msg: - msg = 'disconnected' + msg - self.logger.info(msg) - - def count_pending_items(self): - return len(self.connection.pending_requests()) - - def semaphore(self): - return Semaphores([self.group.semaphore]) - - def sub_count(self): - return 0 - - async def handle_request(self, request): - """Handle an incoming request. ElectrumX doesn't receive - notifications from client sessions. - """ - self.session_manager.request_count_metric.labels(method=request.method, version=self.client_version).inc() - if isinstance(request, Request): - handler = self.request_handlers.get(request.method) - handler = partial(handler, self) - else: - handler = None - coro = handler_invocation(handler, request)() - return await coro - - -class LBRYSessionManager(SessionManager): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.websocket = None - # self.metrics = ServerLoadData() - # self.metrics_loop = None - self.running = False - if self.env.websocket_host is not None and self.env.websocket_port is not None: - self.websocket = AdminWebSocket(self) - - # async def process_metrics(self): - # while self.running: - # data = self.metrics.to_json_and_reset({ - # 'sessions': self.session_count(), - # 'height': self.db.db_height, - # }) - # if self.websocket is not None: - # self.websocket.send_message(data) - # await asyncio.sleep(1) - - async def start_other(self): - self.running = True - if self.websocket is not None: - await self.websocket.start() - - async def stop_other(self): - self.running = False - if self.websocket is not None: - await self.websocket.stop() - - -class LBRYElectrumX(SessionBase): - """A TCP server that handles incoming Electrum connections.""" - - PROTOCOL_MIN = VERSION.PROTOCOL_MIN - PROTOCOL_MAX = VERSION.PROTOCOL_MAX - max_errors = math.inf # don't disconnect people for errors! let them happen... - session_manager: LBRYSessionManager - version = lbry.__version__ - cached_server_features = {} - - @classmethod - def initialize_request_handlers(cls): - cls.request_handlers.update({ - 'blockchain.block.get_chunk': cls.block_get_chunk, - 'blockchain.block.get_header': cls.block_get_header, - 'blockchain.estimatefee': cls.estimatefee, - 'blockchain.relayfee': cls.relayfee, - # 'blockchain.scripthash.get_balance': cls.scripthash_get_balance, - 'blockchain.scripthash.get_history': cls.scripthash_get_history, - 'blockchain.scripthash.get_mempool': cls.scripthash_get_mempool, - # 'blockchain.scripthash.listunspent': cls.scripthash_listunspent, - 'blockchain.scripthash.subscribe': cls.scripthash_subscribe, - 'blockchain.transaction.broadcast': cls.transaction_broadcast, - 'blockchain.transaction.get': cls.transaction_get, - 'blockchain.transaction.get_batch': cls.transaction_get_batch, - 'blockchain.transaction.info': cls.transaction_info, - 'blockchain.transaction.get_merkle': cls.transaction_merkle, - # 'server.add_peer': cls.add_peer, - 'server.banner': cls.banner, - 'server.payment_address': cls.payment_address, - 'server.donation_address': cls.donation_address, - 'server.features': cls.server_features_async, - 'server.peers.subscribe': cls.peers_subscribe, - 'server.version': cls.server_version, - 'blockchain.transaction.get_height': cls.transaction_get_height, - 'blockchain.claimtrie.search': cls.claimtrie_search, - 'blockchain.claimtrie.resolve': cls.claimtrie_resolve, - 'blockchain.claimtrie.getclaimbyid': cls.claimtrie_getclaimbyid, - # 'blockchain.claimtrie.getclaimsbyids': cls.claimtrie_getclaimsbyids, - 'blockchain.block.get_server_height': cls.get_server_height, - 'mempool.get_fee_histogram': cls.mempool_compact_histogram, - 'blockchain.block.headers': cls.block_headers, - 'server.ping': cls.ping, - 'blockchain.headers.subscribe': cls.headers_subscribe_False, - # 'blockchain.address.get_balance': cls.address_get_balance, - 'blockchain.address.get_history': cls.address_get_history, - 'blockchain.address.get_mempool': cls.address_get_mempool, - # 'blockchain.address.listunspent': cls.address_listunspent, - 'blockchain.address.subscribe': cls.address_subscribe, - 'blockchain.address.unsubscribe': cls.address_unsubscribe, - }) - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - if not LBRYElectrumX.request_handlers: - LBRYElectrumX.initialize_request_handlers() - if not LBRYElectrumX.cached_server_features: - LBRYElectrumX.set_server_features(self.env) - self.subscribe_headers = False - self.subscribe_headers_raw = False - self.subscribe_peers = False - self.connection.max_response_size = self.env.max_send - self.hashX_subs = {} - self.sv_seen = False - self.protocol_tuple = self.PROTOCOL_MIN - self.protocol_string = None - self.daemon = self.session_manager.daemon - self.db: HubDB = self.session_manager.db - - @classmethod - def protocol_min_max_strings(cls): - return [util.version_string(ver) - for ver in (cls.PROTOCOL_MIN, cls.PROTOCOL_MAX)] - - @classmethod - def set_server_features(cls, env): - """Return the server features dictionary.""" - min_str, max_str = cls.protocol_min_max_strings() - cls.cached_server_features.update({ - 'hosts': env.hosts_dict(), - 'pruning': None, - 'server_version': cls.version, - 'protocol_min': min_str, - 'protocol_max': max_str, - 'genesis_hash': env.coin.GENESIS_HASH, - 'description': env.description, - 'payment_address': env.payment_address, - 'donation_address': env.donation_address, - 'daily_fee': env.daily_fee, - 'hash_function': 'sha256', - 'trending_algorithm': 'fast_ar' - }) - - async def server_features_async(self): - return self.cached_server_features - - @classmethod - def server_version_args(cls): - """The arguments to a server.version RPC call to a peer.""" - return [cls.version, cls.protocol_min_max_strings()] - - def protocol_version_string(self): - return util.version_string(self.protocol_tuple) - - def sub_count(self): - return len(self.hashX_subs) - - async def get_hashX_status(self, hashX: bytes): - mempool_history = self.mempool.transaction_summaries(hashX) - history = ''.join(f'{hash_to_hex_str(tx_hash)}:' - f'{height:d}:' - for tx_hash, height in await self.session_manager.limited_history(hashX)) - history += ''.join(f'{hash_to_hex_str(tx.hash)}:' - f'{-tx.has_unconfirmed_inputs:d}:' - for tx in mempool_history) - if history: - status = sha256(history.encode()).hex() - else: - status = None - return history, status, len(mempool_history) > 0 - - async def send_history_notifications(self, *hashXes: typing.Iterable[bytes]): - notifications = [] - for hashX in hashXes: - alias = self.hashX_subs[hashX] - if len(alias) == 64: - method = 'blockchain.scripthash.subscribe' - else: - method = 'blockchain.address.subscribe' - start = time.perf_counter() - history, status, mempool_status = await self.get_hashX_status(hashX) - if mempool_status: - self.session_manager.mempool_statuses[hashX] = status - else: - self.session_manager.mempool_statuses.pop(hashX, None) - - self.session_manager.address_history_metric.observe(time.perf_counter() - start) - notifications.append((method, (alias, status))) - - start = time.perf_counter() - self.session_manager.notifications_in_flight_metric.inc() - for method, args in notifications: - self.NOTIFICATION_COUNT.labels(method=method, version=self.client_version).inc() - try: - await self.send_notifications( - Batch([Notification(method, (alias, status)) for (method, (alias, status)) in notifications]) - ) - self.session_manager.notifications_sent_metric.observe(time.perf_counter() - start) - finally: - self.session_manager.notifications_in_flight_metric.dec() - - # def get_metrics_or_placeholder_for_api(self, query_name): - # """ Do not hold on to a reference to the metrics - # returned by this method past an `await` or - # you may be working with a stale metrics object. - # """ - # if self.env.track_metrics: - # # return self.session_manager.metrics.for_api(query_name) - # else: - # return APICallMetrics(query_name) - - - # async def run_and_cache_query(self, query_name, kwargs): - # start = time.perf_counter() - # if isinstance(kwargs, dict): - # kwargs['release_time'] = format_release_time(kwargs.get('release_time')) - # try: - # self.session_manager.pending_query_metric.inc() - # return await self.db.search_index.session_query(query_name, kwargs) - # except ConnectionTimeout: - # self.session_manager.interrupt_count_metric.inc() - # raise RPCError(JSONRPC.QUERY_TIMEOUT, 'query timed out') - # finally: - # self.session_manager.pending_query_metric.dec() - # self.session_manager.executor_time_metric.observe(time.perf_counter() - start) - - async def mempool_compact_histogram(self): - return [] #self.mempool.compact_fee_histogram() - - async def claimtrie_search(self, **kwargs): - start = time.perf_counter() - if 'release_time' in kwargs: - release_time = kwargs.pop('release_time') - release_times = release_time if isinstance(release_time, list) else [release_time] - try: - kwargs['release_time'] = [format_release_time(release_time) for release_time in release_times] - except ValueError: - pass - try: - self.session_manager.pending_query_metric.inc() - if 'channel' in kwargs: - channel_url = kwargs.pop('channel') - _, channel_claim, _, _ = await self.db.resolve(channel_url) - if not channel_claim or isinstance(channel_claim, (ResolveCensoredError, LookupError, ValueError)): - return Outputs.to_base64([], [], 0, None, None) - kwargs['channel_id'] = channel_claim.claim_hash.hex() - return await self.session_manager.search_index.cached_search(kwargs) - except ConnectionTimeout: - self.session_manager.interrupt_count_metric.inc() - raise RPCError(JSONRPC.QUERY_TIMEOUT, 'query timed out') - except TooManyClaimSearchParametersError as err: - await asyncio.sleep(2) - self.logger.warning("Got an invalid query from %s, for %s with more than %d elements.", - self.peer_address()[0], err.key, err.limit) - return RPCError(1, str(err)) - finally: - self.session_manager.pending_query_metric.dec() - self.session_manager.executor_time_metric.observe(time.perf_counter() - start) - - async def _cached_resolve_url(self, url): - if url not in self.session_manager.resolve_cache: - self.session_manager.resolve_cache[url] = await self.loop.run_in_executor(self.db._executor, self.db._resolve, url) - return self.session_manager.resolve_cache[url] - - async def claimtrie_resolve(self, *urls) -> str: - sorted_urls = tuple(sorted(urls)) - self.session_manager.urls_to_resolve_count_metric.inc(len(sorted_urls)) - try: - if sorted_urls in self.session_manager.resolve_outputs_cache: - return self.session_manager.resolve_outputs_cache[sorted_urls] - rows, extra = [], [] - for url in urls: - if url not in self.session_manager.resolve_cache: - self.session_manager.resolve_cache[url] = await self._cached_resolve_url(url) - stream, channel, repost, reposted_channel = self.session_manager.resolve_cache[url] - if isinstance(channel, ResolveCensoredError): - rows.append(channel) - extra.append(channel.censor_row) - elif isinstance(stream, ResolveCensoredError): - rows.append(stream) - extra.append(stream.censor_row) - elif channel and not stream: - rows.append(channel) - # print("resolved channel", channel.name.decode()) - if repost: - extra.append(repost) - if reposted_channel: - extra.append(reposted_channel) - elif stream: - # print("resolved stream", stream.name.decode()) - rows.append(stream) - if channel: - # print("and channel", channel.name.decode()) - extra.append(channel) - if repost: - extra.append(repost) - if reposted_channel: - extra.append(reposted_channel) - await asyncio.sleep(0) - self.session_manager.resolve_outputs_cache[sorted_urls] = result = await self.loop.run_in_executor( - None, Outputs.to_base64, rows, extra, 0, None, None - ) - return result - finally: - self.session_manager.resolved_url_count_metric.inc(len(sorted_urls)) - - async def get_server_height(self): - return self.db.db_height - - async def transaction_get_height(self, tx_hash): - self.assert_tx_hash(tx_hash) - - def get_height(): - v = self.db.prefix_db.tx_num.get(tx_hash) - if v: - return bisect_right(self.db.tx_counts, v.tx_num) - return self.mempool.get_mempool_height(tx_hash) - - return await asyncio.get_event_loop().run_in_executor(self.db._executor, get_height) - - async def claimtrie_getclaimbyid(self, claim_id): - rows = [] - extra = [] - stream = await self.db.fs_getclaimbyid(claim_id) - if not stream: - stream = LookupError(f"Could not find claim at {claim_id}") - rows.append(stream) - return Outputs.to_base64(rows, extra, 0, None, None) - - def assert_tx_hash(self, value): - '''Raise an RPCError if the value is not a valid transaction - hash.''' - try: - if len(util.hex_to_bytes(value)) == 32: - return - except Exception: - pass - raise RPCError(1, f'{value} should be a transaction hash') - - async def subscribe_headers_result(self): - """The result of a header subscription or notification.""" - return self.session_manager.hsub_results[self.subscribe_headers_raw] - - async def _headers_subscribe(self, raw): - """Subscribe to get headers of new blocks.""" - self.subscribe_headers_raw = assert_boolean(raw) - self.subscribe_headers = True - return await self.subscribe_headers_result() - - async def headers_subscribe(self): - """Subscribe to get raw headers of new blocks.""" - return await self._headers_subscribe(True) - - async def headers_subscribe_True(self, raw=True): - """Subscribe to get headers of new blocks.""" - return await self._headers_subscribe(raw) - - async def headers_subscribe_False(self, raw=False): - """Subscribe to get headers of new blocks.""" - return await self._headers_subscribe(raw) - - async def add_peer(self, features): - """Add a peer (but only if the peer resolves to the source).""" - return await self.peer_mgr.on_add_peer(features, self.peer_address()) - - async def peers_subscribe(self): - """Return the server peers as a list of (ip, host, details) tuples.""" - self.subscribe_peers = True - return self.env.peer_hubs - - async def address_status(self, hashX): - """Returns an address status. - - Status is a hex string, but must be None if there is no history. - """ - # Note history is ordered and mempool unordered in electrum-server - # For mempool, height is -1 if it has unconfirmed inputs, otherwise 0 - _, status, has_mempool_history = await self.get_hashX_status(hashX) - if has_mempool_history: - self.session_manager.mempool_statuses[hashX] = status - else: - self.session_manager.mempool_statuses.pop(hashX, None) - return status - - # async def hashX_listunspent(self, hashX): - # """Return the list of UTXOs of a script hash, including mempool - # effects.""" - # utxos = await self.db.all_utxos(hashX) - # utxos = sorted(utxos) - # utxos.extend(await self.mempool.unordered_UTXOs(hashX)) - # spends = await self.mempool.potential_spends(hashX) - # - # return [{'tx_hash': hash_to_hex_str(utxo.tx_hash), - # 'tx_pos': utxo.tx_pos, - # 'height': utxo.height, 'value': utxo.value} - # for utxo in utxos - # if (utxo.tx_hash, utxo.tx_pos) not in spends] - - async def hashX_subscribe(self, hashX, alias): - self.hashX_subs[hashX] = alias - self.session_manager.hashx_subscriptions_by_session[hashX].add(id(self)) - return await self.address_status(hashX) - - async def hashX_unsubscribe(self, hashX, alias): - sessions = self.session_manager.hashx_subscriptions_by_session[hashX] - sessions.remove(id(self)) - if not sessions: - self.hashX_subs.pop(hashX, None) - - def address_to_hashX(self, address): - try: - return self.coin.address_to_hashX(address) - except Exception: - pass - raise RPCError(BAD_REQUEST, f'{address} is not a valid address') - - # async def address_get_balance(self, address): - # """Return the confirmed and unconfirmed balance of an address.""" - # hashX = self.address_to_hashX(address) - # return await self.get_balance(hashX) - - async def address_get_history(self, address): - """Return the confirmed and unconfirmed history of an address.""" - hashX = self.address_to_hashX(address) - return await self.confirmed_and_unconfirmed_history(hashX) - - async def address_get_mempool(self, address): - """Return the mempool transactions touching an address.""" - hashX = self.address_to_hashX(address) - return self.unconfirmed_history(hashX) - - # async def address_listunspent(self, address): - # """Return the list of UTXOs of an address.""" - # hashX = self.address_to_hashX(address) - # return await self.hashX_listunspent(hashX) - - async def address_subscribe(self, *addresses): - """Subscribe to an address. - - address: the address to subscribe to""" - if len(addresses) > 1000: - raise RPCError(BAD_REQUEST, f'too many addresses in subscription request: {len(addresses)}') - results = [] - for address in addresses: - results.append(await self.hashX_subscribe(self.address_to_hashX(address), address)) - await asyncio.sleep(0) - return results - - async def address_unsubscribe(self, address): - """Unsubscribe an address. - - address: the address to unsubscribe""" - hashX = self.address_to_hashX(address) - return await self.hashX_unsubscribe(hashX, address) - - # async def get_balance(self, hashX): - # utxos = await self.db.all_utxos(hashX) - # confirmed = sum(utxo.value for utxo in utxos) - # unconfirmed = await self.mempool.balance_delta(hashX) - # return {'confirmed': confirmed, 'unconfirmed': unconfirmed} - - # async def scripthash_get_balance(self, scripthash): - # """Return the confirmed and unconfirmed balance of a scripthash.""" - # hashX = scripthash_to_hashX(scripthash) - # return await self.get_balance(hashX) - - def unconfirmed_history(self, hashX): - # Note unconfirmed history is unordered in electrum-server - # height is -1 if it has unconfirmed inputs, otherwise 0 - return [{'tx_hash': hash_to_hex_str(tx.hash), - 'height': -tx.has_unconfirmed_inputs, - 'fee': tx.fee} - for tx in self.mempool.transaction_summaries(hashX)] - - async def confirmed_and_unconfirmed_history(self, hashX): - # Note history is ordered but unconfirmed is unordered in e-s - history = await self.session_manager.limited_history(hashX) - conf = [{'tx_hash': hash_to_hex_str(tx_hash), 'height': height} - for tx_hash, height in history] - return conf + self.unconfirmed_history(hashX) - - async def scripthash_get_history(self, scripthash): - """Return the confirmed and unconfirmed history of a scripthash.""" - hashX = scripthash_to_hashX(scripthash) - return await self.confirmed_and_unconfirmed_history(hashX) - - async def scripthash_get_mempool(self, scripthash): - """Return the mempool transactions touching a scripthash.""" - hashX = scripthash_to_hashX(scripthash) - return self.unconfirmed_history(hashX) - - # async def scripthash_listunspent(self, scripthash): - # """Return the list of UTXOs of a scripthash.""" - # hashX = scripthash_to_hashX(scripthash) - # return await self.hashX_listunspent(hashX) - - async def scripthash_subscribe(self, scripthash): - """Subscribe to a script hash. - - scripthash: the SHA256 hash of the script to subscribe to""" - hashX = scripthash_to_hashX(scripthash) - return await self.hashX_subscribe(hashX, scripthash) - - async def _merkle_proof(self, cp_height, height): - max_height = self.db.db_height - if not height <= cp_height <= max_height: - raise RPCError(BAD_REQUEST, - f'require header height {height:,d} <= ' - f'cp_height {cp_height:,d} <= ' - f'chain height {max_height:,d}') - branch, root = await self.db.header_branch_and_root(cp_height + 1, height) - return { - 'branch': [hash_to_hex_str(elt) for elt in branch], - 'root': hash_to_hex_str(root), - } - - async def block_headers(self, start_height, count, cp_height=0, b64=False): - """Return count concatenated block headers as hex for the main chain; - starting at start_height. - - start_height and count must be non-negative integers. At most - MAX_CHUNK_SIZE headers will be returned. - """ - start_height = non_negative_integer(start_height) - count = non_negative_integer(count) - cp_height = non_negative_integer(cp_height) - - max_size = self.MAX_CHUNK_SIZE - count = min(count, max_size) - headers, count = await self.db.read_headers(start_height, count) - - if b64: - headers = self.db.encode_headers(start_height, count, headers) - else: - headers = headers.hex() - result = { - 'base64' if b64 else 'hex': headers, - 'count': count, - 'max': max_size - } - if count and cp_height: - last_height = start_height + count - 1 - result.update(await self._merkle_proof(cp_height, last_height)) - return result - - async def block_get_chunk(self, index): - """Return a chunk of block headers as a hexadecimal string. - - index: the chunk index""" - index = non_negative_integer(index) - size = self.coin.CHUNK_SIZE - start_height = index * size - headers, _ = await self.db.read_headers(start_height, size) - return headers.hex() - - async def block_get_header(self, height): - """The deserialized header at a given height. - - height: the header's height""" - height = non_negative_integer(height) - return await self.session_manager.electrum_header(height) - - def is_tor(self): - """Try to detect if the connection is to a tor hidden service we are - running.""" - peername = self.peer_mgr.proxy_peername() - if not peername: - return False - peer_address = self.peer_address() - return peer_address and peer_address[0] == peername[0] - - async def replaced_banner(self, banner): - network_info = await self.daemon_request('getnetworkinfo') - ni_version = network_info['version'] - major, minor = divmod(ni_version, 1000000) - minor, revision = divmod(minor, 10000) - revision //= 100 - daemon_version = f'{major:d}.{minor:d}.{revision:d}' - for pair in [ - ('$SERVER_VERSION', self.version), - ('$DAEMON_VERSION', daemon_version), - ('$DAEMON_SUBVERSION', network_info['subversion']), - ('$PAYMENT_ADDRESS', self.env.payment_address), - ('$DONATION_ADDRESS', self.env.donation_address), - ]: - banner = banner.replace(*pair) - return banner - - async def payment_address(self): - """Return the payment address as a string, empty if there is none.""" - return self.env.payment_address - - async def donation_address(self): - """Return the donation address as a string, empty if there is none.""" - return self.env.donation_address - - async def banner(self): - """Return the server banner text.""" - banner = f'You are connected to an {self.version} server.' - banner_file = self.env.banner_file - if banner_file: - try: - with codecs.open(banner_file, 'r', 'utf-8') as f: - banner = f.read() - except Exception as e: - self.logger.error(f'reading banner file {banner_file}: {e!r}') - else: - banner = await self.replaced_banner(banner) - - return banner - - async def relayfee(self): - """The minimum fee a low-priority tx must pay in order to be accepted - to the daemon's memory pool.""" - return await self.daemon_request('relayfee') - - async def estimatefee(self, number): - """The estimated transaction fee per kilobyte to be paid for a - transaction to be included within a certain number of blocks. - - number: the number of blocks - """ - number = non_negative_integer(number) - return await self.daemon_request('estimatefee', number) - - async def ping(self): - """Serves as a connection keep-alive mechanism and for the client to - confirm the server is still responding. - """ - return None - - async def server_version(self, client_name='', protocol_version=None): - """Returns the server version as a string. - - client_name: a string identifying the client - protocol_version: the protocol version spoken by the client - """ - if self.protocol_string is not None: - return self.version, self.protocol_string - if self.sv_seen and self.protocol_tuple >= (1, 4): - raise RPCError(BAD_REQUEST, f'server.version already sent') - self.sv_seen = True - - if client_name: - client_name = str(client_name) - if self.env.drop_client is not None and \ - self.env.drop_client.match(client_name): - self.close_after_send = True - raise RPCError(BAD_REQUEST, f'unsupported client: {client_name}') - if self.client_version != client_name[:17]: - self.session_manager.session_count_metric.labels(version=self.client_version).dec() - self.client_version = client_name[:17] - self.session_manager.session_count_metric.labels(version=self.client_version).inc() - self.session_manager.client_version_metric.labels(version=self.client_version).inc() - - # Find the highest common protocol version. Disconnect if - # that protocol version in unsupported. - ptuple, client_min = util.protocol_version(protocol_version, self.PROTOCOL_MIN, self.PROTOCOL_MAX) - if ptuple is None: - ptuple, client_min = util.protocol_version(protocol_version, (1, 1, 0), (1, 4, 0)) - if ptuple is None: - self.close_after_send = True - raise RPCError(BAD_REQUEST, f'unsupported protocol version: {protocol_version}') - - self.protocol_tuple = ptuple - self.protocol_string = util.version_string(ptuple) - return self.version, self.protocol_string - - async def transaction_broadcast(self, raw_tx): - """Broadcast a raw transaction to the network. - - raw_tx: the raw transaction as a hexadecimal string""" - # This returns errors as JSON RPC errors, as is natural - try: - hex_hash = await self.session_manager.broadcast_transaction(raw_tx) - self.txs_sent += 1 - # self.mempool.wakeup.set() - # await asyncio.sleep(0.5) - self.logger.info(f'sent tx: {hex_hash}') - return hex_hash - except DaemonError as e: - error, = e.args - message = error['message'] - self.logger.info(f'error sending transaction: {message}') - raise RPCError(BAD_REQUEST, 'the transaction was rejected by ' - f'network rules.\n\n{message}\n[{raw_tx}]') - - async def transaction_info(self, tx_hash: str): - return (await self.transaction_get_batch(tx_hash))[tx_hash] - - async def transaction_get_batch(self, *tx_hashes): - self.session_manager.tx_request_count_metric.inc(len(tx_hashes)) - if len(tx_hashes) > 100: - raise RPCError(BAD_REQUEST, f'too many tx hashes in request: {len(tx_hashes)}') - for tx_hash in tx_hashes: - assert_tx_hash(tx_hash) - batch_result = await self.db.get_transactions_and_merkles(tx_hashes) - needed_merkles = {} - - for tx_hash in tx_hashes: - if tx_hash in batch_result and batch_result[tx_hash][0]: - continue - tx_hash_bytes = bytes.fromhex(tx_hash)[::-1] - mempool_tx = self.mempool.txs.get(tx_hash_bytes, None) - if mempool_tx: - raw_tx, block_hash = mempool_tx.raw_tx.hex(), None - else: - tx_info = await self.daemon_request('getrawtransaction', tx_hash, 1) - raw_tx = tx_info['hex'] - block_hash = tx_info.get('blockhash') - if block_hash: - block = await self.daemon.deserialised_block(block_hash) - height = block['height'] - try: - pos = block['tx'].index(tx_hash) - except ValueError: - raise RPCError(BAD_REQUEST, f'tx hash {tx_hash} not in ' - f'block {block_hash} at height {height:,d}') - needed_merkles[tx_hash] = raw_tx, block['tx'], pos, height - else: - batch_result[tx_hash] = [raw_tx, {'block_height': -1}] - - if needed_merkles: - for tx_hash, (raw_tx, block_txs, pos, block_height) in needed_merkles.items(): - batch_result[tx_hash] = raw_tx, { - 'merkle': self._get_merkle_branch(block_txs, pos), - 'pos': pos, - 'block_height': block_height - } - await asyncio.sleep(0) # heavy call, give other tasks a chance - self.session_manager.tx_replied_count_metric.inc(len(tx_hashes)) - return batch_result - - async def transaction_get(self, tx_hash, verbose=False): - """Return the serialized raw transaction given its hash - - tx_hash: the transaction hash as a hexadecimal string - verbose: passed on to the daemon - """ - assert_tx_hash(tx_hash) - if verbose not in (True, False): - raise RPCError(BAD_REQUEST, f'"verbose" must be a boolean') - - return await self.daemon_request('getrawtransaction', tx_hash, int(verbose)) - - def _get_merkle_branch(self, tx_hashes, tx_pos): - """Return a merkle branch to a transaction. - - tx_hashes: ordered list of hex strings of tx hashes in a block - tx_pos: index of transaction in tx_hashes to create branch for - """ - hashes = [hex_str_to_hash(hash) for hash in tx_hashes] - branch, root = self.db.merkle.branch_and_root(hashes, tx_pos) - branch = [hash_to_hex_str(hash) for hash in branch] - return branch - - async def transaction_merkle(self, tx_hash, height): - """Return the markle branch to a confirmed transaction given its hash - and height. - - tx_hash: the transaction hash as a hexadecimal string - height: the height of the block it is in - """ - assert_tx_hash(tx_hash) - result = await self.transaction_get_batch(tx_hash) - if tx_hash not in result or result[tx_hash][1]['block_height'] <= 0: - raise RPCError(BAD_REQUEST, f'tx hash {tx_hash} not in ' - f'block at height {height:,d}') - return result[tx_hash][1] - - -class LocalRPC(SessionBase): - """A local TCP RPC server session.""" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.client = 'RPC' - self.connection._max_response_size = 0 - - def protocol_version_string(self): - return 'RPC' - - -def get_from_possible_keys(dictionary, *keys): - for key in keys: - if key in dictionary: - return dictionary[key] - - -def format_release_time(release_time): - # round release time to 1000 so it caches better - # also set a default so we dont show claims in the future - def roundup_time(number, factor=360): - return int(1 + int(number / factor)) * factor - if isinstance(release_time, str) and len(release_time) > 0: - time_digits = ''.join(filter(str.isdigit, release_time)) - time_prefix = release_time[:-len(time_digits)] - return time_prefix + str(roundup_time(int(time_digits))) - elif isinstance(release_time, int): - return roundup_time(release_time) diff --git a/lbry/wallet/server/tx.py b/lbry/wallet/server/tx.py deleted file mode 100644 index fefe2fd6f..000000000 --- a/lbry/wallet/server/tx.py +++ /dev/null @@ -1,626 +0,0 @@ -# Copyright (c) 2016-2017, Neil Booth -# Copyright (c) 2017, the ElectrumX authors -# -# All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# and warranty status of this software. - -"""Transaction-related classes and functions.""" -import typing -from collections import namedtuple - -from lbry.wallet.server.hash import sha256, double_sha256, hash_to_hex_str -from lbry.wallet.server.script import OpCodes -from lbry.wallet.server.util import ( - unpack_le_int32_from, unpack_le_int64_from, unpack_le_uint16_from, - unpack_le_uint32_from, unpack_le_uint64_from, pack_le_int32, pack_varint, - pack_le_uint32, pack_le_int64, pack_varbytes, -) - -ZERO = bytes(32) -MINUS_1 = 4294967295 - - -class Tx(typing.NamedTuple): - version: int - inputs: typing.List['TxInput'] - outputs: typing.List['TxOutput'] - locktime: int - raw: bytes - - -class TxInput(typing.NamedTuple): - prev_hash: bytes - prev_idx: int - script: bytes - sequence: int - - """Class representing a transaction input.""" - def __str__(self): - script = self.script.hex() - prev_hash = hash_to_hex_str(self.prev_hash) - return (f"Input({prev_hash}, {self.prev_idx:d}, script={script}, sequence={self.sequence:d})") - - def is_generation(self): - """Test if an input is generation/coinbase like""" - return self.prev_idx == MINUS_1 and self.prev_hash == ZERO - - def serialize(self): - return b''.join(( - self.prev_hash, - pack_le_uint32(self.prev_idx), - pack_varbytes(self.script), - pack_le_uint32(self.sequence), - )) - - -class TxOutput(typing.NamedTuple): - value: int - pk_script: bytes - - def serialize(self): - return b''.join(( - pack_le_int64(self.value), - pack_varbytes(self.pk_script), - )) - - -class Deserializer: - """Deserializes blocks into transactions. - - External entry points are read_tx(), read_tx_and_hash(), - read_tx_and_vsize() and read_block(). - - This code is performance sensitive as it is executed 100s of - millions of times during sync. - """ - - TX_HASH_FN = staticmethod(double_sha256) - - def __init__(self, binary, start=0): - assert isinstance(binary, bytes), f"type {type(binary)} is not 'bytes'" - self.binary = binary - self.binary_length = len(binary) - self.cursor = start - self.flags = 0 - - def read_tx(self): - """Return a deserialized transaction.""" - start = self.cursor - return Tx( - self._read_le_int32(), # version - self._read_inputs(), # inputs - self._read_outputs(), # outputs - self._read_le_uint32(), # locktime - self.binary[start:self.cursor], - ) - - def read_tx_and_hash(self): - """Return a (deserialized TX, tx_hash) pair. - - The hash needs to be reversed for human display; for efficiency - we process it in the natural serialized order. - """ - start = self.cursor - return self.read_tx(), self.TX_HASH_FN(self.binary[start:self.cursor]) - - def read_tx_and_vsize(self): - """Return a (deserialized TX, vsize) pair.""" - return self.read_tx(), self.binary_length - - def read_tx_block(self): - """Returns a list of (deserialized_tx, tx_hash) pairs.""" - read = self.read_tx_and_hash - # Some coins have excess data beyond the end of the transactions - return [read() for _ in range(self._read_varint())] - - def _read_inputs(self): - read_input = self._read_input - return [read_input() for i in range(self._read_varint())] - - def _read_input(self): - return TxInput( - self._read_nbytes(32), # prev_hash - self._read_le_uint32(), # prev_idx - self._read_varbytes(), # script - self._read_le_uint32() # sequence - ) - - def _read_outputs(self): - read_output = self._read_output - return [read_output() for i in range(self._read_varint())] - - def _read_output(self): - return TxOutput( - self._read_le_int64(), # value - self._read_varbytes(), # pk_script - ) - - def _read_byte(self): - cursor = self.cursor - self.cursor += 1 - return self.binary[cursor] - - def _read_nbytes(self, n): - cursor = self.cursor - self.cursor = end = cursor + n - assert self.binary_length >= end - return self.binary[cursor:end] - - def _read_varbytes(self): - return self._read_nbytes(self._read_varint()) - - def _read_varint(self): - n = self.binary[self.cursor] - self.cursor += 1 - if n < 253: - return n - if n == 253: - return self._read_le_uint16() - if n == 254: - return self._read_le_uint32() - return self._read_le_uint64() - - def _read_le_int32(self): - result, = unpack_le_int32_from(self.binary, self.cursor) - self.cursor += 4 - return result - - def _read_le_int64(self): - result, = unpack_le_int64_from(self.binary, self.cursor) - self.cursor += 8 - return result - - def _read_le_uint16(self): - result, = unpack_le_uint16_from(self.binary, self.cursor) - self.cursor += 2 - return result - - def _read_le_uint32(self): - result, = unpack_le_uint32_from(self.binary, self.cursor) - self.cursor += 4 - return result - - def _read_le_uint64(self): - result, = unpack_le_uint64_from(self.binary, self.cursor) - self.cursor += 8 - return result - - -class TxSegWit(namedtuple("Tx", "version marker flag inputs outputs " - "witness locktime raw")): - """Class representing a SegWit transaction.""" - - -class DeserializerSegWit(Deserializer): - - # https://bitcoincore.org/en/segwit_wallet_dev/#transaction-serialization - - def _read_witness(self, fields): - read_witness_field = self._read_witness_field - return [read_witness_field() for i in range(fields)] - - def _read_witness_field(self): - read_varbytes = self._read_varbytes - return [read_varbytes() for i in range(self._read_varint())] - - def _read_tx_parts(self): - """Return a (deserialized TX, tx_hash, vsize) tuple.""" - start = self.cursor - marker = self.binary[self.cursor + 4] - if marker: - tx = super().read_tx() - tx_hash = self.TX_HASH_FN(self.binary[start:self.cursor]) - return tx, tx_hash, self.binary_length - - # Ugh, this is nasty. - version = self._read_le_int32() - orig_ser = self.binary[start:self.cursor] - - marker = self._read_byte() - flag = self._read_byte() - - start = self.cursor - inputs = self._read_inputs() - outputs = self._read_outputs() - orig_ser += self.binary[start:self.cursor] - - base_size = self.cursor - start - witness = self._read_witness(len(inputs)) - - start = self.cursor - locktime = self._read_le_uint32() - orig_ser += self.binary[start:self.cursor] - vsize = (3 * base_size + self.binary_length) // 4 - - return TxSegWit(version, marker, flag, inputs, outputs, witness, - locktime, orig_ser), self.TX_HASH_FN(orig_ser), vsize - - def read_tx(self): - return self._read_tx_parts()[0] - - def read_tx_and_hash(self): - tx, tx_hash, vsize = self._read_tx_parts() - return tx, tx_hash - - def read_tx_and_vsize(self): - tx, tx_hash, vsize = self._read_tx_parts() - return tx, vsize - - -class DeserializerAuxPow(Deserializer): - VERSION_AUXPOW = (1 << 8) - - def read_header(self, height, static_header_size): - """Return the AuxPow block header bytes""" - start = self.cursor - version = self._read_le_uint32() - if version & self.VERSION_AUXPOW: - # We are going to calculate the block size then read it as bytes - self.cursor = start - self.cursor += static_header_size # Block normal header - self.read_tx() # AuxPow transaction - self.cursor += 32 # Parent block hash - merkle_size = self._read_varint() - self.cursor += 32 * merkle_size # Merkle branch - self.cursor += 4 # Index - merkle_size = self._read_varint() - self.cursor += 32 * merkle_size # Chain merkle branch - self.cursor += 4 # Chain index - self.cursor += 80 # Parent block header - header_end = self.cursor - else: - header_end = static_header_size - self.cursor = start - return self._read_nbytes(header_end) - - -class DeserializerAuxPowSegWit(DeserializerSegWit, DeserializerAuxPow): - pass - - -class DeserializerEquihash(Deserializer): - def read_header(self, height, static_header_size): - """Return the block header bytes""" - start = self.cursor - # We are going to calculate the block size then read it as bytes - self.cursor += static_header_size - solution_size = self._read_varint() - self.cursor += solution_size - header_end = self.cursor - self.cursor = start - return self._read_nbytes(header_end) - - -class DeserializerEquihashSegWit(DeserializerSegWit, DeserializerEquihash): - pass - - -class TxJoinSplit(namedtuple("Tx", "version inputs outputs locktime")): - """Class representing a JoinSplit transaction.""" - - -class DeserializerZcash(DeserializerEquihash): - def read_tx(self): - header = self._read_le_uint32() - overwintered = ((header >> 31) == 1) - if overwintered: - version = header & 0x7fffffff - self.cursor += 4 # versionGroupId - else: - version = header - - is_overwinter_v3 = version == 3 - is_sapling_v4 = version == 4 - - base_tx = TxJoinSplit( - version, - self._read_inputs(), # inputs - self._read_outputs(), # outputs - self._read_le_uint32() # locktime - ) - - if is_overwinter_v3 or is_sapling_v4: - self.cursor += 4 # expiryHeight - - has_shielded = False - if is_sapling_v4: - self.cursor += 8 # valueBalance - shielded_spend_size = self._read_varint() - self.cursor += shielded_spend_size * 384 # vShieldedSpend - shielded_output_size = self._read_varint() - self.cursor += shielded_output_size * 948 # vShieldedOutput - has_shielded = shielded_spend_size > 0 or shielded_output_size > 0 - - if base_tx.version >= 2: - joinsplit_size = self._read_varint() - if joinsplit_size > 0: - joinsplit_desc_len = 1506 + (192 if is_sapling_v4 else 296) - # JSDescription - self.cursor += joinsplit_size * joinsplit_desc_len - self.cursor += 32 # joinSplitPubKey - self.cursor += 64 # joinSplitSig - - if is_sapling_v4 and has_shielded: - self.cursor += 64 # bindingSig - - return base_tx - - -class TxTime(namedtuple("Tx", "version time inputs outputs locktime")): - """Class representing transaction that has a time field.""" - - -class DeserializerTxTime(Deserializer): - def read_tx(self): - return TxTime( - self._read_le_int32(), # version - self._read_le_uint32(), # time - self._read_inputs(), # inputs - self._read_outputs(), # outputs - self._read_le_uint32(), # locktime - ) - - -class DeserializerReddcoin(Deserializer): - def read_tx(self): - version = self._read_le_int32() - inputs = self._read_inputs() - outputs = self._read_outputs() - locktime = self._read_le_uint32() - if version > 1: - time = self._read_le_uint32() - else: - time = 0 - - return TxTime(version, time, inputs, outputs, locktime) - - -class DeserializerTxTimeAuxPow(DeserializerTxTime): - VERSION_AUXPOW = (1 << 8) - - def is_merged_block(self): - start = self.cursor - self.cursor = 0 - version = self._read_le_uint32() - self.cursor = start - if version & self.VERSION_AUXPOW: - return True - return False - - def read_header(self, height, static_header_size): - """Return the AuxPow block header bytes""" - start = self.cursor - version = self._read_le_uint32() - if version & self.VERSION_AUXPOW: - # We are going to calculate the block size then read it as bytes - self.cursor = start - self.cursor += static_header_size # Block normal header - self.read_tx() # AuxPow transaction - self.cursor += 32 # Parent block hash - merkle_size = self._read_varint() - self.cursor += 32 * merkle_size # Merkle branch - self.cursor += 4 # Index - merkle_size = self._read_varint() - self.cursor += 32 * merkle_size # Chain merkle branch - self.cursor += 4 # Chain index - self.cursor += 80 # Parent block header - header_end = self.cursor - else: - header_end = static_header_size - self.cursor = start - return self._read_nbytes(header_end) - - -class DeserializerBitcoinAtom(DeserializerSegWit): - FORK_BLOCK_HEIGHT = 505888 - - def read_header(self, height, static_header_size): - """Return the block header bytes""" - header_len = static_header_size - if height >= self.FORK_BLOCK_HEIGHT: - header_len += 4 # flags - return self._read_nbytes(header_len) - - -class DeserializerGroestlcoin(DeserializerSegWit): - TX_HASH_FN = staticmethod(sha256) - - -class TxInputTokenPay(TxInput): - """Class representing a TokenPay transaction input.""" - - OP_ANON_MARKER = 0xb9 - # 2byte marker (cpubkey + sigc + sigr) - MIN_ANON_IN_SIZE = 2 + (33 + 32 + 32) - - def _is_anon_input(self): - return (len(self.script) >= self.MIN_ANON_IN_SIZE and - self.script[0] == OpCodes.OP_RETURN and - self.script[1] == self.OP_ANON_MARKER) - - def is_generation(self): - # Transactions coming in from stealth addresses are seen by - # the blockchain as newly minted coins. The reverse, where coins - # are sent TO a stealth address, are seen by the blockchain as - # a coin burn. - if self._is_anon_input(): - return True - return super().is_generation() - - -class TxInputTokenPayStealth( - namedtuple("TxInput", "keyimage ringsize script sequence")): - """Class representing a TokenPay stealth transaction input.""" - - def __str__(self): - script = self.script.hex() - keyimage = bytes(self.keyimage).hex() - return (f"Input({keyimage}, {self.ringsize[1]:d}, script={script}, sequence={self.sequence:d})") - - def is_generation(self): - return True - - def serialize(self): - return b''.join(( - self.keyimage, - self.ringsize, - pack_varbytes(self.script), - pack_le_uint32(self.sequence), - )) - - -class DeserializerTokenPay(DeserializerTxTime): - - def _read_input(self): - txin = TxInputTokenPay( - self._read_nbytes(32), # prev_hash - self._read_le_uint32(), # prev_idx - self._read_varbytes(), # script - self._read_le_uint32(), # sequence - ) - if txin._is_anon_input(): - # Not sure if this is actually needed, and seems - # extra work for no immediate benefit, but it at - # least correctly represents a stealth input - raw = txin.serialize() - deserializer = Deserializer(raw) - txin = TxInputTokenPayStealth( - deserializer._read_nbytes(33), # keyimage - deserializer._read_nbytes(3), # ringsize - deserializer._read_varbytes(), # script - deserializer._read_le_uint32() # sequence - ) - return txin - - -# Decred -class TxInputDcr(namedtuple("TxInput", "prev_hash prev_idx tree sequence")): - """Class representing a Decred transaction input.""" - - def __str__(self): - prev_hash = hash_to_hex_str(self.prev_hash) - return (f"Input({prev_hash}, {self.prev_idx:d}, tree={self.tree}, sequence={self.sequence:d})") - - def is_generation(self): - """Test if an input is generation/coinbase like""" - return self.prev_idx == MINUS_1 and self.prev_hash == ZERO - - -class TxOutputDcr(namedtuple("TxOutput", "value version pk_script")): - """Class representing a Decred transaction output.""" - pass - - -class TxDcr(namedtuple("Tx", "version inputs outputs locktime expiry " - "witness")): - """Class representing a Decred transaction.""" - - -class DeserializerDecred(Deserializer): - @staticmethod - def blake256(data): - from blake256.blake256 import blake_hash - return blake_hash(data) - - @staticmethod - def blake256d(data): - from blake256.blake256 import blake_hash - return blake_hash(blake_hash(data)) - - def read_tx(self): - return self._read_tx_parts(produce_hash=False)[0] - - def read_tx_and_hash(self): - tx, tx_hash, vsize = self._read_tx_parts() - return tx, tx_hash - - def read_tx_and_vsize(self): - tx, tx_hash, vsize = self._read_tx_parts(produce_hash=False) - return tx, vsize - - def read_tx_block(self): - """Returns a list of (deserialized_tx, tx_hash) pairs.""" - read = self.read_tx_and_hash - txs = [read() for _ in range(self._read_varint())] - stxs = [read() for _ in range(self._read_varint())] - return txs + stxs - - def read_tx_tree(self): - """Returns a list of deserialized_tx without tx hashes.""" - read_tx = self.read_tx - return [read_tx() for _ in range(self._read_varint())] - - def _read_input(self): - return TxInputDcr( - self._read_nbytes(32), # prev_hash - self._read_le_uint32(), # prev_idx - self._read_byte(), # tree - self._read_le_uint32(), # sequence - ) - - def _read_output(self): - return TxOutputDcr( - self._read_le_int64(), # value - self._read_le_uint16(), # version - self._read_varbytes(), # pk_script - ) - - def _read_witness(self, fields): - read_witness_field = self._read_witness_field - assert fields == self._read_varint() - return [read_witness_field() for _ in range(fields)] - - def _read_witness_field(self): - value_in = self._read_le_int64() - block_height = self._read_le_uint32() - block_index = self._read_le_uint32() - script = self._read_varbytes() - return value_in, block_height, block_index, script - - def _read_tx_parts(self, produce_hash=True): - start = self.cursor - version = self._read_le_int32() - inputs = self._read_inputs() - outputs = self._read_outputs() - locktime = self._read_le_uint32() - expiry = self._read_le_uint32() - end_prefix = self.cursor - witness = self._read_witness(len(inputs)) - - if produce_hash: - # TxSerializeNoWitness << 16 == 0x10000 - no_witness_header = pack_le_uint32(0x10000 | (version & 0xffff)) - prefix_tx = no_witness_header + self.binary[start+4:end_prefix] - tx_hash = self.blake256(prefix_tx) - else: - tx_hash = None - - return TxDcr( - version, - inputs, - outputs, - locktime, - expiry, - witness - ), tx_hash, self.cursor - start diff --git a/lbry/wallet/server/util.py b/lbry/wallet/server/util.py deleted file mode 100644 index d78b23bb5..000000000 --- a/lbry/wallet/server/util.py +++ /dev/null @@ -1,361 +0,0 @@ -# Copyright (c) 2016-2017, Neil Booth -# -# All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# and warranty status of this software. - -"""Miscellaneous utility classes and functions.""" - - -import array -import inspect -from ipaddress import ip_address -import logging -import re -import sys -from collections import Container, Mapping -from struct import pack, Struct - -# Logging utilities - - -class ConnectionLogger(logging.LoggerAdapter): - """Prepends a connection identifier to a logging message.""" - def process(self, msg, kwargs): - conn_id = self.extra.get('conn_id', 'unknown') - return f'[{conn_id}] {msg}', kwargs - - -class CompactFormatter(logging.Formatter): - """Strips the module from the logger name to leave the class only.""" - def format(self, record): - record.name = record.name.rpartition('.')[-1] - return super().format(record) - - -def make_logger(name, *, handler, level): - """Return the root ElectrumX logger.""" - logger = logging.getLogger(name) - logger.addHandler(handler) - logger.setLevel(logging.INFO) - logger.propagate = False - return logger - - -def class_logger(path, classname): - """Return a hierarchical logger for a class.""" - return logging.getLogger(path).getChild(classname) - - -# Method decorator. To be used for calculations that will always -# deliver the same result. The method cannot take any arguments -# and should be accessed as an attribute. -class cachedproperty: - - def __init__(self, f): - self.f = f - - def __get__(self, obj, type): - obj = obj or type - value = self.f(obj) - setattr(obj, self.f.__name__, value) - return value - - -def formatted_time(t, sep=' '): - """Return a number of seconds as a string in days, hours, mins and - maybe secs.""" - t = int(t) - fmts = (('{:d}d', 86400), ('{:02d}h', 3600), ('{:02d}m', 60)) - parts = [] - for fmt, n in fmts: - val = t // n - if parts or val: - parts.append(fmt.format(val)) - t %= n - if len(parts) < 3: - parts.append(f'{t:02d}s') - return sep.join(parts) - - -def deep_getsizeof(obj): - """Find the memory footprint of a Python object. - - Based on code from code.tutsplus.com: http://goo.gl/fZ0DXK - - This is a recursive function that drills down a Python object graph - like a dictionary holding nested dictionaries with lists of lists - and tuples and sets. - - The sys.getsizeof function does a shallow size of only. It counts each - object inside a container as pointer only regardless of how big it - really is. - """ - - ids = set() - - def size(o): - if id(o) in ids: - return 0 - - r = sys.getsizeof(o) - ids.add(id(o)) - - if isinstance(o, (str, bytes, bytearray, array.array)): - return r - - if isinstance(o, Mapping): - return r + sum(size(k) + size(v) for k, v in o.items()) - - if isinstance(o, Container): - return r + sum(size(x) for x in o) - - return r - - return size(obj) - - -def subclasses(base_class, strict=True): - """Return a list of subclasses of base_class in its module.""" - def select(obj): - return (inspect.isclass(obj) and issubclass(obj, base_class) and - (not strict or obj != base_class)) - - pairs = inspect.getmembers(sys.modules[base_class.__module__], select) - return [pair[1] for pair in pairs] - - -def chunks(items, size): - """Break up items, an iterable, into chunks of length size.""" - for i in range(0, len(items), size): - yield items[i: i + size] - - -def resolve_limit(limit): - if limit is None: - return -1 - assert isinstance(limit, int) and limit >= 0 - return limit - - -def bytes_to_int(be_bytes): - """Interprets a big-endian sequence of bytes as an integer""" - return int.from_bytes(be_bytes, 'big') - - -def int_to_bytes(value): - """Converts an integer to a big-endian sequence of bytes""" - return value.to_bytes((value.bit_length() + 7) // 8, 'big') - - -def increment_byte_string(bs): - """Return the lexicographically next byte string of the same length. - - Return None if there is none (when the input is all 0xff bytes).""" - for n in range(1, len(bs) + 1): - if bs[-n] != 0xff: - return bs[:-n] + bytes([bs[-n] + 1]) + bytes(n - 1) - return None - - -class LogicalFile: - """A logical binary file split across several separate files on disk.""" - - def __init__(self, prefix, digits, file_size): - digit_fmt = f'{{:0{digits:d}d}}' - self.filename_fmt = prefix + digit_fmt - self.file_size = file_size - - def read(self, start, size=-1): - """Read up to size bytes from the virtual file, starting at offset - start, and return them. - - If size is -1 all bytes are read.""" - parts = [] - while size != 0: - try: - with self.open_file(start, False) as f: - part = f.read(size) - if not part: - break - except FileNotFoundError: - break - parts.append(part) - start += len(part) - if size > 0: - size -= len(part) - return b''.join(parts) - - def write(self, start, b): - """Write the bytes-like object, b, to the underlying virtual file.""" - while b: - size = min(len(b), self.file_size - (start % self.file_size)) - with self.open_file(start, True) as f: - f.write(b if size == len(b) else b[:size]) - b = b[size:] - start += size - - def open_file(self, start, create): - """Open the virtual file and seek to start. Return a file handle. - Raise FileNotFoundError if the file does not exist and create - is False. - """ - file_num, offset = divmod(start, self.file_size) - filename = self.filename_fmt.format(file_num) - f = open_file(filename, create) - f.seek(offset) - return f - - -def open_file(filename, create=False): - """Open the file name. Return its handle.""" - try: - return open(filename, 'rb+') - except FileNotFoundError: - if create: - return open(filename, 'wb+') - raise - - -def open_truncate(filename): - """Open the file name. Return its handle.""" - return open(filename, 'wb+') - - -def address_string(address): - """Return an address as a correctly formatted string.""" - fmt = '{}:{:d}' - host, port = address - try: - host = ip_address(host) - except ValueError: - pass - else: - if host.version == 6: - fmt = '[{}]:{:d}' - return fmt.format(host, port) - -# See http://stackoverflow.com/questions/2532053/validate-a-hostname-string -# Note underscores are valid in domain names, but strictly invalid in host -# names. We ignore that distinction. - - -SEGMENT_REGEX = re.compile("(?!-)[A-Z_\\d-]{1,63}(? 255: - return False - # strip exactly one dot from the right, if present - if hostname and hostname[-1] == ".": - hostname = hostname[:-1] - return all(SEGMENT_REGEX.match(x) for x in hostname.split(".")) - - -def protocol_tuple(s): - """Converts a protocol version number, such as "1.0" to a tuple (1, 0). - - If the version number is bad, (0, ) indicating version 0 is returned.""" - try: - return tuple(int(part) for part in s.split('.')) - except Exception: - return (0, ) - - -def version_string(ptuple): - """Convert a version tuple such as (1, 2) to "1.2". - There is always at least one dot, so (1, ) becomes "1.0".""" - while len(ptuple) < 2: - ptuple += (0, ) - return '.'.join(str(p) for p in ptuple) - - -def protocol_version(client_req, min_tuple, max_tuple): - """Given a client's protocol version string, return a pair of - protocol tuples: - - (negotiated version, client min request) - - If the request is unsupported, the negotiated protocol tuple is - None. - """ - if client_req is None: - client_min = client_max = min_tuple - else: - if isinstance(client_req, list) and len(client_req) == 2: - client_min, client_max = client_req - else: - client_min = client_max = client_req - client_min = protocol_tuple(client_min) - client_max = protocol_tuple(client_max) - - result = min(client_max, max_tuple) - if result < max(client_min, min_tuple) or result == (0, ): - result = None - - return result, client_min - - -struct_le_i = Struct('H') -struct_be_I = Struct('>I') -structB = Struct('B') - -unpack_le_int32_from = struct_le_i.unpack_from -unpack_le_int64_from = struct_le_q.unpack_from -unpack_le_uint16_from = struct_le_H.unpack_from -unpack_le_uint32_from = struct_le_I.unpack_from -unpack_le_uint64_from = struct_le_Q.unpack_from -unpack_be_uint16_from = struct_be_H.unpack_from -unpack_be_uint32_from = struct_be_I.unpack_from - -unpack_be_uint64 = lambda x: int.from_bytes(x, byteorder='big') - -pack_le_int32 = struct_le_i.pack -pack_le_int64 = struct_le_q.pack -pack_le_uint16 = struct_le_H.pack -pack_le_uint32 = struct_le_I.pack -pack_be_uint64 = lambda x: x.to_bytes(8, byteorder='big') -pack_be_uint16 = lambda x: x.to_bytes(2, byteorder='big') -pack_be_uint32 = struct_be_I.pack -pack_byte = structB.pack - -hex_to_bytes = bytes.fromhex - - -def pack_varint(n): - if n < 253: - return pack_byte(n) - if n < 65536: - return pack_byte(253) + pack_le_uint16(n) - if n < 4294967296: - return pack_byte(254) + pack_le_uint32(n) - return pack_byte(255) + pack_le_uint64(n) - - -def pack_varbytes(data): - return pack_varint(len(data)) + data diff --git a/lbry/wallet/server/version.py b/lbry/wallet/server/version.py deleted file mode 100644 index d6044c3a0..000000000 --- a/lbry/wallet/server/version.py +++ /dev/null @@ -1,3 +0,0 @@ -# need this to avoid circular import -PROTOCOL_MIN = (0, 54, 0) -PROTOCOL_MAX = (0, 199, 0) diff --git a/lbry/wallet/server/websocket.py b/lbry/wallet/server/websocket.py deleted file mode 100644 index 9620918cb..000000000 --- a/lbry/wallet/server/websocket.py +++ /dev/null @@ -1,55 +0,0 @@ -import asyncio -from weakref import WeakSet - -from aiohttp.web import Application, AppRunner, WebSocketResponse, TCPSite -from aiohttp.http_websocket import WSMsgType, WSCloseCode - - -class AdminWebSocket: - - def __init__(self, manager): - self.manager = manager - self.app = Application() - self.app['websockets'] = WeakSet() - self.app.router.add_get('/', self.on_connect) - self.app.on_shutdown.append(self.on_shutdown) - self.runner = AppRunner(self.app) - - async def on_status(self, _): - if not self.app['websockets']: - return - self.send_message({ - 'type': 'status', - 'height': self.manager.daemon.cached_height(), - }) - - def send_message(self, msg): - for web_socket in self.app['websockets']: - asyncio.create_task(web_socket.send_json(msg)) - - async def start(self): - await self.runner.setup() - await TCPSite(self.runner, self.manager.env.websocket_host, self.manager.env.websocket_port).start() - - async def stop(self): - await self.runner.cleanup() - - async def on_connect(self, request): - web_socket = WebSocketResponse() - await web_socket.prepare(request) - self.app['websockets'].add(web_socket) - try: - async for msg in web_socket: - if msg.type == WSMsgType.TEXT: - await self.on_status(None) - elif msg.type == WSMsgType.ERROR: - print('web socket connection closed with exception %s' % - web_socket.exception()) - finally: - self.app['websockets'].discard(web_socket) - return web_socket - - @staticmethod - async def on_shutdown(app): - for web_socket in set(app['websockets']): - await web_socket.close(code=WSCloseCode.GOING_AWAY, message='Server shutdown') diff --git a/lbry/wallet/transaction.py b/lbry/wallet/transaction.py index 120482c55..7b30b8203 100644 --- a/lbry/wallet/transaction.py +++ b/lbry/wallet/transaction.py @@ -7,18 +7,18 @@ from typing import List, Iterable, Optional, Tuple from lbry.error import InsufficientFundsError from lbry.crypto.hash import hash160, sha256 from lbry.crypto.base58 import Base58 -from lbry.schema.url import normalize_name -from lbry.schema.claim import Claim -from lbry.schema.base import Signable -from lbry.schema.purchase import Purchase -from lbry.schema.support import Support +from scribe.schema.url import normalize_name +from scribe.schema.claim import Claim +from scribe.schema.base import Signable +from scribe.schema.purchase import Purchase +from scribe.schema.support import Support +from scribe.bip32 import PrivateKey, PublicKey from .script import InputScript, OutputScript from .constants import COIN, DUST, NULL_HASH32 from .bcd_data_stream import BCDataStream from .hash import TXRef, TXRefImmutable from .util import ReadOnlyList -from .bip32 import PrivateKey, PublicKey if typing.TYPE_CHECKING: from lbry.wallet.account import Account diff --git a/lbry/wallet/udp.py b/lbry/wallet/udp.py index 6396d0b3e..4a6f44f0e 100644 --- a/lbry/wallet/udp.py +++ b/lbry/wallet/udp.py @@ -4,7 +4,7 @@ from time import perf_counter import logging from typing import Optional, Tuple, NamedTuple from lbry.utils import LRUCache, is_valid_public_ipv4 -from lbry.schema.attrs import country_str_to_int, country_int_to_str +from scribe.schema.attrs import country_str_to_int, country_int_to_str # from prometheus_client import Counter diff --git a/setup.py b/setup.py index ecb6937d5..c0f077d7b 100644 --- a/setup.py +++ b/setup.py @@ -30,9 +30,6 @@ setup( entry_points={ 'console_scripts': [ 'lbrynet=lbry.extras.cli:main', - 'lbry-hub-writer=lbry.wallet.server.cli:run_writer_forever', - 'lbry-hub-server=lbry.wallet.server.cli:run_server_forever', - 'lbry-hub-elastic-sync=lbry.wallet.server.cli:run_es_sync_forever', 'orchstr8=lbry.wallet.orchstr8.cli:main' ], }, diff --git a/test_rocksdb.py b/test_rocksdb.py deleted file mode 100755 index 39470457e..000000000 --- a/test_rocksdb.py +++ /dev/null @@ -1,47 +0,0 @@ -#! python - -import os -import shutil -import rocksdb -import tempfile -import logging - -log = logging.getLogger() -log.addHandler(logging.StreamHandler()) -log.setLevel(logging.INFO) - -def _main(db_loc): - opts = rocksdb.Options(create_if_missing=True) - db = rocksdb.DB(os.path.join(db_loc, "test"), opts) - secondary_location = os.path.join(db_loc, "secondary") - secondary = rocksdb.DB( - os.path.join(db_loc, "test"), - rocksdb.Options(create_if_missing=True, max_open_files=-1), - secondary_name=secondary_location - ) - try: - assert secondary.get(b"a") is None - db.put(b"a", b"b") - assert db.get(b"a") == b"b" - assert secondary.get(b"a") is None - - secondary.try_catch_up_with_primary() - assert secondary.get(b"a") == b"b" - finally: - secondary.close() - db.close() - - -def main(): - db_dir = tempfile.mkdtemp() - try: - _main(db_dir) - log.info("rocksdb %s (%s) works!", rocksdb.__version__, rocksdb.ROCKSDB_VERSION) - except: - log.exception("boom") - finally: - shutil.rmtree(db_dir) - - -if __name__ == "__main__": - main() diff --git a/tests/integration/blockchain/test_network.py b/tests/integration/blockchain/test_network.py index 26dd4c3dc..0b9fcac68 100644 --- a/tests/integration/blockchain/test_network.py +++ b/tests/integration/blockchain/test_network.py @@ -1,15 +1,16 @@ import asyncio +import scribe -import lbry from unittest.mock import Mock -from lbry_schema.coin import LBCRegTest +from scribe.blockchain.network import LBCRegTest +from scribe.hub.udp import StatusServer +from scribe.hub.session import LBRYElectrumX from lbry.wallet.network import Network from lbry.wallet.orchstr8 import Conductor from lbry.wallet.orchstr8.node import SPVNode from lbry.wallet.rpc import RPCSession -from lbry.wallet.udp import StatusServer from lbry.testcase import IntegrationTestCase, AsyncioTestCase from lbry.conf import Config @@ -34,7 +35,7 @@ class NetworkTests(IntegrationTestCase): 'payment_address': '', 'donation_address': '', 'daily_fee': '0', - 'server_version': lbry.__version__, + 'server_version': scribe.__version__, 'trending_algorithm': 'fast_ar', }, await self.ledger.network.get_server_features()) # await self.conductor.spv_node.stop() @@ -50,7 +51,6 @@ class NetworkTests(IntegrationTestCase): self.conductor.spv_node.server.env.description = 'Fastest server in the west.' self.conductor.spv_node.server.env.daily_fee = '42' - from scribe.server.session import LBRYElectrumX LBRYElectrumX.set_server_features(self.conductor.spv_node.server.env) # await self.ledger.network.on_connected.first @@ -65,7 +65,7 @@ class NetworkTests(IntegrationTestCase): 'payment_address': payment_address, 'donation_address': donation_address, 'daily_fee': '42', - 'server_version': lbry.__version__, + 'server_version': scribe.__version__, 'trending_algorithm': 'fast_ar', }, await self.ledger.network.get_server_features()) diff --git a/tests/integration/blockchain/test_purchase_command.py b/tests/integration/blockchain/test_purchase_command.py index 8d3d45374..a83c9809a 100644 --- a/tests/integration/blockchain/test_purchase_command.py +++ b/tests/integration/blockchain/test_purchase_command.py @@ -1,6 +1,6 @@ from typing import Optional from lbry.testcase import CommandTestCase -from lbry.schema.purchase import Purchase +from scribe.schema.purchase import Purchase from lbry.wallet.transaction import Transaction from lbry.wallet.dewies import lbc_to_dewies, dewies_to_lbc diff --git a/tests/integration/blockchain/test_wallet_server_sessions.py b/tests/integration/blockchain/test_wallet_server_sessions.py index ba56dff22..d56a8ccb3 100644 --- a/tests/integration/blockchain/test_wallet_server_sessions.py +++ b/tests/integration/blockchain/test_wallet_server_sessions.py @@ -1,11 +1,11 @@ import asyncio -import lbry -import lbry.wallet +import scribe +from scribe.hub.session import LBRYElectrumX + from lbry.error import ServerPaymentFeeAboveMaxAllowedError from lbry.wallet.network import ClientSession from lbry.wallet.rpc import RPCError -from scribe.server.session import LBRYElectrumX from lbry.testcase import IntegrationTestCase, CommandTestCase from lbry.wallet.orchstr8.node import SPVNode @@ -34,7 +34,7 @@ class TestSessions(IntegrationTestCase): async def test_proper_version(self): info = await self.ledger.network.get_server_features() - self.assertEqual(lbry.__version__, info['server_version']) + self.assertEqual(scribe.__version__, info['server_version']) async def test_client_errors(self): # Goal is ensuring thsoe are raised and not trapped accidentally diff --git a/tests/integration/datanetwork/test_file_commands.py b/tests/integration/datanetwork/test_file_commands.py index e7360d316..0c94a3576 100644 --- a/tests/integration/datanetwork/test_file_commands.py +++ b/tests/integration/datanetwork/test_file_commands.py @@ -4,7 +4,7 @@ import asyncio import os from binascii import hexlify -from lbry.schema import Claim +from scribe.schema import Claim from lbry.stream.background_downloader import BackgroundDownloader from lbry.stream.descriptor import StreamDescriptor from lbry.testcase import CommandTestCase diff --git a/tests/integration/takeovers/test_resolve_command.py b/tests/integration/takeovers/test_resolve_command.py index 292d8cee4..6a266dff4 100644 --- a/tests/integration/takeovers/test_resolve_command.py +++ b/tests/integration/takeovers/test_resolve_command.py @@ -8,7 +8,7 @@ from collections import defaultdict from typing import NamedTuple, List from lbry.testcase import CommandTestCase from lbry.wallet.transaction import Transaction, Output -from lbry.schema.compat import OldClaimMessage +from scribe.schema.compat import OldClaimMessage from lbry.crypto.hash import sha256 from lbry.crypto.base58 import Base58 diff --git a/tests/integration/transactions/test_internal_transaction_api.py b/tests/integration/transactions/test_internal_transaction_api.py index 04817b2be..8d2312426 100644 --- a/tests/integration/transactions/test_internal_transaction_api.py +++ b/tests/integration/transactions/test_internal_transaction_api.py @@ -3,7 +3,7 @@ import asyncio from lbry.testcase import IntegrationTestCase import lbry.wallet -from lbry.schema.claim import Claim +from scribe.schema.claim import Claim from lbry.wallet.transaction import Transaction, Output, Input from lbry.wallet.dewies import dewies_to_lbc as d2l, lbc_to_dewies as l2d diff --git a/tests/unit/lbrynet_daemon/test_exchange_rate_manager.py b/tests/unit/lbrynet_daemon/test_exchange_rate_manager.py index 3fa2ff814..2b2454874 100644 --- a/tests/unit/lbrynet_daemon/test_exchange_rate_manager.py +++ b/tests/unit/lbrynet_daemon/test_exchange_rate_manager.py @@ -1,7 +1,7 @@ import asyncio from decimal import Decimal from time import time -from lbry.schema.claim import Claim +from scribe.schema.claim import Claim from lbry.extras.daemon.exchange_rate_manager import ( ExchangeRate, ExchangeRateManager, CurrencyConversionError, BittrexUSDFeed, BittrexBTCFeed, diff --git a/tests/unit/lbrynet_daemon/test_mime_types.py b/tests/unit/lbrynet_daemon/test_mime_types.py index 8272e3dbc..90b5f5d9b 100644 --- a/tests/unit/lbrynet_daemon/test_mime_types.py +++ b/tests/unit/lbrynet_daemon/test_mime_types.py @@ -1,5 +1,5 @@ import unittest -from lbry.schema import mime_types +from scribe.schema import mime_types class TestMimeTypes(unittest.TestCase): diff --git a/tests/unit/schema/test_claim_from_bytes.py b/tests/unit/schema/test_claim_from_bytes.py index 68664b233..35708e5a3 100644 --- a/tests/unit/schema/test_claim_from_bytes.py +++ b/tests/unit/schema/test_claim_from_bytes.py @@ -1,7 +1,7 @@ from unittest import TestCase from binascii import unhexlify -from lbry.schema import Claim +from scribe.schema import Claim class TestOldJSONSchemaCompatibility(TestCase): diff --git a/tests/unit/schema/test_mime_types.py b/tests/unit/schema/test_mime_types.py index 6d5beed2b..a9cc3de45 100644 --- a/tests/unit/schema/test_mime_types.py +++ b/tests/unit/schema/test_mime_types.py @@ -2,7 +2,7 @@ import unittest import tempfile import os -from lbry.schema.mime_types import guess_media_type +from scribe.schema.mime_types import guess_media_type class MediaTypeTests(unittest.TestCase): def test_guess_media_type_from_path_only(self): diff --git a/tests/unit/schema/test_models.py b/tests/unit/schema/test_models.py index 13c868ed0..486e051ca 100644 --- a/tests/unit/schema/test_models.py +++ b/tests/unit/schema/test_models.py @@ -1,7 +1,7 @@ from unittest import TestCase from decimal import Decimal -from lbry.schema.claim import Claim, Stream, Collection +from scribe.schema.claim import Claim, Stream, Collection from lbry.error import InputValueIsNoneError diff --git a/tests/unit/schema/test_tags.py b/tests/unit/schema/test_tags.py index 4dd637a29..3003c688f 100644 --- a/tests/unit/schema/test_tags.py +++ b/tests/unit/schema/test_tags.py @@ -1,6 +1,6 @@ import unittest -from lbry.schema.tags import normalize_tag, clean_tags +from scribe.schema.tags import normalize_tag, clean_tags class TestTagNormalization(unittest.TestCase): diff --git a/tests/unit/schema/test_url.py b/tests/unit/schema/test_url.py index 4fded5bc4..e14bb37b3 100644 --- a/tests/unit/schema/test_url.py +++ b/tests/unit/schema/test_url.py @@ -1,6 +1,6 @@ import unittest -from lbry.schema.url import URL +from scribe.schema.url import URL claim_id = "63f2da17b0d90042c559cc73b6b17f853945c43e" diff --git a/tests/unit/stream/test_stream_manager.py b/tests/unit/stream/test_stream_manager.py index a6767509f..370ff3476 100644 --- a/tests/unit/stream/test_stream_manager.py +++ b/tests/unit/stream/test_stream_manager.py @@ -22,7 +22,7 @@ from lbry.stream.descriptor import StreamDescriptor from lbry.dht.node import Node from lbry.dht.protocol.protocol import KademliaProtocol from lbry.dht.protocol.routing_table import TreeRoutingTable -from lbry.schema.claim import Claim +from scribe.schema.claim import Claim def get_mock_node(peer=None): diff --git a/tests/unit/wallet/test_bip32.py b/tests/unit/wallet/test_bip32.py index ef492bff4..53eddec31 100644 --- a/tests/unit/wallet/test_bip32.py +++ b/tests/unit/wallet/test_bip32.py @@ -1,7 +1,7 @@ from binascii import unhexlify, hexlify from lbry.testcase import AsyncioTestCase -from lbry.wallet.bip32 import PublicKey, PrivateKey, from_extended_key_string +from scribe.bip32 import PublicKey, PrivateKey, from_extended_key_string from lbry.wallet import Ledger, Database, Headers from tests.unit.wallet.key_fixtures import expected_ids, expected_privkeys, expected_hardened_privkeys diff --git a/tests/unit/wallet/test_schema_signing.py b/tests/unit/wallet/test_schema_signing.py index 901c40adb..1e0892a29 100644 --- a/tests/unit/wallet/test_schema_signing.py +++ b/tests/unit/wallet/test_schema_signing.py @@ -2,10 +2,10 @@ from binascii import unhexlify from lbry.testcase import AsyncioTestCase from lbry.wallet.constants import CENT, NULL_HASH32 -from lbry.wallet.bip32 import PrivateKey, KeyPath +from scribe.bip32 import PrivateKey, KeyPath from lbry.wallet.mnemonic import Mnemonic from lbry.wallet import Ledger, Database, Headers, Transaction, Input, Output -from lbry.schema.claim import Claim +from scribe.schema.claim import Claim from lbry.crypto.hash import sha256