diff --git a/lbrynet/__init__.py b/lbrynet/__init__.py index f18ed3b6d..7a0017b58 100644 --- a/lbrynet/__init__.py +++ b/lbrynet/__init__.py @@ -2,7 +2,7 @@ import logging log = logging.getLogger(__name__) logging.getLogger(__name__).addHandler(logging.NullHandler()) -log.setLevel(logging.ERROR) +log.setLevel(logging.INFO) __version__ = "0.3.11" version = tuple(__version__.split('.')) \ No newline at end of file diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 6bf8df48c..96e8b18dc 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -10,6 +10,7 @@ MAX_RESPONSE_INFO_SIZE = 2**16 MAX_BLOB_INFOS_TO_REQUEST = 20 BLOBFILES_DIR = ".blobfiles" BLOB_SIZE = 2**21 + MIN_BLOB_DATA_PAYMENT_RATE = .005 # points/megabyte MIN_BLOB_INFO_PAYMENT_RATE = .02 # points/1000 infos MIN_VALUABLE_BLOB_INFO_PAYMENT_RATE = .05 # points/1000 infos @@ -23,6 +24,9 @@ KNOWN_DHT_NODES = [('104.236.42.182', 4000), POINTTRADER_SERVER = 'http://ec2-54-187-192-68.us-west-2.compute.amazonaws.com:2424' #POINTTRADER_SERVER = 'http://127.0.0.1:2424' +SEARCH_SERVERS = ["http://lighthouse1.lbry.io:50005", + "http://lighthouse2.lbry.io:50005", + "http://lighthouse3.lbry.io:50005"] LOG_FILE_NAME = "lbrynet.log" LOG_POST_URL = "https://lbry.io/log-upload" @@ -42,11 +46,14 @@ DEFAULT_WALLET = "lbryum" WALLET_TYPES = ["lbryum", "lbrycrd"] DEFAULT_TIMEOUT = 30 DEFAULT_MAX_SEARCH_RESULTS = 25 -DEFAULT_MAX_KEY_FEE = 100.0 +DEFAULT_MAX_KEY_FEE = {'USD': {'amount': 25.0, 'address': ''}} DEFAULT_SEARCH_TIMEOUT = 3.0 DEFAULT_CACHE_TIME = 3600 DEFAULT_UI_BRANCH = "master" SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih'] -BASE_METADATA_FIELDS = ['title', 'description', 'author', 'language', 'license', 'content-type'] -OPTIONAL_METADATA_FIELDS = ['thumbnail', 'preview', 'fee', 'contact', 'pubkey'] +CURRENCIES = { + 'BTC': {'type': 'crypto'}, + 'LBC': {'type': 'crypto'}, + 'USD': {'type': 'fiat'}, + } diff --git a/lbrynet/core/Error.py b/lbrynet/core/Error.py index b2c4c3360..8146dc169 100644 --- a/lbrynet/core/Error.py +++ b/lbrynet/core/Error.py @@ -22,6 +22,10 @@ class ConnectionClosedBeforeResponseError(Exception): pass +class KeyFeeAboveMaxAllowed(Exception): + pass + + class UnknownNameError(Exception): def __init__(self, name): self.name = name @@ -30,6 +34,14 @@ class UnknownNameError(Exception): return repr(self.name) +class InvalidNameError(Exception): + def __init__(self, name): + self.name = name + + def __str__(self): + return repr(self.name) + + class UnknownStreamTypeError(Exception): def __init__(self, stream_type): self.stream_type = stream_type diff --git a/lbrynet/core/LBRYMetadata.py b/lbrynet/core/LBRYMetadata.py new file mode 100644 index 000000000..656b55c99 --- /dev/null +++ b/lbrynet/core/LBRYMetadata.py @@ -0,0 +1,128 @@ +import requests +import json +import time + +from copy import deepcopy +from googlefinance import getQuotes +from lbrynet.conf import CURRENCIES +from lbrynet.core import utils +import logging + +log = logging.getLogger(__name__) + +BITTREX_FEE = 0.0025 + +# Metadata version +SOURCE_TYPES = ['lbry_sd_hash', 'url', 'btih'] +NAME_ALLOWED_CHARSET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0987654321-' +BASE_METADATA_FIELDS = ['title', 'description', 'author', 'language', 'license', 'content-type', 'sources'] +OPTIONAL_METADATA_FIELDS = ['thumbnail', 'preview', 'fee', 'contact', 'pubkey'] + +MV001 = "0.0.1" +MV002 = "0.0.2" +CURRENT_METADATA_VERSION = MV002 + +METADATA_REVISIONS = {} +METADATA_REVISIONS[MV001] = {'required': BASE_METADATA_FIELDS, 'optional': OPTIONAL_METADATA_FIELDS} +METADATA_REVISIONS[MV002] = {'required': ['nsfw', 'ver'], 'optional': ['license_url']} + +# Fee version +BASE_FEE_FIELDS = ['amount', 'address'] + +FV001 = "0.0.1" +CURRENT_FEE_REVISION = FV001 + +FEE_REVISIONS = {} +FEE_REVISIONS[FV001] = {'required': BASE_FEE_FIELDS, 'optional': []} + + +def verify_name_characters(name): + for c in name: + assert c in NAME_ALLOWED_CHARSET, "Invalid character" + return True + + +class LBRYFeeValidator(dict): + def __init__(self, fee_dict): + dict.__init__(self) + assert len(fee_dict) == 1 + self.fee_version = None + self.currency_symbol = None + + fee_to_load = deepcopy(fee_dict) + + for currency in fee_dict: + self._verify_fee(currency, fee_to_load) + + self.amount = self._get_amount() + self.address = self[self.currency_symbol]['address'] + + def _get_amount(self): + amt = self[self.currency_symbol]['amount'] + if isinstance(amt, float): + return amt + else: + try: + return float(amt) + except TypeError: + log.error('Failed to convert %s to float', amt) + raise + + def _verify_fee(self, currency, f): + # str in case someone made a claim with a wierd fee + assert currency in CURRENCIES, "Unsupported currency: %s" % str(currency) + self.currency_symbol = currency + self.update({currency: {}}) + for version in FEE_REVISIONS: + self._load_revision(version, f) + if not f: + self.fee_version = version + break + assert f[self.currency_symbol] == {}, "Unknown fee keys: %s" % json.dumps(f.keys()) + + def _load_revision(self, version, f): + for k in FEE_REVISIONS[version]['required']: + assert k in f[self.currency_symbol], "Missing required fee field: %s" % k + self[self.currency_symbol].update({k: f[self.currency_symbol].pop(k)}) + for k in FEE_REVISIONS[version]['optional']: + if k in f[self.currency_symbol]: + self[self.currency_symbol].update({k: f[self.currency_symbol].pop(k)}) + + +class Metadata(dict): + def __init__(self, metadata): + dict.__init__(self) + self.meta_version = None + metadata_to_load = deepcopy(metadata) + + self._verify_sources(metadata_to_load) + self._verify_metadata(metadata_to_load) + + def _load_revision(self, version, metadata): + for k in METADATA_REVISIONS[version]['required']: + assert k in metadata, "Missing required metadata field: %s" % k + self.update({k: metadata.pop(k)}) + for k in METADATA_REVISIONS[version]['optional']: + if k == 'fee': + self._load_fee(metadata) + elif k in metadata: + self.update({k: metadata.pop(k)}) + + def _load_fee(self, metadata): + if 'fee' in metadata: + self['fee'] = LBRYFeeValidator(metadata.pop('fee')) + + def _verify_sources(self, metadata): + assert "sources" in metadata, "No sources given" + for source in metadata['sources']: + assert source in SOURCE_TYPES, "Unknown source type" + + def _verify_metadata(self, metadata): + for version in METADATA_REVISIONS: + self._load_revision(version, metadata) + if not metadata: + self.meta_version = version + if utils.version_is_greater_than(self.meta_version, "0.0.1"): + assert self.meta_version == self['ver'], "version mismatch" + break + assert metadata == {}, "Unknown metadata keys: %s" % json.dumps(metadata.keys()) diff --git a/lbrynet/core/LBRYWallet.py b/lbrynet/core/LBRYWallet.py index c4391bf88..60696624b 100644 --- a/lbrynet/core/LBRYWallet.py +++ b/lbrynet/core/LBRYWallet.py @@ -1,16 +1,12 @@ import sys -from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, ILBRYWallet -from lbrynet.core.client.ClientRequest import ClientRequest -from lbrynet.core.Error import UnknownNameError, InvalidStreamInfoError, RequestCanceledError -from lbrynet.core.Error import InsufficientFundsError -from lbrynet.core.sqlite_helpers import rerun_if_locked -from lbrynet.conf import BASE_METADATA_FIELDS, SOURCE_TYPES, OPTIONAL_METADATA_FIELDS - -from lbryum import SimpleConfig, Network -from lbryum.lbrycrd import COIN, TYPE_ADDRESS -from lbryum.wallet import WalletStorage, Wallet -from lbryum.commands import known_commands, Commands -from lbryum.transaction import Transaction +import datetime +import logging +import json +import subprocess +import socket +import time +import os +import requests from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException from twisted.internet import threads, reactor, defer, task @@ -19,13 +15,21 @@ from twisted.enterprise import adbapi from collections import defaultdict, deque from zope.interface import implements from decimal import Decimal -import datetime -import logging -import json -import subprocess -import socket -import time -import os +from googlefinance import getQuotes + +from lbryum import SimpleConfig, Network +from lbryum.lbrycrd import COIN, TYPE_ADDRESS +from lbryum.wallet import WalletStorage, Wallet +from lbryum.commands import known_commands, Commands +from lbryum.transaction import Transaction + +from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, ILBRYWallet +from lbrynet.core.client.ClientRequest import ClientRequest +from lbrynet.core.Error import UnknownNameError, InvalidStreamInfoError, RequestCanceledError +from lbrynet.core.Error import InsufficientFundsError +from lbrynet.core.sqlite_helpers import rerun_if_locked +from lbrynet.conf import SOURCE_TYPES +from lbrynet.core.LBRYMetadata import Metadata log = logging.getLogger(__name__) alert = logging.getLogger("lbryalert." + __name__) @@ -97,6 +101,7 @@ class LBRYWallet(object): def stop(self): self.stopped = True + # If self.next_manage_call is None, then manage is currently running or else # start has not been called, so set stopped and do nothing else. if self.next_manage_call is not None: @@ -315,54 +320,66 @@ class LBRYWallet(object): return d def _get_stream_info_from_value(self, result, name): - r_dict = {} - if 'value' in result: - value = result['value'] + def _check_result_fields(r): + for k in ['value', 'txid', 'n', 'height', 'amount']: + assert k in r, "getvalueforname response missing field %s" % k - try: - value_dict = json.loads(value) - except (ValueError, TypeError): - return Failure(InvalidStreamInfoError(name)) - r_dict['sources'] = value_dict['sources'] - for field in BASE_METADATA_FIELDS: - r_dict[field] = value_dict[field] - for field in value_dict: - if field in OPTIONAL_METADATA_FIELDS: - r_dict[field] = value_dict[field] - - if 'txid' in result: - d = self._save_name_metadata(name, r_dict['sources']['lbry_sd_hash'], str(result['txid'])) - d.addCallback(lambda _: r_dict) - return d - elif 'error' in result: + if 'error' in result: log.warning("Got an error looking up a name: %s", result['error']) return Failure(UnknownNameError(name)) - else: - log.warning("Got an error looking up a name: %s", json.dumps(result)) + + _check_result_fields(result) + + try: + metadata = Metadata(json.loads(result['value'])) + except (ValueError, TypeError): + return Failure(InvalidStreamInfoError(name)) + + d = self._save_name_metadata(name, str(result['txid']), metadata['sources']['lbry_sd_hash']) + d.addCallback(lambda _: log.info("lbry://%s complies with %s" % (name, metadata.meta_version))) + d.addCallback(lambda _: metadata) + return d + + def _get_claim_info(self, result, name): + def _check_result_fields(r): + for k in ['value', 'txid', 'n', 'height', 'amount']: + assert k in r, "getvalueforname response missing field %s" % k + + def _build_response(m, result): + result['value'] = m + return result + + if 'error' in result: + log.warning("Got an error looking up a name: %s", result['error']) return Failure(UnknownNameError(name)) - def claim_name(self, name, bid, sources, metadata, fee=None): - value = {'sources': {}} - for k in SOURCE_TYPES: - if k in sources: - value['sources'][k] = sources[k] - if value['sources'] == {}: - return defer.fail("No source given") - for k in BASE_METADATA_FIELDS: - if k not in metadata: - return defer.fail("Missing required field '%s'" % k) - value[k] = metadata[k] - for k in metadata: - if k not in BASE_METADATA_FIELDS: - value[k] = metadata[k] - if fee is not None: - if "LBC" in fee: - value['fee'] = {'LBC': {'amount': fee['LBC']['amount'], 'address': fee['LBC']['address']}} + _check_result_fields(result) - d = self._send_name_claim(name, json.dumps(value), bid) + try: + metadata = Metadata(json.loads(result['value'])) + except (ValueError, TypeError): + return Failure(InvalidStreamInfoError(name)) + + d = self._save_name_metadata(name, str(result['txid']), metadata['sources']['lbry_sd_hash']) + d.addCallback(lambda _: log.info("lbry://%s complies with %s" % (name, metadata.meta_version))) + d.addCallback(lambda _: _build_response(metadata, result)) + return d + + def get_claim_info(self, name): + d = self._get_value_for_name(name) + d.addCallback(lambda r: self._get_claim_info(r, name)) + return d + + + def claim_name(self, name, bid, m): + + metadata = Metadata(m) + + d = self._send_name_claim(name, json.dumps(metadata), bid) def _save_metadata(txid): - d = self._save_name_metadata(name, value['sources']['lbry_sd_hash'], txid) + log.info("Saving metadata for claim %s" % txid) + d = self._save_name_metadata(name, txid, metadata['sources']['lbry_sd_hash']) d.addCallback(lambda _: txid) return d @@ -407,10 +424,12 @@ class LBRYWallet(object): d.addCallback(self._get_decoded_tx) return d - def update_name(self, name, value, amount): + def update_name(self, name, bid, value, old_txid): d = self._get_value_for_name(name) - d.addCallback(lambda r: (self._update_name(r['txid'], json.dumps(value), amount), r['txid'])) - d.addCallback(lambda (new_txid, old_txid): self._update_name_metadata(name, value['sources']['lbry_sd_hash'], old_txid, new_txid)) + d.addCallback(lambda r: self.abandon_name(r['txid'] if not old_txid else old_txid)) + d.addCallback(lambda r: log.info("Abandon claim tx %s" % str(r))) + d.addCallback(lambda _: self.claim_name(name, bid, value)) + return d def get_name_and_validity_for_sd_hash(self, sd_hash): @@ -520,19 +539,13 @@ class LBRYWallet(object): " txid text, " + " sd_hash text)") - def _save_name_metadata(self, name, sd_hash, txid): - d = self.db.runQuery("select * from name_metadata where txid=?", (txid,)) + def _save_name_metadata(self, name, txid, sd_hash): + d = self.db.runQuery("select * from name_metadata where name=? and txid=? and sd_hash=?", (name, txid, sd_hash)) d.addCallback(lambda r: self.db.runQuery("insert into name_metadata values (?, ?, ?)", (name, txid, sd_hash)) if not len(r) else None) return d - def _update_name_metadata(self, name, sd_hash, old_txid, new_txid): - d = self.db.runQuery("delete * from name_metadata where txid=? and sd_hash=?", (old_txid, sd_hash)) - d.addCallback(lambda _: self.db.runQuery("insert into name_metadata values (?, ?, ?)", (name, new_txid, sd_hash))) - d.addCallback(lambda _: new_txid) - return d - def _get_claim_metadata_for_sd_hash(self, sd_hash): d = self.db.runQuery("select name, txid from name_metadata where sd_hash=?", (sd_hash,)) d.addCallback(lambda r: r[0] if len(r) else None) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 6cd843067..ea9124d57 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -1,6 +1,4 @@ import binascii -from datetime import datetime -from decimal import Decimal import distutils.version import locale import logging.handlers @@ -13,18 +11,21 @@ import socket import string import subprocess import sys -from urllib2 import urlopen - -from appdirs import user_data_dir import base58 import requests import simplejson as json +import pkg_resources + +from urllib2 import urlopen +from appdirs import user_data_dir +from datetime import datetime +from decimal import Decimal from twisted.web import server from twisted.internet import defer, threads, error, reactor from twisted.internet.task import LoopingCall from txjsonrpc import jsonrpclib from txjsonrpc.web import jsonrpc -from txjsonrpc.web.jsonrpc import Handler +from txjsonrpc.web.jsonrpc import Handler, Proxy from lbrynet import __version__ as lbrynet_version from lbryum.version import LBRYUM_VERSION as lbryum_version @@ -32,19 +33,21 @@ from lbrynet.core.PaymentRateManager import PaymentRateManager from lbrynet.core.server.BlobAvailabilityHandler import BlobAvailabilityHandlerFactory from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory from lbrynet.core.server.ServerProtocol import ServerProtocolFactory -from lbrynet.core.Error import UnknownNameError, InsufficientFundsError +from lbrynet.core.Error import UnknownNameError, InsufficientFundsError, InvalidNameError from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType from lbrynet.lbryfile.client.LBRYFileDownloader import LBRYFileSaverFactory, LBRYFileOpenerFactory from lbrynet.lbryfile.client.LBRYFileOptions import add_lbry_file_to_sd_identifier from lbrynet.lbrynet_daemon.LBRYUIManager import LBRYUIManager from lbrynet.lbrynet_daemon.LBRYDownloader import GetStream from lbrynet.lbrynet_daemon.LBRYPublisher import Publisher +from lbrynet.lbrynet_daemon.LBRYExchangeRateManager import ExchangeRateManager from lbrynet.core import utils +from lbrynet.core.LBRYMetadata import verify_name_characters from lbrynet.core.utils import generate_id from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, DEFAULT_MAX_SEARCH_RESULTS, KNOWN_DHT_NODES, DEFAULT_MAX_KEY_FEE, \ - DEFAULT_WALLET, DEFAULT_SEARCH_TIMEOUT, DEFAULT_CACHE_TIME, DEFAULT_UI_BRANCH, LOG_POST_URL, LOG_FILE_NAME, \ - BASE_METADATA_FIELDS, OPTIONAL_METADATA_FIELDS, SOURCE_TYPES + DEFAULT_WALLET, DEFAULT_SEARCH_TIMEOUT, DEFAULT_CACHE_TIME, DEFAULT_UI_BRANCH, LOG_POST_URL, LOG_FILE_NAME, SOURCE_TYPES +from lbrynet.conf import SEARCH_SERVERS from lbrynet.conf import DEFAULT_TIMEOUT, WALLET_TYPES from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob from lbrynet.core.Session import LBRYSession @@ -68,7 +71,6 @@ lbrynet_log = os.path.join(log_dir, LOG_FILE_NAME) log = logging.getLogger(__name__) - if os.path.isfile(lbrynet_log): with open(lbrynet_log, 'r') as f: PREVIOUS_LBRYNET_LOG = len(f.read()) @@ -157,8 +159,10 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.current_db_revision = 1 self.run_server = True self.session = None + self.exchange_rate_manager = ExchangeRateManager() self.waiting_on = {} self.streams = {} + self.pending_claims = {} self.known_dht_nodes = KNOWN_DHT_NODES self.first_run_after_update = False self.uploaded_temp_files = [] @@ -242,6 +246,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.session_settings['last_version'] = self.default_settings['last_version'] self.first_run_after_update = True log.info("First run after update") + log.info("lbrynet %s --> %s" % (self.session_settings['last_version']['lbrynet'], self.default_settings['last_version']['lbrynet'])) + log.info("lbryum %s --> %s" % (self.session_settings['last_version']['lbryum'], self.default_settings['last_version']['lbryum'])) f = open(self.daemon_conf, "w") f.write(json.dumps(self.session_settings)) @@ -347,6 +353,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.internet_connection_checker = LoopingCall(self._check_network_connection) self.version_checker = LoopingCall(self._check_remote_versions) self.connection_problem_checker = LoopingCall(self._check_connection_problems) + self.pending_claim_checker = LoopingCall(self._check_pending_claims) # self.lbrynet_connection_checker = LoopingCall(self._check_lbrynet_connection) self.sd_identifier = StreamDescriptorIdentifier() @@ -412,6 +419,10 @@ class LBRYDaemon(jsonrpc.JSONRPC): return server.NOT_DONE_YET def _cbRender(self, result, request, id, version): + def default_decimal(obj): + if isinstance(obj, Decimal): + return float(obj) + if isinstance(result, Handler): result = result.result @@ -423,7 +434,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): result = (result,) # Convert the result (python) to JSON-RPC try: - s = jsonrpclib.dumps(result, version=version) + s = jsonrpclib.dumps(result, version=version, default=default_decimal) except: f = jsonrpclib.Fault(self.FAILURE, "can't serialize output") s = jsonrpclib.dumps(f, version=version) @@ -480,6 +491,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.internet_connection_checker.start(3600) self.version_checker.start(3600 * 12) self.connection_problem_checker.start(1) + self.exchange_rate_manager.start() + if host_ui: self.lbry_ui_manager.update_checker.start(1800, now=False) @@ -603,6 +616,47 @@ class LBRYDaemon(jsonrpc.JSONRPC): if not self.connected_to_internet: self.connection_problem = CONNECTION_PROBLEM_CODES[1] + def _add_to_pending_claims(self, name, txid): + log.info("Adding lbry://%s to pending claims, txid %s" % (name, txid)) + self.pending_claims[name] = txid + return txid + + def _check_pending_claims(self): + # TODO: this was blatantly copied from jsonrpc_start_lbry_file. Be DRY. + def _start_file(f): + d = self.lbry_file_manager.toggle_lbry_file_running(f) + return defer.succeed("Started LBRY file") + + def _get_and_start_file(name): + d = defer.succeed(self.pending_claims.pop(name)) + d.addCallback(lambda _: self._get_lbry_file("name", name, return_json=False)) + d.addCallback(lambda l: _start_file(l) if l.stopped else "LBRY file was already running") + + def re_add_to_pending_claims(name): + txid = self.pending_claims.pop(name) + self._add_to_pending_claims(name, txid) + + def _process_lbry_file(name, lbry_file): + # lbry_file is an instance of ManagedLBRYFileDownloader or None + # TODO: check for sd_hash in addition to txid + ready_to_start = ( + lbry_file and + self.pending_claims[name] == lbry_file.txid + ) + if ready_to_start: + _get_and_start_file(name) + else: + re_add_to_pending_claims(name) + + for name in self.pending_claims: + log.info("Checking if new claim for lbry://%s is confirmed" % name) + d = self._resolve_name(name, force_refresh=True) + d.addCallback(lambda _: self._get_lbry_file_by_uri(name)) + d.addCallbacks( + lambda lbry_file: _process_lbry_file(name, lbry_file), + lambda _: re_add_to_pending_claims(name) + ) + def _start_server(self): if self.peer_port is not None: @@ -721,6 +775,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.connection_problem_checker.stop() if self.lbry_ui_manager.update_checker.running: self.lbry_ui_manager.update_checker.stop() + if self.pending_claim_checker.running: + self.pending_claim_checker.stop() self._clean_up_temp_files() @@ -1089,8 +1145,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): return defer.succeed(None) self.streams[name] = GetStream(self.sd_identifier, self.session, self.session.wallet, - self.lbry_file_manager, max_key_fee=self.max_key_fee, - data_rate=self.data_rate, timeout=timeout, + self.lbry_file_manager, self.exchange_rate_manager, + max_key_fee=self.max_key_fee, data_rate=self.data_rate, timeout=timeout, download_directory=download_directory, file_name=file_name) d = self.streams[name].start(stream_info, name) if wait_for_write: @@ -1120,6 +1176,12 @@ class LBRYDaemon(jsonrpc.JSONRPC): return defer.succeed(True) def _resolve_name(self, name, force_refresh=False): + try: + verify_name_characters(name) + except AssertionError: + log.error("Bad name") + return defer.fail(InvalidNameError("Bad name")) + def _cache_stream_info(stream_info): def _add_txid(txid): self.name_cache[name]['txid'] = txid @@ -1186,7 +1248,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): def _add_key_fee(data_cost): d = self._resolve_name(name) - d.addCallback(lambda info: data_cost if 'fee' not in info else data_cost + info['fee']['LBC']['amount']) + d.addCallback(lambda info: self.exchange_rate_manager.to_lbc(info.get('fee', None))) + d.addCallback(lambda fee: data_cost if fee is None else data_cost + fee.amount) return d d = self._resolve_name(name) @@ -1196,8 +1259,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(self.sd_identifier.get_metadata_for_sd_blob) d.addCallback(lambda metadata: metadata.validator.info_to_show()) d.addCallback(lambda info: int(dict(info)['stream_size']) / 1000000 * self.data_rate) - d.addCallback(_add_key_fee) - d.addErrback(lambda _: _add_key_fee(0.0)) + d.addCallbacks(_add_key_fee, lambda _: _add_key_fee(0.0)) reactor.callLater(self.search_timeout, _check_est, d, name) return d @@ -1305,7 +1367,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): if f.txid: d = self._resolve_name(f.uri) - d.addCallback(_add_to_dict) + d.addCallbacks(_add_to_dict, lambda _: _add_to_dict("Pending confirmation")) else: d = defer.succeed(message) return d @@ -1363,6 +1425,10 @@ class LBRYDaemon(jsonrpc.JSONRPC): return defer.succeed(None) + def _search(self, search): + proxy = Proxy(random.choice(SEARCH_SERVERS)) + return proxy.callRemote('search', search) + def _render_response(self, result, code): return defer.succeed({'result': result, 'code': code}) @@ -1482,10 +1548,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): 'ui_version': self.ui_version, 'remote_lbrynet': self.git_lbrynet_version, 'remote_lbryum': self.git_lbryum_version, - 'lbrynet_update_available': utils.version_is_greater_than( - self.git_lbrynet_version, lbrynet_version), - 'lbryum_update_available': utils.version_is_greater_than( - self.git_lbryum_version, lbryum_version) + 'lbrynet_update_available': utils.version_is_greater_than(self.git_lbrynet_version, lbrynet_version), + 'lbryum_update_available': utils.version_is_greater_than(self.git_lbryum_version, lbryum_version), } log.info("Get version info: " + json.dumps(msg)) @@ -1667,15 +1731,37 @@ class LBRYDaemon(jsonrpc.JSONRPC): metadata from name claim """ - if 'name' in p.keys(): + force = p.get('force', False) + + if 'name' in p: name = p['name'] else: return self._render_response(None, BAD_REQUEST) - d = self._resolve_name(name) + d = self._resolve_name(name, force_refresh=force) d.addCallbacks(lambda info: self._render_response(info, OK_CODE), lambda _: server.failure) return d + def jsonrpc_get_claim_info(self, p): + """ + Resolve claim info from a LBRY uri + + Args: + 'name': name to look up, string, do not include lbry:// prefix + Returns: + txid, amount, value, n, height + """ + + def _convert_amount_to_float(r): + r['amount'] = float(r['amount']) / 10**8 + return r + + name = p['name'] + d = self.session.wallet.get_claim_info(name) + d.addCallback(_convert_amount_to_float) + d.addCallback(lambda r: self._render_response(r, OK_CODE)) + return d + def jsonrpc_get(self, p): """ Download stream from a LBRY uri @@ -1786,9 +1872,24 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d + def jsonrpc_get_est_cost(self, p): + """ + Get estimated cost for a lbry uri + + Args: + 'name': lbry uri + Returns: + estimated cost + """ + + name = p['name'] + d = self._get_est_cost(name) + d.addCallback(lambda r: self._render_response(r, OK_CODE)) + return d + def jsonrpc_search_nametrie(self, p): """ - Search the nametrie for claims beginning with search (yes, this is a dumb search, it'll be made better) + Search the nametrie for claims Args: 'search': search query, string @@ -1796,6 +1897,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): List of search results """ + # TODO: change this function to "search", and use cached stream size info from the search server + if 'search' in p.keys(): search = p['search'] else: @@ -1805,44 +1908,31 @@ class LBRYDaemon(jsonrpc.JSONRPC): t = [] for i in n: if i[0]: - if i[1][0][0] and i[1][1][0] and i[1][2][0]: - i[1][0][1]['value'] = str(i[1][0][1]['value']) - t.append([i[1][0][1], i[1][1][1], i[1][2][1]]) + tr = {} + tr.update(i[1][0]['value']) + thumb = tr.get('thumbnail', None) + if thumb is None: + tr['thumbnail'] = "img/Free-speech-flag.svg" + tr['name'] = i[1][0]['name'] + tr['cost_est'] = i[1][1] + t.append(tr) return t - def resolve_claims(claims): - ds = [] - for claim in claims: - d1 = defer.succeed(claim) - d2 = self._resolve_name(claim['name']) - d3 = self._get_est_cost(claim['name']) - dl = defer.DeferredList([d1, d2, d3], consumeErrors=True) - ds.append(dl) - return defer.DeferredList(ds) + def get_est_costs(results): + def _save_cost(search_result): + d = self._get_est_cost(search_result['name']) + d.addCallback(lambda p: [search_result, p]) + return d - def _disp(results): - log.info('Found ' + str(len(results)) + ' search results') - consolidated_results = [] - for r in results: - t = {} - t.update(r[0]) - if not 'thumbnail' in r[1].keys(): - r[1]['thumbnail'] = "img/Free-speech-flag.svg" - t.update(r[1]) - t['cost_est'] = r[2] - consolidated_results.append(t) - # log.info(str(t)) + dl = defer.DeferredList([_save_cost(r) for r in results], consumeErrors=True) + return dl - return consolidated_results + log.info('Search: %s' % search) - log.info('Search nametrie: ' + search) - - d = self.session.wallet.get_nametrie() - d.addCallback(lambda trie: [claim for claim in trie if claim['name'].startswith(search) and 'txid' in claim]) + d = self._search(search) d.addCallback(lambda claims: claims[:self.max_search_results]) - d.addCallback(resolve_claims) + d.addCallback(get_est_costs) d.addCallback(_clean) - d.addCallback(_disp) d.addCallback(lambda results: self._render_response(results, OK_CODE)) return d @@ -1889,21 +1979,49 @@ class LBRYDaemon(jsonrpc.JSONRPC): Returns: Claim txid """ - # start(self, name, file_path, bid, metadata, fee=None, sources=None): + name = p['name'] + try: + verify_name_characters(name) + except: + log.error("Bad name") + return defer.fail(InvalidNameError("Bad name")) bid = p['bid'] file_path = p['file_path'] metadata = p['metadata'] + + def _set_address(address, currency): + log.info("Generated new address for key fee: " + str(address)) + metadata['fee'][currency]['address'] = address + return defer.succeed(None) + + def _delete_data(lbry_file): + txid = lbry_file.txid + d = self._delete_lbry_file(lbry_file, delete_file=False) + d.addCallback(lambda _: txid) + return d + + if not self.pending_claim_checker.running: + self.pending_claim_checker.start(30) + + d = self._resolve_name(name, force_refresh=True) + d.addErrback(lambda _: None) + if 'fee' in p: - fee = p['fee'] - else: - fee = None + metadata['fee'] = p['fee'] + assert len(metadata['fee']) == 1, "Too many fees" + for c in metadata['fee']: + if 'address' not in metadata['fee'][c]: + d.addCallback(lambda _: self.session.wallet.get_new_address()) + d.addCallback(lambda addr: _set_address(addr, c)) pub = Publisher(self.session, self.lbry_file_manager, self.session.wallet) - - d = pub.start(name, file_path, bid, metadata, fee) - d.addCallbacks(lambda msg: self._render_response(msg, OK_CODE), - lambda err: self._render_response(err.getTraceback(), BAD_REQUEST)) + d.addCallback(lambda _: self._get_lbry_file_by_uri(name)) + d.addCallbacks(lambda l: None if not l else _delete_data(l), lambda _: None) + d.addCallback(lambda r: pub.start(name, file_path, bid, metadata, r)) + d.addCallback(lambda txid: self._add_to_pending_claims(name, txid)) + d.addCallback(lambda r: self._render_response(r, OK_CODE)) + d.addErrback(lambda err: self._render_response(err.getTraceback(), BAD_REQUEST)) return d @@ -2166,27 +2284,6 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d - def jsonrpc_update_name(self, p): - """ - Update name claim - - Args: - 'name': the uri of the claim to be updated - 'metadata': new metadata dict - 'amount': bid amount of updated claim - Returns: - txid - """ - - name = p['name'] - metadata = p['metadata'] if isinstance(p['metadata'], dict) else json.loads(p['metadata']) - amount = p['amount'] - - d = self.session.wallet.update_name(name, metadata, amount) - d.addCallback(lambda r: self._render_response(r, OK_CODE)) - - return d - def jsonrpc_log(self, p): """ Log message @@ -2282,7 +2379,6 @@ class LBRYDaemon(jsonrpc.JSONRPC): # No easy way to reveal specific files on Linux, so just open the containing directory d = threads.deferToThread(subprocess.Popen, ['xdg-open', os.dirname(path)]) - d.addCallback(lambda _: self._render_response(True, OK_CODE)) return d diff --git a/lbrynet/lbrynet_daemon/LBRYDownloader.py b/lbrynet/lbrynet_daemon/LBRYDownloader.py index 0fceb441d..62d9cd33d 100644 --- a/lbrynet/lbrynet_daemon/LBRYDownloader.py +++ b/lbrynet/lbrynet_daemon/LBRYDownloader.py @@ -3,14 +3,16 @@ import logging import os import sys +from copy import deepcopy from appdirs import user_data_dir from datetime import datetime from twisted.internet import defer from twisted.internet.task import LoopingCall -from lbrynet.core.Error import InvalidStreamInfoError, InsufficientFundsError +from lbrynet.core.Error import InvalidStreamInfoError, InsufficientFundsError, KeyFeeAboveMaxAllowed from lbrynet.core.PaymentRateManager import PaymentRateManager from lbrynet.core.StreamDescriptor import download_sd_blob +from lbrynet.core.LBRYMetadata import Metadata, LBRYFeeValidator from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloaderFactory from lbrynet.conf import DEFAULT_TIMEOUT, LOG_FILE_NAME @@ -40,17 +42,17 @@ log = logging.getLogger(__name__) class GetStream(object): - def __init__(self, sd_identifier, session, wallet, lbry_file_manager, max_key_fee, data_rate=0.5, - timeout=DEFAULT_TIMEOUT, download_directory=None, file_name=None): + def __init__(self, sd_identifier, session, wallet, lbry_file_manager, exchange_rate_manager, + max_key_fee, data_rate=0.5, timeout=DEFAULT_TIMEOUT, download_directory=None, file_name=None): self.wallet = wallet self.resolved_name = None self.description = None - self.key_fee = None - self.key_fee_address = None + self.fee = None self.data_rate = data_rate self.name = None self.file_name = file_name self.session = session + self.exchange_rate_manager = exchange_rate_manager self.payment_rate_manager = PaymentRateManager(self.session.base_payment_rate_manager) self.lbry_file_manager = lbry_file_manager self.sd_identifier = sd_identifier @@ -64,7 +66,7 @@ class GetStream(object): self.download_directory = download_directory self.download_path = None self.downloader = None - self.finished = defer.Deferred() + self.finished = defer.Deferred(None) self.checker = LoopingCall(self.check_status) self.code = STREAM_STAGES[0] @@ -83,30 +85,16 @@ class GetStream(object): self.code = STREAM_STAGES[4] self.finished.callback(False) - def start(self, stream_info, name): - self.resolved_name = name - self.stream_info = stream_info - if 'sources' in self.stream_info: - self.stream_hash = self.stream_info['sources']['lbry_sd_hash'] - else: - raise InvalidStreamInfoError(self.stream_info) - if 'description' in self.stream_info: - self.description = self.stream_info['description'] - if 'fee' in self.stream_info: - if 'LBC' in self.stream_info['fee']: - self.key_fee = float(self.stream_info['fee']['LBC']['amount']) - self.key_fee_address = self.stream_info['fee']['LBC']['address'] - else: - self.key_fee_address = None - else: - self.key_fee = None - self.key_fee_address = None - if self.key_fee > self.max_key_fee: - log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.key_fee, self.max_key_fee, self.resolved_name)) - return defer.fail(None) - else: - pass + def _convert_max_fee(self): + if isinstance(self.max_key_fee, dict): + max_fee = LBRYFeeValidator(self.max_key_fee) + if max_fee.currency_symbol == "LBC": + return max_fee.amount + return self.exchange_rate_manager.to_lbc(self.fee).amount + elif isinstance(self.max_key_fee, float): + return float(self.max_key_fee) + def start(self, stream_info, name): def _cause_timeout(err): log.error(err) log.debug('Forcing a timeout') @@ -131,6 +119,23 @@ class GetStream(object): download_directory=self.download_directory, file_name=self.file_name) + self.resolved_name = name + self.stream_info = deepcopy(stream_info) + self.description = self.stream_info['description'] + self.stream_hash = self.stream_info['sources']['lbry_sd_hash'] + + if 'fee' in self.stream_info: + self.fee = LBRYFeeValidator(self.stream_info['fee']) + max_key_fee = self._convert_max_fee() + if self.exchange_rate_manager.to_lbc(self.fee).amount > max_key_fee: + log.info("Key fee %f above limit of %f didn't download lbry://%s" % (self.fee.amount, + self.max_key_fee, + self.resolved_name)) + return defer.fail(KeyFeeAboveMaxAllowed()) + log.info("Key fee %s below limit of %f, downloading lbry://%s" % (json.dumps(self.fee), + max_key_fee, + self.resolved_name)) + self.checker.start(1) self.d.addCallback(lambda _: _set_status(None, DOWNLOAD_METADATA_CODE)) @@ -146,17 +151,20 @@ class GetStream(object): def _start_download(self, downloader): def _pay_key_fee(): - if self.key_fee is not None and self.key_fee_address is not None: - reserved_points = self.wallet.reserve_points(self.key_fee_address, self.key_fee) + if self.fee is not None: + fee_lbc = self.exchange_rate_manager.to_lbc(self.fee).amount + reserved_points = self.wallet.reserve_points(self.fee.address, fee_lbc) if reserved_points is None: return defer.fail(InsufficientFundsError()) - log.info("Key fee: %f --> %s" % (self.key_fee, self.key_fee_address)) - return self.wallet.send_points_to_address(reserved_points, self.key_fee) + return self.wallet.send_points_to_address(reserved_points, fee_lbc) + return defer.succeed(None) d = _pay_key_fee() + self.downloader = downloader self.download_path = os.path.join(downloader.download_directory, downloader.file_name) + d.addCallback(lambda _: log.info("Downloading %s --> %s", self.stream_hash, self.downloader.file_name)) d.addCallback(lambda _: self.downloader.start()) diff --git a/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py new file mode 100644 index 000000000..0af938954 --- /dev/null +++ b/lbrynet/lbrynet_daemon/LBRYExchangeRateManager.py @@ -0,0 +1,215 @@ +import time +import requests +import logging +import json +import googlefinance +from twisted.internet import defer, reactor +from twisted.internet.task import LoopingCall + +from lbrynet.core.LBRYMetadata import LBRYFeeValidator + +log = logging.getLogger(__name__) + +CURRENCY_PAIRS = ["USDBTC", "BTCLBC"] +BITTREX_FEE = 0.0025 +COINBASE_FEE = 0.0 #add fee + + +class ExchangeRate(object): + def __init__(self, market, spot, ts): + assert int(time.time()) - ts < 600 + self.currency_pair = (market[0:3], market[3:6]) + self.spot = spot + self.ts = ts + + def as_dict(self): + return {'spot': self.spot, 'ts': self.ts} + + +class MarketFeed(object): + def __init__(self, market, name, url, params, fee): + self.market = market + self.name = name + self.url = url + self.params = params + self.fee = fee + self.rate = None + self._updater = LoopingCall(self._update_price) + + def _make_request(self): + r = requests.get(self.url, self.params) + return r.text + + def _handle_response(self, response): + return NotImplementedError + + def _subtract_fee(self, from_amount): + return defer.succeed(from_amount / (1.0 - self.fee)) + + def _save_price(self, price): + log.info("Saving price update %f for %s" % (price, self.market)) + self.rate = ExchangeRate(self.market, price, int(time.time())) + + def _update_price(self): + d = defer.succeed(self._make_request()) + d.addCallback(self._handle_response) + d.addCallback(self._subtract_fee) + d.addCallback(self._save_price) + + def start(self): + if not self._updater.running: + self._updater.start(300) + + def stop(self): + if self._updater.running: + self._updater.stop() + + +class BittrexFeed(MarketFeed): + def __init__(self): + MarketFeed.__init__( + self, + "BTCLBC", + "Bittrex", + "https://bittrex.com/api/v1.1/public/getmarkethistory", + {'market': 'BTC-LBC', 'count': 50}, + BITTREX_FEE + ) + + def _handle_response(self, response): + trades = json.loads(response)['result'] + vwap = sum([i['Total'] for i in trades]) / sum([i['Quantity'] for i in trades]) + return defer.succeed(float(1.0 / vwap)) + + +class GoogleBTCFeed(MarketFeed): + def __init__(self): + MarketFeed.__init__( + self, + "USDBTC", + "Coinbase via Google finance", + None, + None, + COINBASE_FEE + ) + + def _make_request(self): + return googlefinance.getQuotes('CURRENCY:USDBTC')[0] + + def _handle_response(self, response): + return float(response['LastTradePrice']) + + +def get_default_market_feed(currency_pair): + currencies = None + if isinstance(currency_pair, str): + currencies = (currency_pair[0:3], currency_pair[3:6]) + elif isinstance(currency_pair, tuple): + currencies = currency_pair + assert currencies is not None + + if currencies == ("USD", "BTC"): + return GoogleBTCFeed() + elif currencies == ("BTC", "LBC"): + return BittrexFeed() + + +class ExchangeRateManager(object): + def __init__(self): + reactor.addSystemEventTrigger('before', 'shutdown', self.stop) + self.market_feeds = [get_default_market_feed(currency_pair) for currency_pair in CURRENCY_PAIRS] + + def start(self): + log.info("Starting exchange rate manager") + for feed in self.market_feeds: + feed.start() + + def stop(self): + log.info("Stopping exchange rate manager") + for source in self.market_feeds: + source.stop() + + def convert_currency(self, from_currency, to_currency, amount): + log.info("Converting %f %s to %s" % (amount, from_currency, to_currency)) + if from_currency == to_currency: + return amount + for market in self.market_feeds: + if market.rate.currency_pair == (from_currency, to_currency): + return amount * market.rate.spot + for market in self.market_feeds: + if market.rate.currency_pair[0] == from_currency: + return self.convert_currency(market.rate.currency_pair[1], to_currency, amount * market.rate.spot) + raise Exception('Unable to convert {} from {} to {}'.format(amount, from_currency, to_currency)) + + def fee_dict(self): + return {market: market.rate.as_dict() for market in self.market_feeds} + + def to_lbc(self, fee): + if fee is None: + return None + if not isinstance(fee, LBRYFeeValidator): + fee_in = LBRYFeeValidator(fee) + else: + fee_in = fee + + return LBRYFeeValidator({fee_in.currency_symbol: + { + 'amount': self.convert_currency(fee_in.currency_symbol, "LBC", fee_in.amount), + 'address': fee_in.address + } + }) + + +class DummyBTCLBCFeed(MarketFeed): + def __init__(self): + MarketFeed.__init__( + self, + "BTCLBC", + "market name", + "derp.com", + None, + 0.0 + ) + + +class DummyUSDBTCFeed(MarketFeed): + def __init__(self): + MarketFeed.__init__( + self, + "USDBTC", + "market name", + "derp.com", + None, + 0.0 + ) + + +class DummyExchangeRateManager(object): + def __init__(self, rates): + self.market_feeds = [DummyBTCLBCFeed(), DummyUSDBTCFeed()] + for feed in self.market_feeds: + feed.rate = ExchangeRate(feed.market, rates[feed.market]['spot'], rates[feed.market]['ts']) + + def convert_currency(self, from_currency, to_currency, amount): + log.info("Converting %f %s to %s" % (amount, from_currency, to_currency)) + for market in self.market_feeds: + if market.rate.currency_pair == (from_currency, to_currency): + return amount * market.rate.spot + for market in self.market_feeds: + if market.rate.currency_pair[0] == from_currency: + return self.convert_currency(market.rate.currency_pair[1], to_currency, amount * market.rate.spot) + + def to_lbc(self, fee): + if fee is None: + return None + if not isinstance(fee, LBRYFeeValidator): + fee_in = LBRYFeeValidator(fee) + else: + fee_in = fee + + return LBRYFeeValidator({fee_in.currency_symbol: + { + 'amount': self.convert_currency(fee_in.currency_symbol, "LBC", fee_in.amount), + 'address': fee_in.address + } + }) \ No newline at end of file diff --git a/lbrynet/lbrynet_daemon/LBRYPublisher.py b/lbrynet/lbrynet_daemon/LBRYPublisher.py index 2839e46cb..3e1b78325 100644 --- a/lbrynet/lbrynet_daemon/LBRYPublisher.py +++ b/lbrynet/lbrynet_daemon/LBRYPublisher.py @@ -10,6 +10,7 @@ from lbrynet.core.Error import InsufficientFundsError from lbrynet.lbryfilemanager.LBRYFileCreator import create_lbry_file from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob from lbrynet.core.PaymentRateManager import PaymentRateManager +from lbrynet.core.LBRYMetadata import Metadata, CURRENT_METADATA_VERSION from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader from lbrynet.conf import LOG_FILE_NAME from twisted.internet import threads, defer @@ -39,10 +40,10 @@ class Publisher(object): self.verified = False self.lbry_file = None self.txid = None - self.sources = {} - self.fee = None + self.stream_hash = None + self.metadata = {} - def start(self, name, file_path, bid, metadata, fee=None, sources={}): + def start(self, name, file_path, bid, metadata, old_txid): def _show_result(): log.info("Published %s --> lbry://%s txid: %s", self.file_name, self.publish_name, self.txid) @@ -51,8 +52,8 @@ class Publisher(object): self.publish_name = name self.file_path = file_path self.bid_amount = bid - self.fee = fee self.metadata = metadata + self.old_txid = old_txid d = self._check_file_path(self.file_path) d.addCallback(lambda _: create_lbry_file(self.session, self.lbry_file_manager, @@ -60,6 +61,7 @@ class Publisher(object): d.addCallback(self.add_to_lbry_files) d.addCallback(lambda _: self._create_sd_blob()) d.addCallback(lambda _: self._claim_name()) + d.addCallback(lambda _: self.set_status()) d.addCallbacks(lambda _: _show_result(), self._show_publish_error) return d @@ -72,26 +74,15 @@ class Publisher(object): return True return threads.deferToThread(check_file_threaded) - def _get_new_address(self): - d = self.wallet.get_new_address() - - def set_address(address): - self.key_fee_address = address - return True - - d.addCallback(set_address) - return d - - def set_status(self, lbry_file_downloader): + def set_lbry_file(self, lbry_file_downloader): self.lbry_file = lbry_file_downloader - d = self.lbry_file_manager.change_lbry_file_status(self.lbry_file, ManagedLBRYFileDownloader.STATUS_FINISHED) - d.addCallback(lambda _: lbry_file_downloader.restore()) - return d + return defer.succeed(None) def add_to_lbry_files(self, stream_hash): + self.stream_hash = stream_hash prm = PaymentRateManager(self.session.base_payment_rate_manager) d = self.lbry_file_manager.add_lbry_file(stream_hash, prm) - d.addCallback(self.set_status) + d.addCallback(self.set_lbry_file) return d def _create_sd_blob(self): @@ -99,19 +90,34 @@ class Publisher(object): self.lbry_file.stream_hash) def set_sd_hash(sd_hash): - self.sources['lbry_sd_hash'] = sd_hash + if 'sources' not in self.metadata: + self.metadata['sources'] = {} + self.metadata['sources']['lbry_sd_hash'] = sd_hash d.addCallback(set_sd_hash) return d + def set_status(self): + d = self.lbry_file_manager.change_lbry_file_status(self.lbry_file, ManagedLBRYFileDownloader.STATUS_FINISHED) + d.addCallback(lambda _: self.lbry_file.restore()) + return d + def _claim_name(self): self.metadata['content-type'] = mimetypes.guess_type(os.path.join(self.lbry_file.download_directory, self.lbry_file.file_name))[0] - d = self.wallet.claim_name(self.publish_name, - self.bid_amount, - self.sources, - self.metadata, - fee=self.fee) + self.metadata['ver'] = CURRENT_METADATA_VERSION + + if self.old_txid: + + d = self.wallet.abandon_name(self.old_txid) + d.addCallback(lambda tx: log.info("Abandoned tx %s" % str(tx))) + d.addCallback(lambda _: self.wallet.claim_name(self.publish_name, + self.bid_amount, + Metadata(self.metadata))) + else: + d = self.wallet.claim_name(self.publish_name, + self.bid_amount, + Metadata(self.metadata)) def set_tx_hash(txid): self.txid = txid diff --git a/requirements.txt b/requirements.txt index 780abee9d..800ecf2ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,3 +26,4 @@ unqlite==0.2.0 wsgiref==0.1.2 zope.interface==4.1.3 base58==0.2.2 +googlefinance==0.7 \ No newline at end of file diff --git a/setup.py b/setup.py index 60f800df2..847b69ee1 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ console_scripts = ['lbrynet-stdin-uploader = lbrynet.lbrynet_console.LBRYStdinUp requires = ['pycrypto', 'twisted', 'miniupnpc', 'yapsy', 'seccure', 'python-bitcoinrpc==0.1', 'txJSON-RPC', 'requests>=2.4.2', 'unqlite==0.2.0', - 'leveldb', 'lbryum', 'jsonrpc', 'simplejson', 'appdirs', 'six==1.9.0', 'base58'] + 'leveldb', 'lbryum', 'jsonrpc', 'simplejson', 'appdirs', 'six==1.9.0', 'base58', 'googlefinance'] setup(name='lbrynet', description='A decentralized media library and marketplace', diff --git a/tests/lbrynet/core/test_LBRYExchangeRateManager.py b/tests/lbrynet/core/test_LBRYExchangeRateManager.py new file mode 100644 index 000000000..2a6457536 --- /dev/null +++ b/tests/lbrynet/core/test_LBRYExchangeRateManager.py @@ -0,0 +1,38 @@ +import mock +from lbrynet.core import LBRYMetadata +from lbrynet.lbrynet_daemon import LBRYExchangeRateManager + +from twisted.trial import unittest + + +class LBRYFeeFormatTest(unittest.TestCase): + def test_fee_created_with_correct_inputs(self): + fee_dict = { + 'USD': { + 'amount': 10.0, + 'address': "bRcHraa8bYJZL7vkh5sNmGwPDERFUjGPP9" + } + } + fee = LBRYMetadata.LBRYFeeValidator(fee_dict) + self.assertEqual(10.0, fee['USD']['amount']) + + +class LBRYFeeTest(unittest.TestCase): + def setUp(self): + self.patcher = mock.patch('time.time') + self.time = self.patcher.start() + self.time.return_value = 0 + + def tearDown(self): + self.time.stop() + + def test_fee_converts_to_lbc(self): + fee_dict = { + 'USD': { + 'amount': 10.0, + 'address': "bRcHraa8bYJZL7vkh5sNmGwPDERFUjGPP9" + } + } + rates = {'BTCLBC': {'spot': 3.0, 'ts': 2}, 'USDBTC': {'spot': 2.0, 'ts': 3}} + manager = LBRYExchangeRateManager.DummyExchangeRateManager(rates) + self.assertEqual(60.0, manager.to_lbc(fee_dict).amount) \ No newline at end of file diff --git a/tests/lbrynet/core/test_LBRYMetadata.py b/tests/lbrynet/core/test_LBRYMetadata.py new file mode 100644 index 000000000..e5c3255b3 --- /dev/null +++ b/tests/lbrynet/core/test_LBRYMetadata.py @@ -0,0 +1,129 @@ +from lbrynet.core import LBRYMetadata +from twisted.trial import unittest + + +class MetadataTest(unittest.TestCase): + def test_assertion_if_source_is_missing(self): + metadata = {} + with self.assertRaises(AssertionError): + LBRYMetadata.Metadata(metadata) + + def test_metadata_works_without_fee(self): + metadata = { + 'license': 'Oscilloscope Laboratories', + 'description': 'Four couples meet for Sunday brunch only to discover they are stuck in a house together as the world may be about to end.', + 'language': 'en', + 'title': "It's a Disaster", + 'author': 'Written and directed by Todd Berger', + 'sources': { + 'lbry_sd_hash': '8d0d6ea64d09f5aa90faf5807d8a761c32a27047861e06f81f41e35623a348a4b0104052161d5f89cf190f9672bc4ead'}, + 'content-type': 'audio/mpeg', + 'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg' + } + m = LBRYMetadata.Metadata(metadata) + self.assertFalse('key' in m) + + def test_assertion_if_invalid_source(self): + metadata = { + 'license': 'Oscilloscope Laboratories', + 'fee': {'LBC': {'amount': 50.0, 'address': 'bRQJASJrDbFZVAvcpv3NoNWoH74LQd5JNV'}}, + 'description': 'Four couples meet for Sunday brunch only to discover they are stuck in a house together as the world may be about to end.', + 'language': 'en', + 'title': "It's a Disaster", + 'author': 'Written and directed by Todd Berger', + 'sources': { + 'fake': 'source'}, + 'content-type': 'audio/mpeg', + 'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg' + } + with self.assertRaises(AssertionError): + LBRYMetadata.Metadata(metadata) + + def test_assertion_if_missing_v001_field(self): + metadata = { + 'license': 'Oscilloscope Laboratories', + 'fee': {'LBC': {'amount': 50.0, 'address': 'bRQJASJrDbFZVAvcpv3NoNWoH74LQd5JNV'}}, + 'description': 'Four couples meet for Sunday brunch only to discover they are stuck in a house together as the world may be about to end.', + 'language': 'en', + 'author': 'Written and directed by Todd Berger', + 'sources': { + 'lbry_sd_hash': '8d0d6ea64d09f5aa90faf5807d8a761c32a27047861e06f81f41e35623a348a4b0104052161d5f89cf190f9672bc4ead'}, + 'content-type': 'audio/mpeg', + 'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg' + } + with self.assertRaises(AssertionError): + LBRYMetadata.Metadata(metadata) + + def test_version_is_001_if_all_fields_are_present(self): + metadata = { + 'license': 'Oscilloscope Laboratories', + 'fee': {'LBC': {'amount': 50.0, 'address': 'bRQJASJrDbFZVAvcpv3NoNWoH74LQd5JNV'}}, + 'description': 'Four couples meet for Sunday brunch only to discover they are stuck in a house together as the world may be about to end.', + 'language': 'en', + 'title': "It's a Disaster", + 'author': 'Written and directed by Todd Berger', + 'sources': { + 'lbry_sd_hash': '8d0d6ea64d09f5aa90faf5807d8a761c32a27047861e06f81f41e35623a348a4b0104052161d5f89cf190f9672bc4ead'}, + 'content-type': 'audio/mpeg', + 'thumbnail': 'http://ia.media-imdb.com/images/M/MV5BMTQwNjYzMTQ0Ml5BMl5BanBnXkFtZTcwNDUzODM5Nw@@._V1_SY1000_CR0,0,673,1000_AL_.jpg' + } + m = LBRYMetadata.Metadata(metadata) + self.assertEquals('0.0.1', m.meta_version) + + def test_assertion_if_there_is_an_extra_field(self): + metadata = { + 'license': 'NASA', + 'fee': {'USD': {'amount': 0.01, 'address': 'baBYSK7CqGSn5KrEmNmmQwAhBSFgo6v47z'}}, + 'ver': '0.0.2', + 'description': 'SDO captures images of the sun in 10 different wavelengths, each of which helps highlight a different temperature of solar material. Different temperatures can, in turn, show specific structures on the sun such as solar flares, which are gigantic explosions of light and x-rays, or coronal loops, which are stream of solar material travelling up and down looping magnetic field lines', + 'language': 'en', + 'author': 'The SDO Team, Genna Duberstein and Scott Wiessinger', + 'title': 'Thermonuclear Art', + 'sources': { + 'lbry_sd_hash': '8655f713819344980a9a0d67b198344e2c462c90f813e86f0c63789ab0868031f25c54d0bb31af6658e997e2041806eb'}, + 'nsfw': False, + 'content-type': 'video/mp4', + 'thumbnail': 'https://svs.gsfc.nasa.gov/vis/a010000/a012000/a012034/Combined.00_08_16_17.Still004.jpg', + 'MYSTERYFIELD': '?' + } + with self.assertRaises(AssertionError): + LBRYMetadata.Metadata(metadata) + + def test_version_is_002_if_all_fields_are_present(self): + metadata = { + 'license': 'NASA', + 'fee': {'USD': {'amount': 0.01, 'address': 'baBYSK7CqGSn5KrEmNmmQwAhBSFgo6v47z'}}, + 'ver': '0.0.2', + 'description': 'SDO captures images of the sun in 10 different wavelengths, each of which helps highlight a different temperature of solar material. Different temperatures can, in turn, show specific structures on the sun such as solar flares, which are gigantic explosions of light and x-rays, or coronal loops, which are stream of solar material travelling up and down looping magnetic field lines', + 'language': 'en', + 'author': 'The SDO Team, Genna Duberstein and Scott Wiessinger', + 'title': 'Thermonuclear Art', + 'sources': { + 'lbry_sd_hash': '8655f713819344980a9a0d67b198344e2c462c90f813e86f0c63789ab0868031f25c54d0bb31af6658e997e2041806eb'}, + 'nsfw': False, + 'content-type': 'video/mp4', + 'thumbnail': 'https://svs.gsfc.nasa.gov/vis/a010000/a012000/a012034/Combined.00_08_16_17.Still004.jpg' + } + m = LBRYMetadata.Metadata(metadata) + self.assertEquals('0.0.2', m.meta_version) + + def test_version_claimed_is_001_but_version_is_002(self): + metadata = { + 'license': 'NASA', + 'fee': {'USD': {'amount': 0.01, 'address': 'baBYSK7CqGSn5KrEmNmmQwAhBSFgo6v47z'}}, + 'ver': '0.0.1', + 'description': 'SDO captures images of the sun in 10 different wavelengths, each of which helps highlight a different temperature of solar material. Different temperatures can, in turn, show specific structures on the sun such as solar flares, which are gigantic explosions of light and x-rays, or coronal loops, which are stream of solar material travelling up and down looping magnetic field lines', + 'language': 'en', + 'author': 'The SDO Team, Genna Duberstein and Scott Wiessinger', + 'title': 'Thermonuclear Art', + 'sources': { + 'lbry_sd_hash': '8655f713819344980a9a0d67b198344e2c462c90f813e86f0c63789ab0868031f25c54d0bb31af6658e997e2041806eb'}, + 'nsfw': False, + 'content-type': 'video/mp4', + 'thumbnail': 'https://svs.gsfc.nasa.gov/vis/a010000/a012000/a012034/Combined.00_08_16_17.Still004.jpg' + } + with self.assertRaises(AssertionError): + LBRYMetadata.Metadata(metadata) + + +