Madiator2011/lbry-sdk
1
0
Fork 0
forked from LBRYCommunity/lbry-sdk
Code Pull requests Activity

Merge pull request #2 from lbryio/master

Browse source 
Update from base
This commit is contained in:
Dave-A 2016-08-15 20:31:31 -04:00 • committed by GitHub
commit 208610b645
32 changed files with 834 additions and 454 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.bumpversion.cfg
View file

@ -1,5 +1,5 @@
[bumpversion] [bumpversion]
current_version = 0.3.12 current_version = 0.3.17
commit = True commit = True
tag = True tag = True
message = Bump version: {current_version} -> {new_version} message = Bump version: {current_version} -> {new_version}

2
lbrynet/__init__.py
View file

@ -4,5 +4,5 @@ log = logging.getLogger(__name__)
logging.getLogger(__name__).addHandler(logging.NullHandler()) logging.getLogger(__name__).addHandler(logging.NullHandler())
log.setLevel(logging.INFO) log.setLevel(logging.INFO)
__version__ = "0.3.12" __version__ = "0.3.17"
version = tuple(__version__.split('.')) version = tuple(__version__.split('.'))

3
lbrynet/conf.py
View file

@ -48,6 +48,7 @@ DEFAULT_TIMEOUT = 30
DEFAULT_MAX_SEARCH_RESULTS = 25 DEFAULT_MAX_SEARCH_RESULTS = 25
DEFAULT_MAX_KEY_FEE = {'USD': {'amount': 25.0, 'address': ''}} DEFAULT_MAX_KEY_FEE = {'USD': {'amount': 25.0, 'address': ''}}
DEFAULT_SEARCH_TIMEOUT = 3.0 DEFAULT_SEARCH_TIMEOUT = 3.0
DEFAULT_SD_DOWNLOAD_TIMEOUT = 3
DEFAULT_CACHE_TIME = 3600 DEFAULT_CACHE_TIME = 3600
DEFAULT_UI_BRANCH = "master" DEFAULT_UI_BRANCH = "master"
@ -57,3 +58,5 @@ CURRENCIES = {
'LBC': {'type': 'crypto'}, 'LBC': {'type': 'crypto'},
'USD': {'type': 'fiat'}, 'USD': {'type': 'fiat'},
} }
LOGGLY_TOKEN = 'YWRmNGU4NmEtNjkwNC00YjM2LTk3ZjItMGZhODM3ZDhkYzBi'

13
lbrynet/core/LBRYMetadata.py
View file

@ -1,9 +1,6 @@
import requests
import json import json
import time
from copy import deepcopy from copy import deepcopy
from googlefinance import getQuotes
from lbrynet.conf import CURRENCIES from lbrynet.conf import CURRENCIES
from lbrynet.core import utils from lbrynet.core import utils
import logging import logging
@ -90,6 +87,10 @@ class LBRYFeeValidator(dict):
class Metadata(dict): class Metadata(dict):
@classmethod
def load_from_hex(cls, metadata):
return cls(json.loads(metadata.decode('hex')))
def __init__(self, metadata): def __init__(self, metadata):
dict.__init__(self) dict.__init__(self)
self.meta_version = None self.meta_version = None
@ -126,3 +127,9 @@ class Metadata(dict):
assert self.meta_version == self['ver'], "version mismatch" assert self.meta_version == self['ver'], "version mismatch"
break break
assert metadata == {}, "Unknown metadata keys: %s" % json.dumps(metadata.keys()) assert metadata == {}, "Unknown metadata keys: %s" % json.dumps(metadata.keys())
def serialize(self):
return json.dumps(self).encode("hex")
def as_json(self):
return json.dumps(self)

252
lbrynet/core/LBRYWallet.py
View file

@ -6,7 +6,6 @@ import subprocess
import socket import socket
import time import time
import os import os
import requests
from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException
from twisted.internet import threads, reactor, defer, task from twisted.internet import threads, reactor, defer, task
@ -15,10 +14,9 @@ from twisted.enterprise import adbapi
from collections import defaultdict, deque from collections import defaultdict, deque
from zope.interface import implements from zope.interface import implements
from decimal import Decimal from decimal import Decimal
from googlefinance import getQuotes
from lbryum import SimpleConfig, Network from lbryum import SimpleConfig, Network
from lbryum.lbrycrd import COIN, TYPE_ADDRESS from lbryum.lbrycrd import COIN
from lbryum.wallet import WalletStorage, Wallet from lbryum.wallet import WalletStorage, Wallet
from lbryum.commands import known_commands, Commands from lbryum.commands import known_commands, Commands
from lbryum.transaction import Transaction from lbryum.transaction import Transaction
@ -27,8 +25,6 @@ from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHand
from lbrynet.core.client.ClientRequest import ClientRequest from lbrynet.core.client.ClientRequest import ClientRequest
from lbrynet.core.Error import UnknownNameError, InvalidStreamInfoError, RequestCanceledError from lbrynet.core.Error import UnknownNameError, InvalidStreamInfoError, RequestCanceledError
from lbrynet.core.Error import InsufficientFundsError from lbrynet.core.Error import InsufficientFundsError
from lbrynet.core.sqlite_helpers import rerun_if_locked
from lbrynet.conf import SOURCE_TYPES
from lbrynet.core.LBRYMetadata import Metadata from lbrynet.core.LBRYMetadata import Metadata
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -90,6 +86,7 @@ class LBRYWallet(object):
return True return True
d = self._open_db() d = self._open_db()
d.addCallback(lambda _: self._clean_bad_records())
d.addCallback(lambda _: self._start()) d.addCallback(lambda _: self._start())
d.addCallback(lambda _: start_manage()) d.addCallback(lambda _: start_manage())
return d return d
@ -324,6 +321,10 @@ class LBRYWallet(object):
for k in ['value', 'txid', 'n', 'height', 'amount']: for k in ['value', 'txid', 'n', 'height', 'amount']:
assert k in r, "getvalueforname response missing field %s" % k assert k in r, "getvalueforname response missing field %s" % k
def _log_success(claim_id):
log.info("lbry://%s complies with %s, claimid: %s", name, metadata.meta_version, claim_id)
return defer.succeed(None)
if 'error' in result: if 'error' in result:
log.warning("Got an error looking up a name: %s", result['error']) log.warning("Got an error looking up a name: %s", result['error'])
return Failure(UnknownNameError(name)) return Failure(UnknownNameError(name))
@ -335,55 +336,116 @@ class LBRYWallet(object):
except (ValueError, TypeError): except (ValueError, TypeError):
return Failure(InvalidStreamInfoError(name)) return Failure(InvalidStreamInfoError(name))
d = self._save_name_metadata(name, str(result['txid']), metadata['sources']['lbry_sd_hash']) txid = result['txid']
d.addCallback(lambda _: log.info("lbry://%s complies with %s" % (name, metadata.meta_version))) sd_hash = metadata['sources']['lbry_sd_hash']
d = self._save_name_metadata(name, txid, sd_hash)
d.addCallback(lambda _: self.get_claimid(name, txid))
d.addCallback(lambda cid: _log_success(cid))
d.addCallback(lambda _: metadata) d.addCallback(lambda _: metadata)
return d return d
def _get_claim_info(self, result, name): def get_claim(self, name, claim_id):
def _check_result_fields(r): d = self.get_claims_for_name(name)
for k in ['value', 'txid', 'n', 'height', 'amount']: d.addCallback(lambda claims: next(claim for claim in claims['claims'] if claim['claimId'] == claim_id))
assert k in r, "getvalueforname response missing field %s" % k
def _build_response(m, result):
result['value'] = m
return result
if 'error' in result:
log.warning("Got an error looking up a name: %s", result['error'])
return Failure(UnknownNameError(name))
_check_result_fields(result)
try:
metadata = Metadata(json.loads(result['value']))
except (ValueError, TypeError):
return Failure(InvalidStreamInfoError(name))
d = self._save_name_metadata(name, str(result['txid']), metadata['sources']['lbry_sd_hash'])
d.addCallback(lambda _: log.info("lbry://%s complies with %s" % (name, metadata.meta_version)))
d.addCallback(lambda _: _build_response(metadata, result))
return d return d
def get_claim_info(self, name): def get_claimid(self, name, txid):
def _get_id_for_return(claim_id):
if claim_id:
return defer.succeed(claim_id)
else:
d = self.get_claims_from_tx(txid)
d.addCallback(lambda claims: next(c['claimId'] for c in claims if c['name'] == name))
d.addCallback(lambda cid: self._update_claimid(cid, name, txid))
return d
d = self._get_claimid_for_tx(name, txid)
d.addCallback(_get_id_for_return)
return d
def get_claim_info(self, name, txid=None):
if not txid:
d = self._get_value_for_name(name) d = self._get_value_for_name(name)
d.addCallback(lambda r: self._get_claim_info(r, name)) d.addCallback(lambda r: self._get_claim_info(name, r['txid']))
else:
d = self._get_claim_info(name, txid)
d.addErrback(lambda _: False)
return d return d
def _get_claim_info(self, name, txid):
def _build_response(claim):
result = {}
try:
metadata = Metadata(json.loads(claim['value']))
meta_ver = metadata.meta_version
sd_hash = metadata['sources']['lbry_sd_hash']
d = self._save_name_metadata(name, txid, sd_hash)
except AssertionError:
metadata = claim['value']
meta_ver = "Non-compliant"
d = defer.succeed(None)
claim_id = claim['claimId']
result['claim_id'] = claim_id
result['amount'] = claim['nEffectiveAmount']
result['height'] = claim['nHeight']
result['name'] = name
result['txid'] = txid
result['value'] = metadata
result['supports'] = [{'txid': support['txid'], 'n': support['n']} for support in claim['supports']]
result['meta_version'] = meta_ver
log.info("get claim info lbry://%s metadata: %s, claimid: %s", name, meta_ver, claim_id)
d.addCallback(lambda _: self.get_name_claims())
d.addCallback(lambda r: [c['txid'] for c in r])
d.addCallback(lambda my_claims: _add_is_mine(result, my_claims))
return d
def _add_is_mine(response, my_txs):
response['is_mine'] = response['txid'] in my_txs
return response
d = self.get_claimid(name, txid)
d.addCallback(lambda claim_id: self.get_claim(name, claim_id))
d.addCallback(_build_response)
return d
def get_claims_for_name(self, name):
d = self._get_claims_for_name(name)
return d
def update_metadata(self, new_metadata, old_metadata):
meta_for_return = old_metadata if isinstance(old_metadata, dict) else {}
for k in new_metadata:
meta_for_return[k] = new_metadata[k]
return defer.succeed(Metadata(meta_for_return))
def claim_name(self, name, bid, m): def claim_name(self, name, bid, m):
def _save_metadata(txid, metadata):
metadata = Metadata(m)
d = self._send_name_claim(name, json.dumps(metadata), bid)
def _save_metadata(txid):
log.info("Saving metadata for claim %s" % txid) log.info("Saving metadata for claim %s" % txid)
d = self._save_name_metadata(name, txid, metadata['sources']['lbry_sd_hash']) d = self._save_name_metadata(name, txid, metadata['sources']['lbry_sd_hash'])
d.addCallback(lambda _: txid) d.addCallback(lambda _: txid)
return d return d
d.addCallback(_save_metadata) def _claim_or_update(claim, metadata, _bid):
if not claim:
log.info("No claim yet, making a new one")
return self._send_name_claim(name, metadata.as_json(), _bid)
if not claim['is_mine']:
log.info("Making a contesting claim")
return self._send_name_claim(name, metadata.as_json(), _bid)
else:
log.info("Updating over own claim")
d = self.update_metadata(metadata, claim['value'])
d.addCallback(lambda new_metadata: self._send_name_claim_update(name, claim['claim_id'], claim['txid'], new_metadata, _bid))
return d
meta = Metadata(m)
d = self.get_claim_info(name)
d.addCallback(lambda claim: _claim_or_update(claim, meta, bid))
d.addCallback(lambda txid: _save_metadata(txid, meta))
return d return d
def abandon_name(self, txid): def abandon_name(self, txid):
@ -419,19 +481,14 @@ class LBRYWallet(object):
dl.addCallback(abandon) dl.addCallback(abandon)
return dl return dl
def support_claim(self, name, claim_id, amount):
return self._support_claim(name, claim_id, amount)
def get_tx(self, txid): def get_tx(self, txid):
d = self._get_raw_tx(txid) d = self._get_raw_tx(txid)
d.addCallback(self._get_decoded_tx) d.addCallback(self._get_decoded_tx)
return d return d
def update_name(self, name, bid, value, old_txid):
d = self._get_value_for_name(name)
d.addCallback(lambda r: self.abandon_name(r['txid'] if not old_txid else old_txid))
d.addCallback(lambda r: log.info("Abandon claim tx %s" % str(r)))
d.addCallback(lambda _: self.claim_name(name, bid, value))
return d
def get_name_and_validity_for_sd_hash(self, sd_hash): def get_name_and_validity_for_sd_hash(self, sd_hash):
d = self._get_claim_metadata_for_sd_hash(sd_hash) d = self._get_claim_metadata_for_sd_hash(sd_hash)
d.addCallback(lambda name_txid: self._get_status_of_claim(name_txid[1], name_txid[0], sd_hash) if name_txid is not None else None) d.addCallback(lambda name_txid: self._get_status_of_claim(name_txid[1], name_txid[0], sd_hash) if name_txid is not None else None)
@ -534,21 +591,45 @@ class LBRYWallet(object):
def _open_db(self): def _open_db(self):
self.db = adbapi.ConnectionPool('sqlite3', os.path.join(self.db_dir, "blockchainname.db"), self.db = adbapi.ConnectionPool('sqlite3', os.path.join(self.db_dir, "blockchainname.db"),
check_same_thread=False) check_same_thread=False)
return self.db.runQuery("create table if not exists name_metadata (" +
def create_tables(transaction):
transaction.execute("create table if not exists name_metadata (" +
" name text, " + " name text, " +
" txid text, " + " txid text, " +
" sd_hash text)") " sd_hash text)")
transaction.execute("create table if not exists claim_ids (" +
" claimId text, " +
" name text, " +
" txid text)")
return self.db.runInteraction(create_tables)
def _clean_bad_records(self):
d = self.db.runQuery("delete from name_metadata where length(txid) > 64 or txid is null")
return d
def _save_name_metadata(self, name, txid, sd_hash): def _save_name_metadata(self, name, txid, sd_hash):
d = self.db.runQuery("select * from name_metadata where name=? and txid=? and sd_hash=?", (name, txid, sd_hash)) assert len(txid) == 64, "That's not a txid: %s" % str(txid)
d.addCallback(lambda r: self.db.runQuery("insert into name_metadata values (?, ?, ?)", (name, txid, sd_hash)) d = self.db.runQuery("delete from name_metadata where name=? and txid=? and sd_hash=?", (name, txid, sd_hash))
if not len(r) else None) d.addCallback(lambda _: self.db.runQuery("insert into name_metadata values (?, ?, ?)", (name, txid, sd_hash)))
return d return d
def _get_claim_metadata_for_sd_hash(self, sd_hash): def _get_claim_metadata_for_sd_hash(self, sd_hash):
d = self.db.runQuery("select name, txid from name_metadata where sd_hash=?", (sd_hash,)) d = self.db.runQuery("select name, txid from name_metadata where sd_hash=?", (sd_hash,))
d.addCallback(lambda r: r[0] if len(r) else None) d.addCallback(lambda r: r[0] if r else None)
return d
def _update_claimid(self, claim_id, name, txid):
assert len(txid) == 64, "That's not a txid: %s" % str(txid)
d = self.db.runQuery("delete from claim_ids where claimId=? and name=? and txid=?", (claim_id, name, txid))
d.addCallback(lambda r: self.db.runQuery("insert into claim_ids values (?, ?, ?)", (claim_id, name, txid)))
d.addCallback(lambda _: claim_id)
return d
def _get_claimid_for_tx(self, name, txid):
assert len(txid) == 64, "That's not a txid: %s" % str(txid)
d = self.db.runQuery("select claimId from claim_ids where name=? and txid=?", (name, txid))
d.addCallback(lambda r: r[0][0] if r else None)
return d return d
######### Must be overridden ######### ######### Must be overridden #########
@ -571,6 +652,9 @@ class LBRYWallet(object):
def get_name_claims(self): def get_name_claims(self):
return defer.fail(NotImplementedError()) return defer.fail(NotImplementedError())
def _get_claims_for_name(self, name):
return defer.fail(NotImplementedError())
def _check_first_run(self): def _check_first_run(self):
return defer.fail(NotImplementedError()) return defer.fail(NotImplementedError())
@ -586,7 +670,10 @@ class LBRYWallet(object):
def _send_abandon(self, txid, address, amount): def _send_abandon(self, txid, address, amount):
return defer.fail(NotImplementedError()) return defer.fail(NotImplementedError())
def _update_name(self, txid, value, amount): def _send_name_claim_update(self, name, claim_id, txid, value, amount):
return defer.fail(NotImplementedError())
def _support_claim(self, name, claim_id, amount):
return defer.fail(NotImplementedError()) return defer.fail(NotImplementedError())
def _do_send_many(self, payments_to_send): def _do_send_many(self, payments_to_send):
@ -721,9 +808,15 @@ class LBRYcrdWallet(LBRYWallet):
def _send_abandon(self, txid, address, amount): def _send_abandon(self, txid, address, amount):
return threads.deferToThread(self._send_abandon_rpc, txid, address, amount) return threads.deferToThread(self._send_abandon_rpc, txid, address, amount)
def _update_name(self, txid, value, amount): def _send_name_claim_update(self, name, claim_id, txid, value, amount):
return threads.deferToThread(self._update_name_rpc, txid, value, amount) return threads.deferToThread(self._update_name_rpc, txid, value, amount)
def _support_claim(self, name, claim_id, amount):
return threads.deferToThread(self._support_claim_rpc, name, claim_id, amount)
def _get_claims_for_name(self, name):
return threads.deferToThread(self._get_claims_for_name_rpc, name)
def get_claims_from_tx(self, txid): def get_claims_from_tx(self, txid):
return threads.deferToThread(self._get_claims_from_tx_rpc, txid) return threads.deferToThread(self._get_claims_from_tx_rpc, txid)
@ -858,6 +951,11 @@ class LBRYcrdWallet(LBRYWallet):
rpc_conn = self._get_rpc_conn() rpc_conn = self._get_rpc_conn()
return rpc_conn.getclaimsfortx(txid) return rpc_conn.getclaimsfortx(txid)
@_catch_connection_error
def _get_claims_for_name_rpc(self, name):
rpc_conn = self._get_rpc_conn()
return rpc_conn.getclaimsforname(name)
@_catch_connection_error @_catch_connection_error
def _get_nametrie_rpc(self): def _get_nametrie_rpc(self):
rpc_conn = self._get_rpc_conn() rpc_conn = self._get_rpc_conn()
@ -878,6 +976,7 @@ class LBRYcrdWallet(LBRYWallet):
rpc_conn = self._get_rpc_conn() rpc_conn = self._get_rpc_conn()
return rpc_conn.getvalueforname(name) return rpc_conn.getvalueforname(name)
@_catch_connection_error
def _update_name_rpc(self, txid, value, amount): def _update_name_rpc(self, txid, value, amount):
rpc_conn = self._get_rpc_conn() rpc_conn = self._get_rpc_conn()
return rpc_conn.updateclaim(txid, value, amount) return rpc_conn.updateclaim(txid, value, amount)
@ -893,6 +992,11 @@ class LBRYcrdWallet(LBRYWallet):
elif 'message' in e.error: elif 'message' in e.error:
raise ValueError(e.error['message']) raise ValueError(e.error['message'])
@_catch_connection_error
def _support_claim_rpc(self, name, claim_id, amount):
rpc_conn = self._get_rpc_conn()
return rpc_conn.supportclaim(name, claim_id, amount)
@_catch_connection_error @_catch_connection_error
def _get_num_addresses_rpc(self): def _get_num_addresses_rpc(self):
rpc_conn = self._get_rpc_conn() rpc_conn = self._get_rpc_conn()
@ -1106,6 +1210,25 @@ class LBRYumWallet(LBRYWallet):
d.addCallback(self._broadcast_transaction) d.addCallback(self._broadcast_transaction)
return d return d
def _get_claims_for_name(self, name):
cmd = known_commands['getclaimsforname']
func = getattr(self.cmd_runner, cmd.name)
return threads.deferToThread(func, name)
def _send_name_claim_update(self, name, claim_id, txid, value, amount):
def send_claim_update(address):
decoded_claim_id = claim_id.decode('hex')[::-1]
metadata = Metadata(value).as_json()
log.info("updateclaim %s %s %f %s %s '%s'", txid, address, amount, name, decoded_claim_id.encode('hex'), json.dumps(metadata))
cmd = known_commands['updateclaim']
func = getattr(self.cmd_runner, cmd.name)
return threads.deferToThread(func, txid, address, amount, name, decoded_claim_id, metadata)
d = self.get_new_address()
d.addCallback(send_claim_update)
d.addCallback(self._broadcast_transaction)
return d
def _get_decoded_tx(self, raw_tx): def _get_decoded_tx(self, raw_tx):
tx = Transaction(raw_tx) tx = Transaction(raw_tx)
decoded_tx = {} decoded_tx = {}
@ -1117,18 +1240,33 @@ class LBRYumWallet(LBRYWallet):
return decoded_tx return decoded_tx
def _send_abandon(self, txid, address, amount): def _send_abandon(self, txid, address, amount):
log.info("Abandon " + str(txid) + " " + str(address) + " " + str(amount)) log.info("Abandon %s %s %f" % (txid, address, amount))
cmd = known_commands['abandonclaim'] cmd = known_commands['abandonclaim']
func = getattr(self.cmd_runner, cmd.name) func = getattr(self.cmd_runner, cmd.name)
d = threads.deferToThread(func, txid, address, amount) d = threads.deferToThread(func, txid, address, amount)
d.addCallback(self._broadcast_transaction) d.addCallback(self._broadcast_transaction)
return d return d
def _support_claim(self, name, claim_id, amount):
def _send_support(d, a, n, c):
cmd = known_commands['supportclaim']
func = getattr(self.cmd_runner, cmd.name)
d = threads.deferToThread(func, d, a, n, c)
return d
d = self.get_new_address()
d.addCallback(lambda address: _send_support(address, amount, name, claim_id))
d.addCallback(self._broadcast_transaction)
return d
def _broadcast_transaction(self, raw_tx): def _broadcast_transaction(self, raw_tx):
log.info("Broadcast: " + str(raw_tx)) def _log_tx(r):
log.info("Broadcast tx: %s", r)
return r
cmd = known_commands['broadcast'] cmd = known_commands['broadcast']
func = getattr(self.cmd_runner, cmd.name) func = getattr(self.cmd_runner, cmd.name)
d = threads.deferToThread(func, raw_tx) d = threads.deferToThread(func, raw_tx)
d.addCallback(_log_tx)
d.addCallback(lambda r: r if len(r) == 64 else defer.fail(Exception("Transaction rejected")))
d.addCallback(self._save_wallet) d.addCallback(self._save_wallet)
return d return d

9
lbrynet/core/Session.py
View file

@ -173,14 +173,23 @@ class LBRYSession(object):
self.upnp_redirects.append((self.peer_port, 'TCP')) self.upnp_redirects.append((self.peer_port, 'TCP'))
log.info("Set UPnP redirect for TCP port %d", self.peer_port) log.info("Set UPnP redirect for TCP port %d", self.peer_port)
else: else:
# see comment below
log.warning("UPnP redirect already set for TCP port %d", self.peer_port) log.warning("UPnP redirect already set for TCP port %d", self.peer_port)
self.upnp_redirects.append((self.peer_port, 'TCP'))
if self.dht_node_port is not None: if self.dht_node_port is not None:
if u.getspecificportmapping(self.dht_node_port, 'UDP') is None: if u.getspecificportmapping(self.dht_node_port, 'UDP') is None:
u.addportmapping(self.dht_node_port, 'UDP', u.lanaddr, self.dht_node_port, 'LBRY DHT port', '') u.addportmapping(self.dht_node_port, 'UDP', u.lanaddr, self.dht_node_port, 'LBRY DHT port', '')
self.upnp_redirects.append((self.dht_node_port, 'UDP')) self.upnp_redirects.append((self.dht_node_port, 'UDP'))
log.info("Set UPnP redirect for UPD port %d", self.dht_node_port) log.info("Set UPnP redirect for UPD port %d", self.dht_node_port)
else: else:
# TODO: check that the existing redirect was put up by an old lbrynet session before grabbing it
# if such a disconnected redirect exists, then upnp won't work unless the redirect is appended
# or is torn down and set back up. a bad shutdown of lbrynet could leave such a redirect up
# and cause problems on the next start.
# this could be problematic if a previous lbrynet session didn't make the redirect, and it was
# made by another application
log.warning("UPnP redirect already set for UDP port %d", self.dht_node_port) log.warning("UPnP redirect already set for UDP port %d", self.dht_node_port)
self.upnp_redirects.append((self.dht_node_port, 'UDP'))
return True return True
return False return False

103
lbrynet/core/log_support.py
View file

@ -1,24 +1,111 @@
import base64
import json
import logging import logging
import logging.handlers import logging.handlers
import sys import sys
import traceback
import lbrynet
from lbrynet import conf
from requests_futures.sessions import FuturesSession
session = FuturesSession()
def bg_cb(sess, resp):
""" Don't do anything with the response """
pass
class HTTPSHandler(logging.Handler):
def __init__(self, url, fqdn=False, localname=None, facility=None):
logging.Handler.__init__(self)
self.url = url
self.fqdn = fqdn
self.localname = localname
self.facility = facility
def get_full_message(self, record):
if record.exc_info:
return '\n'.join(traceback.format_exception(*record.exc_info))
else:
return record.getMessage()
def emit(self, record):
try:
payload = self.format(record)
session.post(self.url, data=payload, background_callback=bg_cb)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
DEFAULT_FORMAT = "%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s" DEFAULT_FORMAT = "%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s"
DEFAULT_FORMATTER = logging.Formatter(DEFAULT_FORMAT) DEFAULT_FORMATTER = logging.Formatter(DEFAULT_FORMAT)
LOGGLY_URL = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}"
def configureConsole(log=None, level=logging.INFO): def remove_handlers(log, handler_name):
for handler in log.handlers:
if handler.name == handler_name:
log.removeHandler(handler)
def _log_decorator(fn):
def helper(*args, **kwargs):
log = kwargs.pop('log', logging.getLogger())
level = kwargs.pop('level', logging.INFO)
handler = fn(*args, **kwargs)
if handler.name:
remove_handlers(log, handler.name)
log.addHandler(handler)
log.setLevel(level)
return helper
def disable_noisy_loggers():
logging.getLogger('requests').setLevel(logging.WARNING)
@_log_decorator
def configure_console(**kwargs):
"""Convenience function to configure a logger that outputs to stdout""" """Convenience function to configure a logger that outputs to stdout"""
log = log or logging.getLogger()
handler = logging.StreamHandler(sys.stdout) handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(DEFAULT_FORMATTER) handler.setFormatter(DEFAULT_FORMATTER)
log.addHandler(handler) handler.name = 'console'
log.setLevel(level=level) return handler
def configureFileHandler(file_name, log=None, level=logging.INFO): @_log_decorator
log = log or logging.getLogger() def configure_file_handler(file_name, **kwargs):
handler = logging.handlers.RotatingFileHandler(file_name, maxBytes=2097152, backupCount=5) handler = logging.handlers.RotatingFileHandler(file_name, maxBytes=2097152, backupCount=5)
handler.setFormatter(DEFAULT_FORMATTER) handler.setFormatter(DEFAULT_FORMATTER)
log.addHandler(handler) handler.name = 'file'
log.setLevel(level=level) return handler
def get_loggly_url(token=None, version=None):
token = token or base64.b64decode(conf.LOGGLY_TOKEN)
version = version or lbrynet.__version__
return LOGGLY_URL.format(token=token, tag='lbrynet-' + version)
@_log_decorator
def configure_loggly_handler(url=None, **kwargs):
url = url or get_loggly_url()
json_format = {
"loggerName": "%(name)s",
"asciTime": "%(asctime)s",
"fileName": "%(filename)s",
"functionName": "%(funcName)s",
"levelNo": "%(levelno)s",
"lineNo": "%(lineno)d",
"levelName": "%(levelname)s",
"message": "%(message)s",
}
json_format.update(kwargs)
formatter = logging.Formatter(json.dumps(json_format))
handler = HTTPSHandler(url)
handler.setFormatter(formatter)
handler.name = 'loggly'
return handler

2
lbrynet/core/server/DHTHashAnnouncer.py
View file

@ -1,5 +1,5 @@
import binascii import binascii
from twisted.internet import defer, task, reactor from twisted.internet import defer, reactor
import collections import collections

6
lbrynet/core/server/ServerRequestHandler.py
View file

@ -99,8 +99,8 @@ class ServerRequestHandler(object):
d.addCallback(lambda _: self.blob_sender.send_blob_if_requested(self)) d.addCallback(lambda _: self.blob_sender.send_blob_if_requested(self))
d.addCallbacks(lambda _: self.finished_response(), self.request_failure_handler) d.addCallbacks(lambda _: self.finished_response(), self.request_failure_handler)
else: else:
log.info("Request buff not a valid json message") log.debug("Request buff not a valid json message")
log.info("Request buff: %s", str(self.request_buff)) log.debug("Request buff: %s", str(self.request_buff))
else: else:
log.warning("The client sent data when we were uploading a file. This should not happen") log.warning("The client sent data when we were uploading a file. This should not happen")
@ -125,7 +125,7 @@ class ServerRequestHandler(object):
def send_response(self, msg): def send_response(self, msg):
m = json.dumps(msg) m = json.dumps(msg)
log.info("Sending a response of length %s", str(len(m))) log.debug("Sending a response of length %s", str(len(m)))
log.debug("Response: %s", str(m)) log.debug("Response: %s", str(m))
self.response_buff = self.response_buff + m self.response_buff = self.response_buff + m
self._produce_more() self._produce_more()

3
lbrynet/dht/datastore.py
View file

@ -8,10 +8,7 @@
# may be created by processing this file with epydoc: http://epydoc.sf.net # may be created by processing this file with epydoc: http://epydoc.sf.net
import UserDict import UserDict
#import sqlite3
import cPickle as pickle
import time import time
import os
import constants import constants

2
lbrynet/dht/node.py
View file

@ -7,7 +7,7 @@
# The docstrings in this module contain epytext markup; API documentation # The docstrings in this module contain epytext markup; API documentation
# may be created by processing this file with epydoc: http://epydoc.sf.net # may be created by processing this file with epydoc: http://epydoc.sf.net
import hashlib, random, struct, time, math, binascii import hashlib, random, struct, time, binascii
import argparse import argparse
from twisted.internet import defer, error from twisted.internet import defer, error
import constants import constants

4
lbrynet/dhttest.py
View file

@ -20,7 +20,7 @@
import os, sys, time, signal, hashlib, random import sys, hashlib, random
import twisted.internet.reactor import twisted.internet.reactor
from lbrynet.dht.node import Node from lbrynet.dht.node import Node
#from entangled.kademlia.datastore import SQLiteDataStore #from entangled.kademlia.datastore import SQLiteDataStore
@ -106,7 +106,7 @@ def stop():
if __name__ == '__main__': if __name__ == '__main__':
import sys, os import sys
if len(sys.argv) < 2: if len(sys.argv) < 2:
print 'Usage:\n%s UDP_PORT [KNOWN_NODE_IP KNOWN_NODE_PORT]' % sys.argv[0] print 'Usage:\n%s UDP_PORT [KNOWN_NODE_IP KNOWN_NODE_PORT]' % sys.argv[0]
print 'or:\n%s UDP_PORT [FILE_WITH_KNOWN_NODES]' % sys.argv[0] print 'or:\n%s UDP_PORT [FILE_WITH_KNOWN_NODES]' % sys.argv[0]

2
lbrynet/lbryfile/LBRYFileMetadataManager.py
View file

@ -48,7 +48,7 @@ class DBLBRYFileMetadataManager(object):
return self._add_blobs_to_stream(stream_hash, blobs, ignore_duplicate_error=True) return self._add_blobs_to_stream(stream_hash, blobs, ignore_duplicate_error=True)
def get_blobs_for_stream(self, stream_hash, start_blob=None, end_blob=None, count=None, reverse=False): def get_blobs_for_stream(self, stream_hash, start_blob=None, end_blob=None, count=None, reverse=False):
log.info("Getting blobs for a stream. Count is %s", str(count)) log.debug("Getting blobs for a stream. Count is %s", str(count))
def get_positions_of_start_and_end(): def get_positions_of_start_and_end():
if start_blob is not None: if start_blob is not None:

1
lbrynet/lbryfilemanager/LBRYFileCreator.py
View file

@ -11,7 +11,6 @@ from lbrynet import conf
from lbrynet.lbryfile.StreamDescriptor import get_sd_info from lbrynet.lbryfile.StreamDescriptor import get_sd_info
from lbrynet.core.cryptoutils import get_lbry_hash_obj from lbrynet.core.cryptoutils import get_lbry_hash_obj
from twisted.protocols.basic import FileSender from twisted.protocols.basic import FileSender
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader
log = logging.getLogger(__name__) log = logging.getLogger(__name__)

9
lbrynet/lbryfilemanager/LBRYFileDownloader.py
View file

@ -27,6 +27,7 @@ class ManagedLBRYFileDownloader(LBRYFileSaver):
self.sd_hash = None self.sd_hash = None
self.txid = None self.txid = None
self.uri = None self.uri = None
self.claim_id = None
self.rowid = rowid self.rowid = rowid
self.lbry_file_manager = lbry_file_manager self.lbry_file_manager = lbry_file_manager
self.saving_status = False self.saving_status = False
@ -43,10 +44,16 @@ class ManagedLBRYFileDownloader(LBRYFileSaver):
return d return d
def _save_claim_id(claim_id):
self.claim_id = claim_id
return defer.succeed(None)
def _save_claim(name, txid): def _save_claim(name, txid):
self.uri = name self.uri = name
self.txid = txid self.txid = txid
return defer.succeed(None) d = self.wallet.get_claimid(name, txid)
d.addCallback(_save_claim_id)
return d
d.addCallback(_save_sd_hash) d.addCallback(_save_sd_hash)
d.addCallback(lambda r: _save_claim(r[0], r[1]) if r else None) d.addCallback(lambda r: _save_claim(r[0], r[1]) if r else None)

7
lbrynet/lbryfilemanager/LBRYFileManager.py
View file

@ -80,13 +80,16 @@ class LBRYFileManager(object):
d.addCallback(lambda downloader: downloader.restore()) d.addCallback(lambda downloader: downloader.restore())
return d return d
def log_error(err): def log_error(err, rowid, stream_hash, options):
log.error("An error occurred while starting a lbry file: %s", err.getErrorMessage()) log.error("An error occurred while starting a lbry file: %s", err.getErrorMessage())
log.error(rowid)
log.error(stream_hash)
log.error(options)
def start_lbry_files(lbry_files_and_options): def start_lbry_files(lbry_files_and_options):
for rowid, stream_hash, options in lbry_files_and_options: for rowid, stream_hash, options in lbry_files_and_options:
d = set_options_and_restore(rowid, stream_hash, options) d = set_options_and_restore(rowid, stream_hash, options)
d.addErrback(log_error) d.addErrback(lambda err: log_error(err, rowid, stream_hash, options))
return True return True
d = self._get_all_lbry_files() d = self._get_all_lbry_files()

8
lbrynet/lbrynet_console/ControlHandlers.py
View file

@ -1,18 +1,12 @@
import json
import logging import logging
from time import sleep
from bitcoinrpc.authproxy import AuthServiceProxy
from twisted.internet.task import LoopingCall
from zope.interface import implements from zope.interface import implements
#from lbrynet.core.StreamDescriptor import PlainStreamDescriptorWriter, BlobStreamDescriptorWriter
from lbrynet.core.PaymentRateManager import PaymentRateManager from lbrynet.core.PaymentRateManager import PaymentRateManager
from lbrynet.lbryfilemanager.LBRYFileCreator import create_lbry_file from lbrynet.lbryfilemanager.LBRYFileCreator import create_lbry_file
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader
# from lbrynet.lbryfile.StreamDescriptor import get_sd_info
from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob, create_plain_sd from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob, create_plain_sd
from lbrynet.lbrynet_console.interfaces import ICommandHandler, ICommandHandlerFactory from lbrynet.lbrynet_console.interfaces import ICommandHandler, ICommandHandlerFactory
from lbrynet.core.StreamDescriptor import download_sd_blob#, BlobStreamDescriptorReader from lbrynet.core.StreamDescriptor import download_sd_blob
from lbrynet.core.Error import UnknownNameError, InvalidBlobHashError, InsufficientFundsError from lbrynet.core.Error import UnknownNameError, InvalidBlobHashError, InsufficientFundsError
from lbrynet.core.Error import InvalidStreamInfoError from lbrynet.core.Error import InvalidStreamInfoError
from lbrynet.core.utils import is_valid_blobhash from lbrynet.core.utils import is_valid_blobhash

706
lbrynet/lbrynet_daemon/LBRYDaemon.py
View file

@ -1,6 +1,4 @@
import binascii import binascii
import distutils.version
import locale
import logging.handlers import logging.handlers
import mimetypes import mimetypes
import os import os
@ -14,7 +12,6 @@ import sys
import base58 import base58
import requests import requests
import simplejson as json import simplejson as json
import pkg_resources
from urllib2 import urlopen from urllib2 import urlopen
from appdirs import user_data_dir from appdirs import user_data_dir
@ -25,7 +22,7 @@ from twisted.internet import defer, threads, error, reactor
from twisted.internet.task import LoopingCall from twisted.internet.task import LoopingCall
from txjsonrpc import jsonrpclib from txjsonrpc import jsonrpclib
from txjsonrpc.web import jsonrpc from txjsonrpc.web import jsonrpc
from txjsonrpc.web.jsonrpc import Handler, Proxy from txjsonrpc.web.jsonrpc import Handler
from lbrynet import __version__ as lbrynet_version from lbrynet import __version__ as lbrynet_version
from lbryum.version import LBRYUM_VERSION as lbryum_version from lbryum.version import LBRYUM_VERSION as lbryum_version
@ -41,15 +38,20 @@ from lbrynet.lbrynet_daemon.LBRYUIManager import LBRYUIManager
from lbrynet.lbrynet_daemon.LBRYDownloader import GetStream from lbrynet.lbrynet_daemon.LBRYDownloader import GetStream
from lbrynet.lbrynet_daemon.LBRYPublisher import Publisher from lbrynet.lbrynet_daemon.LBRYPublisher import Publisher
from lbrynet.lbrynet_daemon.LBRYExchangeRateManager import ExchangeRateManager from lbrynet.lbrynet_daemon.LBRYExchangeRateManager import ExchangeRateManager
from lbrynet.lbrynet_daemon.Lighthouse import LighthouseClient
from lbrynet.core.LBRYMetadata import Metadata
from lbrynet.core import log_support
from lbrynet.core import utils from lbrynet.core import utils
from lbrynet.core.LBRYMetadata import verify_name_characters from lbrynet.core.LBRYMetadata import verify_name_characters
from lbrynet.core.utils import generate_id from lbrynet.core.utils import generate_id
from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings
from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, DEFAULT_MAX_SEARCH_RESULTS, KNOWN_DHT_NODES, DEFAULT_MAX_KEY_FEE, \ from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, DEFAULT_MAX_SEARCH_RESULTS, \
DEFAULT_WALLET, DEFAULT_SEARCH_TIMEOUT, DEFAULT_CACHE_TIME, DEFAULT_UI_BRANCH, LOG_POST_URL, LOG_FILE_NAME, SOURCE_TYPES KNOWN_DHT_NODES, DEFAULT_MAX_KEY_FEE, DEFAULT_WALLET, \
from lbrynet.conf import SEARCH_SERVERS DEFAULT_SEARCH_TIMEOUT, DEFAULT_CACHE_TIME, DEFAULT_UI_BRANCH, \
from lbrynet.conf import DEFAULT_TIMEOUT, WALLET_TYPES LOG_POST_URL, LOG_FILE_NAME
from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob from lbrynet.conf import DEFAULT_SD_DOWNLOAD_TIMEOUT
from lbrynet.conf import DEFAULT_TIMEOUT
from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob, BlobStreamDescriptorReader
from lbrynet.core.Session import LBRYSession from lbrynet.core.Session import LBRYSession
from lbrynet.core.PTCWallet import PTCWallet from lbrynet.core.PTCWallet import PTCWallet
from lbrynet.core.LBRYWallet import LBRYcrdWallet, LBRYumWallet from lbrynet.core.LBRYWallet import LBRYcrdWallet, LBRYumWallet
@ -131,6 +133,11 @@ OK_CODE = 200
REMOTE_SERVER = "www.google.com" REMOTE_SERVER = "www.google.com"
class Parameters(object):
def __init__(self, **kwargs):
self.__dict__.update(kwargs)
class LBRYDaemon(jsonrpc.JSONRPC): class LBRYDaemon(jsonrpc.JSONRPC):
""" """
LBRYnet daemon, a jsonrpc interface to lbry functions LBRYnet daemon, a jsonrpc interface to lbry functions
@ -160,6 +167,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
self.run_server = True self.run_server = True
self.session = None self.session = None
self.exchange_rate_manager = ExchangeRateManager() self.exchange_rate_manager = ExchangeRateManager()
self.lighthouse_client = LighthouseClient()
self.waiting_on = {} self.waiting_on = {}
self.streams = {} self.streams = {}
self.pending_claims = {} self.pending_claims = {}
@ -312,14 +320,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
else: else:
self.wallet_dir = os.path.join(get_path(FOLDERID.RoamingAppData, UserHandle.current), "lbryum") self.wallet_dir = os.path.join(get_path(FOLDERID.RoamingAppData, UserHandle.current), "lbryum")
elif sys.platform == "darwin": elif sys.platform == "darwin":
# use the path from the bundle if its available. self.lbrycrdd_path = get_darwin_lbrycrdd_path()
try:
import Foundation
bundle = Foundation.NSBundle.mainBundle()
self.lbrycrdd_path = bundle.pathForResource_ofType_('lbrycrdd', None)
except Exception:
log.exception('Failed to get path from bundle, falling back to default')
self.lbrycrdd_path = "./lbrycrdd"
if self.wallet_type == "lbrycrd": if self.wallet_type == "lbrycrd":
self.wallet_dir = user_data_dir("lbrycrd") self.wallet_dir = user_data_dir("lbrycrd")
else: else:
@ -375,6 +376,11 @@ class LBRYDaemon(jsonrpc.JSONRPC):
f.write("rpcpassword=" + password) f.write("rpcpassword=" + password)
log.info("Done writing lbrycrd.conf") log.info("Done writing lbrycrd.conf")
def _responseFailed(self, err, call):
log.debug(err.getTraceback())
if call.active():
call.cancel()
def render(self, request): def render(self, request):
request.content.seek(0, 0) request.content.seek(0, 0)
# Unmarshal the JSON-RPC data. # Unmarshal the JSON-RPC data.
@ -414,8 +420,13 @@ class LBRYDaemon(jsonrpc.JSONRPC):
d = defer.maybeDeferred(function) d = defer.maybeDeferred(function)
else: else:
d = defer.maybeDeferred(function, *args) d = defer.maybeDeferred(function, *args)
# cancel the response if the connection is broken
notify_finish = request.notifyFinish()
notify_finish.addErrback(self._responseFailed, d)
d.addErrback(self._ebRender, id) d.addErrback(self._ebRender, id)
d.addCallback(self._cbRender, request, id, version) d.addCallback(self._cbRender, request, id, version)
d.addErrback(notify_finish.errback)
return server.NOT_DONE_YET return server.NOT_DONE_YET
def _cbRender(self, result, request, id, version): def _cbRender(self, result, request, id, version):
@ -438,6 +449,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
except: except:
f = jsonrpclib.Fault(self.FAILURE, "can't serialize output") f = jsonrpclib.Fault(self.FAILURE, "can't serialize output")
s = jsonrpclib.dumps(f, version=version) s = jsonrpclib.dumps(f, version=version)
request.setHeader("content-length", str(len(s))) request.setHeader("content-length", str(len(s)))
request.write(s) request.write(s)
request.finish() request.finish()
@ -469,8 +481,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
log.info("Scheduling scripts") log.info("Scheduling scripts")
reactor.callLater(3, self._run_scripts) reactor.callLater(3, self._run_scripts)
# self.lbrynet_connection_checker.start(3600)
if self.first_run: if self.first_run:
d = self._upload_log(log_type="first_run") d = self._upload_log(log_type="first_run")
elif self.upload_log: elif self.upload_log:
@ -478,11 +488,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
else: else:
d = defer.succeed(None) d = defer.succeed(None)
# if float(self.session.wallet.wallet_balance) == 0.0:
# d.addCallback(lambda _: self._check_first_run())
# d.addCallback(self._show_first_run_result)
# d.addCallback(lambda _: _wait_for_credits() if self.requested_first_run_credits else _announce())
d.addCallback(lambda _: _announce()) d.addCallback(lambda _: _announce())
return d return d
@ -625,6 +630,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
# TODO: this was blatantly copied from jsonrpc_start_lbry_file. Be DRY. # TODO: this was blatantly copied from jsonrpc_start_lbry_file. Be DRY.
def _start_file(f): def _start_file(f):
d = self.lbry_file_manager.toggle_lbry_file_running(f) d = self.lbry_file_manager.toggle_lbry_file_running(f)
d.addCallback(lambda _: self.lighthouse_client.announce_sd(f.sd_hash))
return defer.succeed("Started LBRY file") return defer.succeed("Started LBRY file")
def _get_and_start_file(name): def _get_and_start_file(name):
@ -916,6 +922,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
d = self.settings.start() d = self.settings.start()
d.addCallback(lambda _: self.settings.get_lbryid()) d.addCallback(lambda _: self.settings.get_lbryid())
d.addCallback(self._set_lbryid) d.addCallback(self._set_lbryid)
d.addCallback(lambda _: self._modify_loggly_formatter())
return d return d
def _set_lbryid(self, lbryid): def _set_lbryid(self, lbryid):
@ -931,6 +938,14 @@ class LBRYDaemon(jsonrpc.JSONRPC):
d = self.settings.save_lbryid(self.lbryid) d = self.settings.save_lbryid(self.lbryid)
return d return d
def _modify_loggly_formatter(self):
session_id = base58.b58encode(generate_id())
log_support.configure_loggly_handler(
lbry_id=base58.b58encode(self.lbryid),
session_id=session_id
)
def _setup_lbry_file_manager(self): def _setup_lbry_file_manager(self):
self.startup_status = STARTUP_STAGES[3] self.startup_status = STARTUP_STAGES[3]
self.lbry_file_metadata_manager = DBLBRYFileMetadataManager(self.db_dir) self.lbry_file_metadata_manager = DBLBRYFileMetadataManager(self.db_dir)
@ -997,62 +1012,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
return dl return dl
# def _check_first_run(self):
# def _set_first_run_false():
# log.info("Not first run")
# self.first_run = False
# self.session_settings['requested_first_run_credits'] = True
# f = open(self.daemon_conf, "w")
# f.write(json.dumps(self.session_settings))
# f.close()
# return 0.0
#
# if self.wallet_type == 'lbryum':
# d = self.session.wallet.is_first_run()
# d.addCallback(lambda is_first_run: self._do_first_run() if is_first_run or not self.requested_first_run_credits
# else _set_first_run_false())
# else:
# d = defer.succeed(None)
# d.addCallback(lambda _: _set_first_run_false())
# return d
#
# def _do_first_run(self):
# def send_request(url, data):
# log.info("Requesting first run credits")
# r = requests.post(url, json=data)
# if r.status_code == 200:
# self.requested_first_run_credits = True
# self.session_settings['requested_first_run_credits'] = True
# f = open(self.daemon_conf, "w")
# f.write(json.dumps(self.session_settings))
# f.close()
# return r.json()['credits_sent']
# return 0.0
#
# def log_error(err):
# log.warning("unable to request free credits. %s", err.getErrorMessage())
# return 0.0
#
# def request_credits(address):
# url = "http://credreq.lbry.io/requestcredits"
# data = {"address": address}
# d = threads.deferToThread(send_request, url, data)
# d.addErrback(log_error)
# return d
#
# self.first_run = True
# d = self.session.wallet.get_new_address()
# d.addCallback(request_credits)
#
# return d
#
# def _show_first_run_result(self, credits_received):
# if credits_received != 0.0:
# points_string = locale.format_string("%.2f LBC", (round(credits_received, 2),), grouping=True)
# self.startup_message = "Thank you for testing the alpha version of LBRY! You have been given %s for free because we love you. Please hang on for a few minutes for the next block to be mined. When you refresh this page and see your credits you're ready to go!." % points_string
# else:
# self.startup_message = None
def _setup_stream_identifier(self): def _setup_stream_identifier(self):
file_saver_factory = LBRYFileSaverFactory(self.session.peer_finder, self.session.rate_limiter, file_saver_factory = LBRYFileSaverFactory(self.session.peer_finder, self.session.rate_limiter,
self.session.blob_manager, self.stream_info_manager, self.session.blob_manager, self.stream_info_manager,
@ -1072,98 +1031,57 @@ class LBRYDaemon(jsonrpc.JSONRPC):
self.sd_identifier.add_stream_downloader_factory(LBRYFileStreamType, downloader_factory) self.sd_identifier.add_stream_downloader_factory(LBRYFileStreamType, downloader_factory)
return defer.succeed(True) return defer.succeed(True)
def _download_sd_blob(self, sd_hash, timeout=DEFAULT_SD_DOWNLOAD_TIMEOUT):
def cb(result):
if not r.called:
r.callback(result)
def eb():
if not r.called:
r.errback(Exception("sd timeout"))
r = defer.Deferred(None)
reactor.callLater(timeout, eb)
d = download_sd_blob(self.session, sd_hash, PaymentRateManager(self.session.base_payment_rate_manager))
d.addCallback(BlobStreamDescriptorReader)
d.addCallback(lambda blob: blob.get_info())
d.addCallback(cb)
return r
def _download_name(self, name, timeout=DEFAULT_TIMEOUT, download_directory=None, def _download_name(self, name, timeout=DEFAULT_TIMEOUT, download_directory=None,
file_name=None, stream_info=None, wait_for_write=True): file_name=None, stream_info=None, wait_for_write=True):
""" """
Add a lbry file to the file manager, start the download, and return the new lbry file. Add a lbry file to the file manager, start the download, and return the new lbry file.
If it already exists in the file manager, return the existing lbry file If it already exists in the file manager, return the existing lbry file
""" """
helper = _DownloadNameHelper(
if not download_directory: self, name, timeout, download_directory, file_name, wait_for_write)
download_directory = self.download_directory
elif not os.path.isdir(download_directory):
download_directory = self.download_directory
def _remove_from_wait(r):
del self.waiting_on[name]
return r
def _setup_stream(stream_info):
if 'sources' in stream_info.keys():
stream_hash = stream_info['sources']['lbry_sd_hash']
else:
stream_hash = stream_info['stream_hash']
d = self._get_lbry_file_by_sd_hash(stream_hash)
def _add_results(l):
if l:
if os.path.isfile(os.path.join(self.download_directory, l.file_name)):
return defer.succeed((stream_info, l))
return defer.succeed((stream_info, None))
d.addCallback(_add_results)
return d
def _wait_on_lbry_file(f):
if os.path.isfile(os.path.join(self.download_directory, f.file_name)):
written_file = file(os.path.join(self.download_directory, f.file_name))
written_file.seek(0, os.SEEK_END)
written_bytes = written_file.tell()
written_file.close()
else:
written_bytes = False
if not written_bytes:
d = defer.succeed(None)
d.addCallback(lambda _: reactor.callLater(1, _wait_on_lbry_file, f))
return d
else:
return defer.succeed(_disp_file(f))
def _disp_file(f):
file_path = os.path.join(self.download_directory, f.file_name)
log.info("Already downloaded: " + str(f.sd_hash) + " --> " + file_path)
return f
def _get_stream(stream_info):
def _wait_for_write():
try:
if os.path.isfile(os.path.join(self.download_directory, self.streams[name].downloader.file_name)):
written_file = file(os.path.join(self.download_directory, self.streams[name].downloader.file_name))
written_file.seek(0, os.SEEK_END)
written_bytes = written_file.tell()
written_file.close()
else:
written_bytes = False
except:
written_bytes = False
if not written_bytes:
d = defer.succeed(None)
d.addCallback(lambda _: reactor.callLater(1, _wait_for_write))
return d
else:
return defer.succeed(None)
self.streams[name] = GetStream(self.sd_identifier, self.session, self.session.wallet,
self.lbry_file_manager, self.exchange_rate_manager,
max_key_fee=self.max_key_fee, data_rate=self.data_rate, timeout=timeout,
download_directory=download_directory, file_name=file_name)
d = self.streams[name].start(stream_info, name)
if wait_for_write:
d.addCallback(lambda _: _wait_for_write())
d.addCallback(lambda _: self.streams[name].downloader)
return d
if not stream_info: if not stream_info:
self.waiting_on[name] = True self.waiting_on[name] = True
d = self._resolve_name(name) d = self._resolve_name(name)
else: else:
d = defer.succeed(stream_info) d = defer.succeed(stream_info)
d.addCallback(_setup_stream) d.addCallback(helper._setup_stream)
d.addCallback(lambda (stream_info, lbry_file): _get_stream(stream_info) if not lbry_file else _wait_on_lbry_file(lbry_file)) d.addCallback(helper.wait_or_get_stream)
if not stream_info: if not stream_info:
d.addCallback(_remove_from_wait) d.addCallback(helper._remove_from_wait)
return d
def add_stream(self, name, timeout, download_directory, file_name, stream_info):
"""Makes, adds and starts a stream"""
self.streams[name] = GetStream(self.sd_identifier,
self.session,
self.session.wallet,
self.lbry_file_manager,
self.exchange_rate_manager,
max_key_fee=self.max_key_fee,
data_rate=self.data_rate,
timeout=timeout,
download_directory=download_directory,
file_name=file_name)
d = self.streams[name].start(stream_info, name)
return d return d
def _get_long_count_timestamp(self): def _get_long_count_timestamp(self):
@ -1176,44 +1094,16 @@ class LBRYDaemon(jsonrpc.JSONRPC):
return defer.succeed(True) return defer.succeed(True)
def _resolve_name(self, name, force_refresh=False): def _resolve_name(self, name, force_refresh=False):
try: """Resolves a name. Checks the cache first before going out to the blockchain.
verify_name_characters(name)
except AssertionError:
log.error("Bad name")
return defer.fail(InvalidNameError("Bad name"))
def _cache_stream_info(stream_info): Args:
def _add_txid(txid): name: the lbry://<name> to resolve
self.name_cache[name]['txid'] = txid force_refresh: if True, always go out to the blockchain to resolve.
return defer.succeed(None) """
if name.startswith('lbry://'):
self.name_cache[name] = {'claim_metadata': stream_info, 'timestamp': self._get_long_count_timestamp()} raise ValueError('name %s should not start with lbry://')
d = self.session.wallet.get_txid_for_name(name) helper = _ResolveNameHelper(self, name, force_refresh)
d.addCallback(_add_txid) return helper.get_deferred()
d.addCallback(lambda _: self._update_claim_cache())
d.addCallback(lambda _: self.name_cache[name]['claim_metadata'])
return d
if not force_refresh:
if name in self.name_cache.keys():
if (self._get_long_count_timestamp() - self.name_cache[name]['timestamp']) < self.cache_time:
log.info("Returning cached stream info for lbry://" + name)
d = defer.succeed(self.name_cache[name]['claim_metadata'])
else:
log.info("Refreshing stream info for lbry://" + name)
d = self.session.wallet.get_stream_info_for_name(name)
d.addCallbacks(_cache_stream_info, lambda _: defer.fail(UnknownNameError))
else:
log.info("Resolving stream info for lbry://" + name)
d = self.session.wallet.get_stream_info_for_name(name)
d.addCallbacks(_cache_stream_info, lambda _: defer.fail(UnknownNameError))
else:
log.info("Resolving stream info for lbry://" + name)
d = self.session.wallet.get_stream_info_for_name(name)
d.addCallbacks(_cache_stream_info, lambda _: defer.fail(UnknownNameError))
return d
def _delete_lbry_file(self, lbry_file, delete_file=True): def _delete_lbry_file(self, lbry_file, delete_file=True):
d = self.lbry_file_manager.delete_lbry_file(lbry_file) d = self.lbry_file_manager.delete_lbry_file(lbry_file)
@ -1239,9 +1129,12 @@ class LBRYDaemon(jsonrpc.JSONRPC):
def _get_est_cost(self, name): def _get_est_cost(self, name):
def _check_est(d, name): def _check_est(d, name):
try:
if isinstance(d.result, float): if isinstance(d.result, float):
log.info("Cost est for lbry://" + name + ": " + str(d.result) + "LBC") log.info("Cost est for lbry://" + name + ": " + str(d.result) + "LBC")
else: return defer.succeed(None)
except AttributeError:
pass
log.info("Timeout estimating cost for lbry://" + name + ", using key fee") log.info("Timeout estimating cost for lbry://" + name + ", using key fee")
d.cancel() d.cancel()
return defer.succeed(None) return defer.succeed(None)
@ -1343,7 +1236,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
'stream_name': f.stream_name, 'stream_name': f.stream_name,
'suggested_file_name': f.suggested_file_name, 'suggested_file_name': f.suggested_file_name,
'upload_allowed': f.upload_allowed, 'sd_hash': f.sd_hash, 'upload_allowed': f.upload_allowed, 'sd_hash': f.sd_hash,
'lbry_uri': f.uri, 'txid': f.txid, 'lbry_uri': f.uri, 'txid': f.txid, 'claim_id': f.claim_id,
'total_bytes': size, 'total_bytes': size,
'written_bytes': written_bytes, 'code': status[0], 'written_bytes': written_bytes, 'code': status[0],
'message': message}) 'message': message})
@ -1355,7 +1248,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
'points_paid': f.points_paid, 'stopped': f.stopped, 'stream_hash': f.stream_hash, 'points_paid': f.points_paid, 'stopped': f.stopped, 'stream_hash': f.stream_hash,
'stream_name': f.stream_name, 'suggested_file_name': f.suggested_file_name, 'stream_name': f.stream_name, 'suggested_file_name': f.suggested_file_name,
'upload_allowed': f.upload_allowed, 'sd_hash': f.sd_hash, 'total_bytes': size, 'upload_allowed': f.upload_allowed, 'sd_hash': f.sd_hash, 'total_bytes': size,
'written_bytes': written_bytes, 'lbry_uri': f.uri, 'txid': f.txid, 'written_bytes': written_bytes, 'lbry_uri': f.uri, 'txid': f.txid, 'claim_id': f.claim_id,
'code': status[0], 'message': status[1]}) 'code': status[0], 'message': status[1]})
return d return d
@ -1386,7 +1279,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
d = self._get_lbry_file_by_sd_hash(val) d = self._get_lbry_file_by_sd_hash(val)
elif search_by == "file_name": elif search_by == "file_name":
d = self._get_lbry_file_by_file_name(val) d = self._get_lbry_file_by_file_name(val)
d.addCallback(_log_get_lbry_file) # d.addCallback(_log_get_lbry_file)
if return_json: if return_json:
d.addCallback(_get_json_for_return) d.addCallback(_get_json_for_return)
return d return d
@ -1426,8 +1319,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
return defer.succeed(None) return defer.succeed(None)
def _search(self, search): def _search(self, search):
proxy = Proxy(random.choice(SEARCH_SERVERS)) return self.lighthouse_client.search(search)
return proxy.callRemote('search', search)
def _render_response(self, result, code): def _render_response(self, result, code):
return defer.succeed({'result': result, 'code': code}) return defer.succeed({'result': result, 'code': code})
@ -1753,74 +1645,67 @@ class LBRYDaemon(jsonrpc.JSONRPC):
""" """
def _convert_amount_to_float(r): def _convert_amount_to_float(r):
if not r:
return False
else:
r['amount'] = float(r['amount']) / 10**8 r['amount'] = float(r['amount']) / 10**8
return r return r
name = p['name'] name = p['name']
d = self.session.wallet.get_claim_info(name) txid = p.get('txid', None)
d = self.session.wallet.get_claim_info(name, txid)
d.addCallback(_convert_amount_to_float) d.addCallback(_convert_amount_to_float)
d.addCallback(lambda r: self._render_response(r, OK_CODE)) d.addCallback(lambda r: self._render_response(r, OK_CODE))
return d return d
def _process_get_parameters(self, p):
"""Extract info from input parameters and fill in default values for `get` call."""
# TODO: this process can be abstracted s.t. each method
# can spec what parameters it expects and how to set default values
timeout = p.get('timeout', self.download_timeout)
download_directory = p.get('download_directory', self.download_directory)
file_name = p.get('file_name')
stream_info = p.get('stream_info')
sd_hash = get_sd_hash(stream_info)
wait_for_write = p.get('wait_for_write', True)
name = p.get('name')
return Parameters(
timeout=timeout,
download_directory=download_directory,
file_name=file_name,
stream_info=stream_info,
sd_hash=sd_hash,
wait_for_write=wait_for_write,
name=name
)
def jsonrpc_get(self, p): def jsonrpc_get(self, p):
""" """Download stream from a LBRY uri.
Download stream from a LBRY uri
Args: Args:
'name': name to download, string 'name': name to download, string
'download_directory': optional, path to directory where file will be saved, string 'download_directory': optional, path to directory where file will be saved, string
'file_name': optional, a user specified name for the downloaded file 'file_name': optional, a user specified name for the downloaded file
'stream_info': optional, specified stream info overrides name 'stream_info': optional, specified stream info overrides name
'timeout': optional
'wait_for_write': optional, defaults to True
Returns: Returns:
'stream_hash': hex string 'stream_hash': hex string
'path': path of download 'path': path of download
""" """
params = self._process_get_parameters(p)
if 'timeout' not in p.keys(): if not params.name:
timeout = self.download_timeout return server.failure
else: if params.name in self.waiting_on:
timeout = p['timeout'] return server.failure
d = self._download_name(name=params.name,
if 'download_directory' not in p.keys(): timeout=params.timeout,
download_directory = self.download_directory download_directory=params.download_directory,
else: stream_info=params.stream_info,
download_directory = p['download_directory'] file_name=params.file_name,
wait_for_write=params.wait_for_write)
if 'file_name' in p.keys(): d.addCallback(get_output_callback(params))
file_name = p['file_name']
else:
file_name = None
if 'stream_info' in p.keys():
stream_info = p['stream_info']
if 'sources' in stream_info.keys():
sd_hash = stream_info['sources']['lbry_sd_hash']
else:
sd_hash = stream_info['stream_hash']
else:
stream_info = None
if 'wait_for_write' in p.keys():
wait_for_write = p['wait_for_write']
else:
wait_for_write = True
if 'name' in p.keys():
name = p['name']
if p['name'] not in self.waiting_on.keys():
d = self._download_name(name=name, timeout=timeout, download_directory=download_directory,
stream_info=stream_info, file_name=file_name, wait_for_write=wait_for_write)
d.addCallback(lambda l: {'stream_hash': sd_hash,
'path': os.path.join(self.download_directory, l.file_name)}
if stream_info else
{'stream_hash': l.sd_hash,
'path': os.path.join(self.download_directory, l.file_name)})
d.addCallback(lambda message: self._render_response(message, OK_CODE)) d.addCallback(lambda message: self._render_response(message, OK_CODE))
else:
d = server.failure
else:
d = server.failure
return d return d
def jsonrpc_stop_lbry_file(self, p): def jsonrpc_stop_lbry_file(self, p):
@ -1897,41 +1782,27 @@ class LBRYDaemon(jsonrpc.JSONRPC):
List of search results List of search results
""" """
# TODO: change this function to "search", and use cached stream size info from the search server # TODO: change this function to "search"
if 'search' in p.keys(): if 'search' in p.keys():
search = p['search'] search = p['search']
else: else:
return self._render_response(None, BAD_REQUEST) return self._render_response(None, BAD_REQUEST)
# TODO: have ui accept the actual outputs
def _clean(n): def _clean(n):
t = [] t = []
for i in n: for i in n:
if i[0]: td = {k: i['value'][k] for k in i['value']}
tr = {} td['cost_est'] = float(i['cost'])
tr.update(i[1][0]['value']) td['thumbnail'] = i['value'].get('thumbnail', "img/Free-speech-flag.svg")
thumb = tr.get('thumbnail', None) td['name'] = i['name']
if thumb is None: t.append(td)
tr['thumbnail'] = "img/Free-speech-flag.svg"
tr['name'] = i[1][0]['name']
tr['cost_est'] = i[1][1]
t.append(tr)
return t return t
def get_est_costs(results):
def _save_cost(search_result):
d = self._get_est_cost(search_result['name'])
d.addCallback(lambda p: [search_result, p])
return d
dl = defer.DeferredList([_save_cost(r) for r in results], consumeErrors=True)
return dl
log.info('Search: %s' % search) log.info('Search: %s' % search)
d = self._search(search) d = self._search(search)
d.addCallback(lambda claims: claims[:self.max_search_results])
d.addCallback(get_est_costs)
d.addCallback(_clean) d.addCallback(_clean)
d.addCallback(lambda results: self._render_response(results, OK_CODE)) d.addCallback(lambda results: self._render_response(results, OK_CODE))
@ -1980,26 +1851,31 @@ class LBRYDaemon(jsonrpc.JSONRPC):
Claim txid Claim txid
""" """
def _set_address(address, currency, m):
log.info("Generated new address for key fee: " + str(address))
m['fee'][currency]['address'] = address
return m
name = p['name'] name = p['name']
log.info("Publish: ")
log.info(p)
try: try:
verify_name_characters(name) verify_name_characters(name)
except: except AssertionError:
log.error("Bad name") log.error("Bad name")
return defer.fail(InvalidNameError("Bad name")) return defer.fail(InvalidNameError("Bad name"))
bid = p['bid'] bid = p['bid']
file_path = p['file_path']
try:
metadata = Metadata(p['metadata'])
make_lbry_file = False
except AssertionError:
make_lbry_file = True
metadata = p['metadata'] metadata = p['metadata']
file_path = p['file_path']
def _set_address(address, currency):
log.info("Generated new address for key fee: " + str(address))
metadata['fee'][currency]['address'] = address
return defer.succeed(None)
def _delete_data(lbry_file):
txid = lbry_file.txid
d = self._delete_lbry_file(lbry_file, delete_file=False)
d.addCallback(lambda _: txid)
return d
if not self.pending_claim_checker.running: if not self.pending_claim_checker.running:
self.pending_claim_checker.start(30) self.pending_claim_checker.start(30)
@ -2013,15 +1889,16 @@ class LBRYDaemon(jsonrpc.JSONRPC):
for c in metadata['fee']: for c in metadata['fee']:
if 'address' not in metadata['fee'][c]: if 'address' not in metadata['fee'][c]:
d.addCallback(lambda _: self.session.wallet.get_new_address()) d.addCallback(lambda _: self.session.wallet.get_new_address())
d.addCallback(lambda addr: _set_address(addr, c)) d.addCallback(lambda addr: _set_address(addr, c, metadata))
else:
d.addCallback(lambda _: metadata)
if make_lbry_file:
pub = Publisher(self.session, self.lbry_file_manager, self.session.wallet) pub = Publisher(self.session, self.lbry_file_manager, self.session.wallet)
d.addCallback(lambda _: self._get_lbry_file_by_uri(name)) d.addCallback(lambda meta: pub.start(name, file_path, bid, meta))
d.addCallbacks(lambda l: None if not l else _delete_data(l), lambda _: None) else:
d.addCallback(lambda r: pub.start(name, file_path, bid, metadata, r)) d.addCallback(lambda meta: self.session.wallet.claim_name(name, bid, meta))
d.addCallback(lambda txid: self._add_to_pending_claims(name, txid)) d.addCallback(lambda txid: self._add_to_pending_claims(name, txid))
d.addCallback(lambda r: self._render_response(r, OK_CODE)) d.addCallback(lambda r: self._render_response(r, OK_CODE))
d.addErrback(lambda err: self._render_response(err.getTraceback(), BAD_REQUEST))
return d return d
@ -2051,6 +1928,25 @@ class LBRYDaemon(jsonrpc.JSONRPC):
return d return d
def jsonrpc_support_claim(self, p):
"""
Support a name claim
Args:
'name': name
'claim_id': claim id of claim to support
'amount': amount to support by
Return:
txid
"""
name = p['name']
claim_id = p['claim_id']
amount = p['amount']
d = self.session.wallet.support_claim(name, claim_id, amount)
d.addCallback(lambda r: self._render_response(r, OK_CODE))
return d
def jsonrpc_get_name_claims(self): def jsonrpc_get_name_claims(self):
""" """
Get my name claims Get my name claims
@ -2074,6 +1970,21 @@ class LBRYDaemon(jsonrpc.JSONRPC):
return d return d
def jsonrpc_get_claims_for_name(self, p):
"""
Get claims for a name
Args:
'name': name
Returns
list of name claims
"""
name = p['name']
d = self.session.wallet.get_claims_for_name(name)
d.addCallback(lambda r: self._render_response(r, OK_CODE))
return d
def jsonrpc_get_transaction_history(self): def jsonrpc_get_transaction_history(self):
""" """
Get transaction history Get transaction history
@ -2236,6 +2147,22 @@ class LBRYDaemon(jsonrpc.JSONRPC):
d.addCallback(lambda r: self._render_response(r, OK_CODE)) d.addCallback(lambda r: self._render_response(r, OK_CODE))
return d return d
def jsonrpc_download_descriptor(self, p):
"""
Download and return a sd blob
Args:
sd_hash
Returns
sd blob, dict
"""
sd_hash = p['sd_hash']
timeout = p.get('timeout', DEFAULT_SD_DOWNLOAD_TIMEOUT)
d = self._download_sd_blob(sd_hash, timeout)
d.addCallbacks(lambda r: self._render_response(r, OK_CODE), lambda _: self._render_response(False, OK_CODE))
return d
def jsonrpc_get_nametrie(self): def jsonrpc_get_nametrie(self):
""" """
Get the nametrie Get the nametrie
@ -2377,11 +2304,28 @@ class LBRYDaemon(jsonrpc.JSONRPC):
d = threads.deferToThread(subprocess.Popen, ['open', '-R', path]) d = threads.deferToThread(subprocess.Popen, ['open', '-R', path])
else: else:
# No easy way to reveal specific files on Linux, so just open the containing directory # No easy way to reveal specific files on Linux, so just open the containing directory
d = threads.deferToThread(subprocess.Popen, ['xdg-open', os.dirname(path)]) d = threads.deferToThread(subprocess.Popen, ['xdg-open', os.path.dirname(path)])
d.addCallback(lambda _: self._render_response(True, OK_CODE)) d.addCallback(lambda _: self._render_response(True, OK_CODE))
return d return d
def jsonrpc_get_peers_for_hash(self, p):
"""
Get peers for blob hash
Args:
'blob_hash': blob hash
Returns:
List of contacts
"""
blob_hash = p['blob_hash']
d = self.session.peer_finder.find_peers_for_blob(blob_hash)
d.addCallback(lambda r: [[c.host, c.port, c.is_available()] for c in r])
d.addCallback(lambda r: self._render_response(r, OK_CODE))
return d
def get_lbrynet_version_from_github(): def get_lbrynet_version_from_github():
"""Return the latest released version from github.""" """Return the latest released version from github."""
@ -2400,3 +2344,183 @@ def get_version_from_tag(tag):
return match.group(1) return match.group(1)
else: else:
raise Exception('Failed to parse version from tag {}'.format(tag)) raise Exception('Failed to parse version from tag {}'.format(tag))
def get_sd_hash(stream_info):
if not stream_info:
return None
try:
return stream_info['sources']['lbry_sd_hash']
except KeyError:
return stream_info.get('stream_hash')
def get_output_callback(params):
def callback(l):
return {
'stream_hash': params.sd_hash if params.stream_info else l.sd_hash,
'path': os.path.join(params.download_directory, l.file_name)
}
return callback
def get_darwin_lbrycrdd_path():
# use the path from the bundle if its available.
default = "./lbrycrdd"
try:
import Foundation
except ImportError:
log.warning('Foundation module not installed, falling back to default lbrycrdd path')
return default
else:
try:
bundle = Foundation.NSBundle.mainBundle()
return bundle.pathForResource_ofType_('lbrycrdd', None)
except Exception:
log.exception('Failed to get path from bundle, falling back to default')
return default
class _DownloadNameHelper(object):
def __init__(self, daemon, name, timeout=DEFAULT_TIMEOUT, download_directory=None,
file_name=None, wait_for_write=True):
self.daemon = daemon
self.name = name
self.timeout = timeout
if not download_directory or not os.path.isdir(download_directory):
self.download_directory = daemon.download_directory
else:
self.download_directory = download_directory
self.file_name = file_name
self.wait_for_write = wait_for_write
def _setup_stream(self, stream_info):
stream_hash = get_sd_hash(stream_info)
d = self.daemon._get_lbry_file_by_sd_hash(stream_hash)
d.addCallback(self._add_results_callback(stream_info))
return d
def _add_results_callback(self, stream_info):
def add_results(l):
if l:
if os.path.isfile(os.path.join(self.download_directory, l.file_name)):
return defer.succeed((stream_info, l))
return defer.succeed((stream_info, None))
return add_results
def wait_or_get_stream(self, args):
stream_info, lbry_file = args
if lbry_file:
return self._wait_on_lbry_file(lbry_file)
else:
return self._get_stream(stream_info)
def _get_stream(self, stream_info):
d = self.daemon.add_stream(
self.name, self.timeout, self.download_directory, self.file_name, stream_info)
if self.wait_for_write:
d.addCallback(lambda _: self._wait_for_write())
d.addCallback(lambda _: self.daemon.streams[self.name].downloader)
return d
def _wait_for_write(self):
d = defer.succeed(None)
if not self.has_downloader_wrote():
d.addCallback(lambda _: reactor.callLater(1, self._wait_for_write))
return d
def has_downloader_wrote(self):
downloader = self.daemon.streams[self.name].downloader
if not downloader:
return False
return self.get_written_bytes(downloader.file_name)
def _wait_on_lbry_file(self, f):
written_bytes = self.get_written_bytes(f.file_name)
if written_bytes:
return defer.succeed(self._disp_file(f))
d = defer.succeed(None)
d.addCallback(lambda _: reactor.callLater(1, self._wait_on_lbry_file, f))
return d
def get_written_bytes(self, file_name):
"""Returns the number of bytes written to `file_name`.
Returns False if there were issues reading `file_name`.
"""
try:
file_path = os.path.join(self.download_directory, file_name)
if os.path.isfile(file_path):
written_file = file(file_path)
written_file.seek(0, os.SEEK_END)
written_bytes = written_file.tell()
written_file.close()
else:
written_bytes = False
except Exception:
writen_bytes = False
return written_bytes
def _disp_file(self, f):
file_path = os.path.join(self.download_directory, f.file_name)
log.info("Already downloaded: %s --> %s", f.sd_hash, file_path)
return f
def _remove_from_wait(self, r):
del self.daemon.waiting_on[self.name]
return r
class _ResolveNameHelper(object):
def __init__(self, daemon, name, force_refresh):
self.daemon = daemon
self.name = name
self.force_refresh = force_refresh
def get_deferred(self):
if self.need_fresh_stream():
log.info("Resolving stream info for lbry://%s", self.name)
d = self.wallet.get_stream_info_for_name(self.name)
d.addCallbacks(self._cache_stream_info, lambda _: defer.fail(UnknownNameError))
else:
log.debug("Returning cached stream info for lbry://%s", self.name)
d = defer.succeed(self.name_data['claim_metadata'])
return d
@property
def name_data(self):
return self.daemon.name_cache[self.name]
@property
def wallet(self):
return self.daemon.session.wallet
def now(self):
return self.daemon._get_long_count_timestamp()
def _add_txid(self, txid):
self.name_data['txid'] = txid
return defer.succeed(None)
def _cache_stream_info(self, stream_info):
self.daemon.name_cache[self.name] = {
'claim_metadata': stream_info,
'timestamp': self.now()
}
d = self.wallet.get_txid_for_name(self.name)
d.addCallback(self._add_txid)
d.addCallback(lambda _: self.daemon._update_claim_cache())
d.addCallback(lambda _: self.name_data['claim_metadata'])
return d
def need_fresh_stream(self):
return self.force_refresh or not self.is_in_cache() or self.is_cached_name_expired()
def is_in_cache(self):
return self.name in self.daemon.name_cache
def is_cached_name_expired(self):
time_in_cache = self.now() - self.name_data['timestamp']
return time_in_cache >= self.daemon.cache_time

2
lbrynet/lbrynet_daemon/LBRYDaemonCLI.py
View file

@ -1,7 +1,7 @@
import sys import sys
import json import json
from lbrynet.conf import API_CONNECTION_STRING, LOG_FILE_NAME from lbrynet.conf import API_CONNECTION_STRING
from jsonrpc.proxy import JSONRPCProxy from jsonrpc.proxy import JSONRPCProxy
help_msg = "Useage: lbrynet-cli method json-args\n" \ help_msg = "Useage: lbrynet-cli method json-args\n" \

13
lbrynet/lbrynet_daemon/LBRYDaemonControl.py
View file

@ -5,7 +5,6 @@ import os
import webbrowser import webbrowser
import sys import sys
import socket import socket
import platform
from appdirs import user_data_dir from appdirs import user_data_dir
from twisted.web import server from twisted.web import server
@ -14,8 +13,8 @@ from jsonrpc.proxy import JSONRPCProxy
from lbrynet.core import log_support from lbrynet.core import log_support
from lbrynet.lbrynet_daemon.LBRYDaemonServer import LBRYDaemonServer, LBRYDaemonRequest from lbrynet.lbrynet_daemon.LBRYDaemonServer import LBRYDaemonServer, LBRYDaemonRequest
from lbrynet.conf import API_CONNECTION_STRING, API_INTERFACE, API_ADDRESS, API_PORT, \ from lbrynet.conf import API_CONNECTION_STRING, API_INTERFACE, API_PORT, \
DEFAULT_WALLET, UI_ADDRESS, DEFAULT_UI_BRANCH, LOG_FILE_NAME UI_ADDRESS, DEFAULT_UI_BRANCH, LOG_FILE_NAME
# TODO: stop it! # TODO: stop it!
if sys.platform != "darwin": if sys.platform != "darwin":
@ -74,11 +73,11 @@ def start():
parser.set_defaults(branch=False, launchui=True, logtoconsole=False, quiet=False) parser.set_defaults(branch=False, launchui=True, logtoconsole=False, quiet=False)
args = parser.parse_args() args = parser.parse_args()
log_support.disable_noisy_loggers()
log_support.configureFileHandler(lbrynet_log) log_support.configure_file_handler(lbrynet_log)
log_support.configure_loggly_handler()
if args.logtoconsole: if args.logtoconsole:
log_support.configureConsole() log_support.configure_console()
try: try:
JSONRPCProxy.from_url(API_CONNECTION_STRING).is_running() JSONRPCProxy.from_url(API_CONNECTION_STRING).is_running()

5
lbrynet/lbrynet_daemon/LBRYDaemonServer.py
View file

@ -9,15 +9,14 @@ import tempfile
import time import time
import cgi import cgi
from datetime import datetime
from appdirs import user_data_dir from appdirs import user_data_dir
from twisted.web import server, static, resource from twisted.web import server, static, resource
from twisted.internet import defer, interfaces, error, reactor, task, threads from twisted.internet import defer, interfaces, error, reactor, threads
from zope.interface import implements from zope.interface import implements
from lbrynet.lbrynet_daemon.LBRYDaemon import LBRYDaemon from lbrynet.lbrynet_daemon.LBRYDaemon import LBRYDaemon
from lbrynet.conf import API_CONNECTION_STRING, API_ADDRESS, DEFAULT_WALLET, UI_ADDRESS, DEFAULT_UI_BRANCH, LOG_FILE_NAME from lbrynet.conf import API_ADDRESS, UI_ADDRESS, DEFAULT_UI_BRANCH, LOG_FILE_NAME
# TODO: omg, this code is essentially duplicated in LBRYDaemon # TODO: omg, this code is essentially duplicated in LBRYDaemon

25
lbrynet/lbrynet_daemon/LBRYDownloader.py
View file

@ -5,14 +5,13 @@ import sys
from copy import deepcopy from copy import deepcopy
from appdirs import user_data_dir from appdirs import user_data_dir
from datetime import datetime
from twisted.internet import defer from twisted.internet import defer
from twisted.internet.task import LoopingCall from twisted.internet.task import LoopingCall
from lbrynet.core.Error import InvalidStreamInfoError, InsufficientFundsError, KeyFeeAboveMaxAllowed from lbrynet.core.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed
from lbrynet.core.PaymentRateManager import PaymentRateManager from lbrynet.core.PaymentRateManager import PaymentRateManager
from lbrynet.core.StreamDescriptor import download_sd_blob from lbrynet.core.StreamDescriptor import download_sd_blob
from lbrynet.core.LBRYMetadata import Metadata, LBRYFeeValidator from lbrynet.core.LBRYMetadata import LBRYFeeValidator
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloaderFactory from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloaderFactory
from lbrynet.conf import DEFAULT_TIMEOUT, LOG_FILE_NAME from lbrynet.conf import DEFAULT_TIMEOUT, LOG_FILE_NAME
@ -150,21 +149,19 @@ class GetStream(object):
return self.finished return self.finished
def _start_download(self, downloader): def _start_download(self, downloader):
def _pay_key_fee(): log.info('Starting download for %s', self.name)
self.downloader = downloader
self.download_path = os.path.join(downloader.download_directory, downloader.file_name)
d = self._pay_key_fee()
d.addCallback(lambda _: log.info("Downloading %s --> %s", self.stream_hash, self.downloader.file_name))
d.addCallback(lambda _: self.downloader.start())
def _pay_key_fee(self):
if self.fee is not None: if self.fee is not None:
fee_lbc = self.exchange_rate_manager.to_lbc(self.fee).amount fee_lbc = self.exchange_rate_manager.to_lbc(self.fee).amount
reserved_points = self.wallet.reserve_points(self.fee.address, fee_lbc) reserved_points = self.wallet.reserve_points(self.fee.address, fee_lbc)
if reserved_points is None: if reserved_points is None:
return defer.fail(InsufficientFundsError()) return defer.fail(InsufficientFundsError())
return self.wallet.send_points_to_address(reserved_points, fee_lbc) return self.wallet.send_points_to_address(reserved_points, fee_lbc)
return defer.succeed(None) return defer.succeed(None)
d = _pay_key_fee()
self.downloader = downloader
self.download_path = os.path.join(downloader.download_directory, downloader.file_name)
d.addCallback(lambda _: log.info("Downloading %s --> %s", self.stream_hash, self.downloader.file_name))
d.addCallback(lambda _: self.downloader.start())

19
lbrynet/lbrynet_daemon/LBRYPublisher.py
View file

@ -4,7 +4,6 @@ import os
import sys import sys
from appdirs import user_data_dir from appdirs import user_data_dir
from datetime import datetime
from lbrynet.core.Error import InsufficientFundsError from lbrynet.core.Error import InsufficientFundsError
from lbrynet.lbryfilemanager.LBRYFileCreator import create_lbry_file from lbrynet.lbryfilemanager.LBRYFileCreator import create_lbry_file
@ -43,7 +42,7 @@ class Publisher(object):
self.stream_hash = None self.stream_hash = None
self.metadata = {} self.metadata = {}
def start(self, name, file_path, bid, metadata, old_txid): def start(self, name, file_path, bid, metadata):
def _show_result(): def _show_result():
log.info("Published %s --> lbry://%s txid: %s", self.file_name, self.publish_name, self.txid) log.info("Published %s --> lbry://%s txid: %s", self.file_name, self.publish_name, self.txid)
@ -53,7 +52,6 @@ class Publisher(object):
self.file_path = file_path self.file_path = file_path
self.bid_amount = bid self.bid_amount = bid
self.metadata = metadata self.metadata = metadata
self.old_txid = old_txid
d = self._check_file_path(self.file_path) d = self._check_file_path(self.file_path)
d.addCallback(lambda _: create_lbry_file(self.session, self.lbry_file_manager, d.addCallback(lambda _: create_lbry_file(self.session, self.lbry_file_manager,
@ -106,21 +104,12 @@ class Publisher(object):
self.metadata['content-type'] = mimetypes.guess_type(os.path.join(self.lbry_file.download_directory, self.metadata['content-type'] = mimetypes.guess_type(os.path.join(self.lbry_file.download_directory,
self.lbry_file.file_name))[0] self.lbry_file.file_name))[0]
self.metadata['ver'] = CURRENT_METADATA_VERSION self.metadata['ver'] = CURRENT_METADATA_VERSION
m = Metadata(self.metadata)
if self.old_txid:
d = self.wallet.abandon_name(self.old_txid)
d.addCallback(lambda tx: log.info("Abandoned tx %s" % str(tx)))
d.addCallback(lambda _: self.wallet.claim_name(self.publish_name,
self.bid_amount,
Metadata(self.metadata)))
else:
d = self.wallet.claim_name(self.publish_name,
self.bid_amount,
Metadata(self.metadata))
def set_tx_hash(txid): def set_tx_hash(txid):
self.txid = txid self.txid = txid
d = self.wallet.claim_name(self.publish_name, self.bid_amount, m)
d.addCallback(set_tx_hash) d.addCallback(set_tx_hash)
return d return d
@ -135,4 +124,4 @@ class Publisher(object):
log.error(error_message) log.error(error_message)
log.error(message, str(self.file_name), str(self.publish_name), err.getTraceback()) log.error(message, str(self.file_name), str(self.publish_name), err.getTraceback())
return defer.succeed(error_message) return defer.fail(Exception("Publish failed"))

27
lbrynet/lbrynet_daemon/Lighthouse.py Normal file
View file

@ -0,0 +1,27 @@
import logging
import random
from txjsonrpc.web.jsonrpc import Proxy
from lbrynet.conf import SEARCH_SERVERS
log = logging.getLogger(__name__)
class LighthouseClient(object):
def __init__(self, servers=None):
self.servers = servers or SEARCH_SERVERS
def _get_random_server(self):
return Proxy(random.choice(self.servers))
def _run_query(self, func, arg):
return self._get_random_server().callRemote(func, arg)
def search(self, search):
return self._run_query('search', search)
def announce_sd(self, sd_hash):
log.info("Announce sd to lighthouse")
return self._run_query('announce_sd', sd_hash)
def check_available(self, sd_hash):
return self._run_query('check_available', sd_hash)

1
lbrynet/node_rpc_cli.py
View file

@ -6,7 +6,6 @@ CLI for sending rpc commands to a DHT node
from twisted.internet import reactor from twisted.internet import reactor
from txjsonrpc.web.jsonrpc import Proxy from txjsonrpc.web.jsonrpc import Proxy
import argparse import argparse
import sys
def print_value(value): def print_value(value):

11
packaging/osx/lbry-osx-app/lbrygui/LBRYApp.py
View file

@ -27,7 +27,7 @@ if not os.path.isfile(lbrycrdd_path_conf):
f.write(lbrycrdd_path) f.write(lbrycrdd_path)
f.close() f.close()
from lbrynet.lbrynet_daemon.LBRYDaemonServer import LBRYDaemonServer from lbrynet.lbrynet_daemon.LBRYDaemonServer import LBRYDaemonServer, LBRYDaemonRequest
from lbrynet.conf import API_PORT, API_INTERFACE, ICON_PATH, APP_NAME from lbrynet.conf import API_PORT, API_INTERFACE, ICON_PATH, APP_NAME
from lbrynet.conf import UI_ADDRESS from lbrynet.conf import UI_ADDRESS
@ -74,16 +74,13 @@ class LBRYDaemonApp(AppKit.NSApplication):
LBRYNotify("LBRY needs an internet connection to start, try again when one is available") LBRYNotify("LBRY needs an internet connection to start, try again when one is available")
sys.exit(0) sys.exit(0)
# if not subprocess.check_output("git ls-remote https://github.com/lbryio/lbry-web-ui.git | grep HEAD | cut -f 1",
# shell=True):
# LBRYNotify(
# "You should have been prompted to install xcode command line tools, please do so and then start LBRY")
# sys.exit(0)
lbry = LBRYDaemonServer() lbry = LBRYDaemonServer()
d = lbry.start() d = lbry.start()
d.addCallback(lambda _: webbrowser.open(UI_ADDRESS)) d.addCallback(lambda _: webbrowser.open(UI_ADDRESS))
reactor.listenTCP(API_PORT, server.Site(lbry.root), interface=API_INTERFACE) lbrynet_server = server.Site(lbry.root)
lbrynet_server.requestFactory = LBRYDaemonRequest
reactor.listenTCP(API_PORT, lbrynet_server, interface=API_INTERFACE)
def openui_(self, sender): def openui_(self, sender):
webbrowser.open(UI_ADDRESS) webbrowser.open(UI_ADDRESS)

5
packaging/osx/lbry-osx-app/setup_app.sh
View file

@ -70,7 +70,10 @@ fi
# add lbrycrdd as a resource. Following # add lbrycrdd as a resource. Following
# http://stackoverflow.com/questions/11370012/can-executables-made-with-py2app-include-other-terminal-scripts-and-run-them # http://stackoverflow.com/questions/11370012/can-executables-made-with-py2app-include-other-terminal-scripts-and-run-them
wget https://github.com/lbryio/lbrycrd/releases/download/v0.3-osx/lbrycrdd # LBRYCRDD_URL="$(curl https://api.github.com/repos/lbryio/lbrycrd/releases/latest | grep 'browser_download_url' | grep osx | cut -d'"' -f4)"
LBRYCRDD_URL="https://github.com/lbryio/lbrycrd/releases/download/v0.3.15/lbrycrd-osx.zip"
wget "${LBRYCRDD_URL}" --output-document lbrycrd-osx.zip
unzip lbrycrd-osx.zip
python setup_app.py py2app --resources lbrycrdd python setup_app.py py2app --resources lbrycrdd
chmod +x "${DEST}/dist/LBRY.app/Contents/Resources/lbrycrdd" chmod +x "${DEST}/dist/LBRY.app/Contents/Resources/lbrycrdd"

3
packaging/travis/install_dependencies_and_run_tests.sh
View file

@ -44,4 +44,5 @@ trial tests
# Ignoring distutils because: https://github.com/PyCQA/pylint/issues/73 # Ignoring distutils because: https://github.com/PyCQA/pylint/issues/73
# TODO: as code quality improves, make pylint be more strict # TODO: as code quality improves, make pylint be more strict
pylint -E --disable=inherit-non-class --disable=no-member --ignored-modules=distutils lbrynet pylint -E --disable=inherit-non-class --disable=no-member --ignored-modules=distutils \
--enable=unused-import lbrynet

2
packaging/ubuntu/lbry.desktop
View file

@ -1,5 +1,5 @@
[Desktop Entry] [Desktop Entry]
Version=0.3.12 Version=0.3.17
Name=LBRY Name=LBRY
Comment=The world's first user-owned content marketplace Comment=The world's first user-owned content marketplace
Icon=lbry Icon=lbry

2
packaging/ubuntu/ubuntu_package_setup.sh
View file

@ -170,7 +170,7 @@ addfile "$PACKAGING_DIR/lbry-temp-symlink" usr/bin/lbry
# add lbrycrdd and lbrycrd-cli # add lbrycrdd and lbrycrd-cli
mkdir -p "$PACKAGING_DIR/bins" mkdir -p "$PACKAGING_DIR/bins"
wget http://s3.amazonaws.com/files.lbry.io/bins.zip --output-document "$PACKAGING_DIR/bins.zip" wget "$(curl https://api.github.com/repos/lbryio/lbrycrd/releases/latest | grep 'browser_download_url' | grep linux | cut -d'"' -f4)" --output-document "$PACKAGING_DIR/bins.zip"
unzip "$PACKAGING_DIR/bins.zip" -d "$PACKAGING_DIR/bins/" unzip "$PACKAGING_DIR/bins.zip" -d "$PACKAGING_DIR/bins/"
addfile "$PACKAGING_DIR/bins/lbrycrdd" usr/bin/lbrycrdd addfile "$PACKAGING_DIR/bins/lbrycrdd" usr/bin/lbrycrdd
addfile "$PACKAGING_DIR/bins/lbrycrd-cli" usr/bin/lbrycrd-cli addfile "$PACKAGING_DIR/bins/lbrycrd-cli" usr/bin/lbrycrd-cli

3
requirements.txt
View file

@ -9,7 +9,7 @@ gmpy==1.17
jsonrpc==1.2 jsonrpc==1.2
jsonrpclib==0.1.7 jsonrpclib==0.1.7
https://github.com/lbryio/lbryum/tarball/master/#egg=lbryum https://github.com/lbryio/lbryum/tarball/master/#egg=lbryum
leveldb==0.193 loggly-python-handler==1.0.0
miniupnpc==1.9 miniupnpc==1.9
pbkdf2==1.3 pbkdf2==1.3
protobuf==3.0.0b3 protobuf==3.0.0b3
@ -17,6 +17,7 @@ pycrypto==2.6.1
python-bitcoinrpc==0.1 python-bitcoinrpc==0.1
qrcode==5.2.2 qrcode==5.2.2
requests==2.9.1 requests==2.9.1
requests_futures==0.9.7
seccure==0.3.1.3 seccure==0.3.1.3
simplejson==3.8.2 simplejson==3.8.2
six==1.9.0 six==1.9.0

2
setup.py
View file

@ -25,7 +25,7 @@ console_scripts = ['lbrynet-stdin-uploader = lbrynet.lbrynet_console.LBRYStdinUp
requires = ['pycrypto', 'twisted', 'miniupnpc', 'yapsy', 'seccure', requires = ['pycrypto', 'twisted', 'miniupnpc', 'yapsy', 'seccure',
'python-bitcoinrpc==0.1', 'txJSON-RPC', 'requests>=2.4.2', 'unqlite==0.2.0', 'python-bitcoinrpc==0.1', 'txJSON-RPC', 'requests>=2.4.2', 'unqlite==0.2.0',
'leveldb', 'lbryum', 'jsonrpc', 'simplejson', 'appdirs', 'six==1.9.0', 'base58', 'googlefinance'] 'leveldb', 'lbryum', 'jsonrpc', 'simplejson', 'appdirs', 'six==1.9.0', 'base58', 'googlefinance', 'requests_futures']
setup(name='lbrynet', setup(name='lbrynet',
description='A decentralized media library and marketplace', description='A decentralized media library and marketplace',