Madiator2011/lbry-sdk
1
0
Fork 0
forked from LBRYCommunity/lbry-sdk
Code Pull requests Activity

Merge branch 'master' into update-claim

Browse source 
# Conflicts:
#	lbrynet/lbrynet_daemon/LBRYDaemon.py
This commit is contained in:
Jack 2016-08-09 03:09:06 -04:00
commit a05ab37373
21 changed files with 158 additions and 133 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
lbrynet/conf.py
View file

@ -58,3 +58,5 @@ CURRENCIES = {
'LBC': {'type': 'crypto'},
'USD': {'type': 'fiat'},
}
LOGGLY_TOKEN = 'YWRmNGU4NmEtNjkwNC00YjM2LTk3ZjItMGZhODM3ZDhkYzBi'

3
lbrynet/core/LBRYMetadata.py
View file

@ -1,9 +1,6 @@
import requests
import json
import time
from copy import deepcopy
from googlefinance import getQuotes
from lbrynet.conf import CURRENCIES
from lbrynet.core import utils
import logging

6
lbrynet/core/LBRYWallet.py
View file

@ -6,7 +6,6 @@ import subprocess
import socket
import time
import os
import requests
from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException
from twisted.internet import threads, reactor, defer, task
@ -15,10 +14,9 @@ from twisted.enterprise import adbapi
from collections import defaultdict, deque
from zope.interface import implements
from decimal import Decimal
from googlefinance import getQuotes
from lbryum import SimpleConfig, Network
from lbryum.lbrycrd import COIN, TYPE_ADDRESS
from lbryum.lbrycrd import COIN
from lbryum.wallet import WalletStorage, Wallet
from lbryum.commands import known_commands, Commands
from lbryum.transaction import Transaction
@ -27,8 +25,6 @@ from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHand
from lbrynet.core.client.ClientRequest import ClientRequest
from lbrynet.core.Error import UnknownNameError, InvalidStreamInfoError, RequestCanceledError
from lbrynet.core.Error import InsufficientFundsError
from lbrynet.core.sqlite_helpers import rerun_if_locked
from lbrynet.conf import SOURCE_TYPES
from lbrynet.core.LBRYMetadata import Metadata
log = logging.getLogger(__name__)

103
lbrynet/core/log_support.py
View file

@ -1,24 +1,111 @@
import base64
import json
import logging
import logging.handlers
import sys
import traceback
import lbrynet
from lbrynet import conf
from requests_futures.sessions import FuturesSession
session = FuturesSession()
def bg_cb(sess, resp):
""" Don't do anything with the response """
pass
class HTTPSHandler(logging.Handler):
def __init__(self, url, fqdn=False, localname=None, facility=None):
logging.Handler.__init__(self)
self.url = url
self.fqdn = fqdn
self.localname = localname
self.facility = facility
def get_full_message(self, record):
if record.exc_info:
return '\n'.join(traceback.format_exception(*record.exc_info))
else:
return record.getMessage()
def emit(self, record):
try:
payload = self.format(record)
session.post(self.url, data=payload, background_callback=bg_cb)
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
DEFAULT_FORMAT = "%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s"
DEFAULT_FORMATTER = logging.Formatter(DEFAULT_FORMAT)
LOGGLY_URL = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}"
def configureConsole(log=None, level=logging.INFO):
def remove_handlers(log, handler_name):
for handler in log.handlers:
if handler.name == handler_name:
log.removeHandler(handler)
def _log_decorator(fn):
def helper(*args, **kwargs):
log = kwargs.pop('log', logging.getLogger())
level = kwargs.pop('level', logging.INFO)
handler = fn(*args, **kwargs)
if handler.name:
remove_handlers(log, handler.name)
log.addHandler(handler)
log.setLevel(level)
return helper
def disable_noisy_loggers():
logging.getLogger('requests').setLevel(logging.WARNING)
@_log_decorator
def configure_console(**kwargs):
"""Convenience function to configure a logger that outputs to stdout"""
log = log or logging.getLogger()
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(DEFAULT_FORMATTER)
log.addHandler(handler)
log.setLevel(level=level)
handler.name = 'console'
return handler
def configureFileHandler(file_name, log=None, level=logging.INFO):
log = log or logging.getLogger()
@_log_decorator
def configure_file_handler(file_name, **kwargs):
handler = logging.handlers.RotatingFileHandler(file_name, maxBytes=2097152, backupCount=5)
handler.setFormatter(DEFAULT_FORMATTER)
log.addHandler(handler)
log.setLevel(level=level)
handler.name = 'file'
return handler
def get_loggly_url(token=None, version=None):
token = token or base64.b64decode(conf.LOGGLY_TOKEN)
version = version or lbrynet.__version__
return LOGGLY_URL.format(token=token, tag='lbrynet-' + version)
@_log_decorator
def configure_loggly_handler(url=None, **kwargs):
url = url or get_loggly_url()
json_format = {
"loggerName": "%(name)s",
"asciTime": "%(asctime)s",
"fileName": "%(filename)s",
"functionName": "%(funcName)s",
"levelNo": "%(levelno)s",
"lineNo": "%(lineno)d",
"levelName": "%(levelname)s",
"message": "%(message)s",
}
json_format.update(kwargs)
formatter = logging.Formatter(json.dumps(json_format))
handler = HTTPSHandler(url)
handler.setFormatter(formatter)
handler.name = 'loggly'
return handler

4
lbrynet/core/server/DHTHashAnnouncer.py
View file

@ -1,5 +1,5 @@
import binascii
from twisted.internet import defer, task, reactor
from twisted.internet import defer, reactor
import collections
@ -78,4 +78,4 @@ class DHTHashSupplier(object):
self.hash_reannounce_time = 60 * 60 # 1 hour
def hashes_to_announce(self):
pass
pass

4
lbrynet/core/server/ServerRequestHandler.py
View file

@ -99,8 +99,8 @@ class ServerRequestHandler(object):
d.addCallback(lambda _: self.blob_sender.send_blob_if_requested(self))
d.addCallbacks(lambda _: self.finished_response(), self.request_failure_handler)
else:
log.info("Request buff not a valid json message")
log.info("Request buff: %s", str(self.request_buff))
log.debug("Request buff not a valid json message")
log.debug("Request buff: %s", str(self.request_buff))
else:
log.warning("The client sent data when we were uploading a file. This should not happen")

3
lbrynet/dht/datastore.py
View file

@ -8,10 +8,7 @@
# may be created by processing this file with epydoc: http://epydoc.sf.net
import UserDict
#import sqlite3
import cPickle as pickle
import time
import os
import constants

4
lbrynet/dht/node.py
View file

@ -7,7 +7,7 @@
# The docstrings in this module contain epytext markup; API documentation
# may be created by processing this file with epydoc: http://epydoc.sf.net
import hashlib, random, struct, time, math, binascii
import hashlib, random, struct, time, binascii
import argparse
from twisted.internet import defer, error
import constants
@ -1017,4 +1017,4 @@ def main():
twisted.internet.reactor.run()
if __name__ == '__main__':
main()
main()

4
lbrynet/dhttest.py
View file

@ -20,7 +20,7 @@
import os, sys, time, signal, hashlib, random
import sys, hashlib, random
import twisted.internet.reactor
from lbrynet.dht.node import Node
#from entangled.kademlia.datastore import SQLiteDataStore
@ -106,7 +106,7 @@ def stop():
if __name__ == '__main__':
import sys, os
import sys
if len(sys.argv) < 2:
print 'Usage:\n%s UDP_PORT [KNOWN_NODE_IP KNOWN_NODE_PORT]' % sys.argv[0]
print 'or:\n%s UDP_PORT [FILE_WITH_KNOWN_NODES]' % sys.argv[0]

3
lbrynet/lbryfilemanager/LBRYFileCreator.py
View file

@ -11,7 +11,6 @@ from lbrynet import conf
from lbrynet.lbryfile.StreamDescriptor import get_sd_info
from lbrynet.core.cryptoutils import get_lbry_hash_obj
from twisted.protocols.basic import FileSender
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader
log = logging.getLogger(__name__)
@ -155,4 +154,4 @@ def create_lbry_file(session, lbry_file_manager, file_name, file_handle, key=Non
d = lbry_file_creator.setup()
d.addCallback(lambda _: start_stream())
return d
return d

8
lbrynet/lbrynet_console/ControlHandlers.py
View file

@ -1,18 +1,12 @@
import json
import logging
from time import sleep
from bitcoinrpc.authproxy import AuthServiceProxy
from twisted.internet.task import LoopingCall
from zope.interface import implements
#from lbrynet.core.StreamDescriptor import PlainStreamDescriptorWriter, BlobStreamDescriptorWriter
from lbrynet.core.PaymentRateManager import PaymentRateManager
from lbrynet.lbryfilemanager.LBRYFileCreator import create_lbry_file
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader
# from lbrynet.lbryfile.StreamDescriptor import get_sd_info
from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob, create_plain_sd
from lbrynet.lbrynet_console.interfaces import ICommandHandler, ICommandHandlerFactory
from lbrynet.core.StreamDescriptor import download_sd_blob#, BlobStreamDescriptorReader
from lbrynet.core.StreamDescriptor import download_sd_blob
from lbrynet.core.Error import UnknownNameError, InvalidBlobHashError, InsufficientFundsError
from lbrynet.core.Error import InvalidStreamInfoError
from lbrynet.core.utils import is_valid_blobhash

113
lbrynet/lbrynet_daemon/LBRYDaemon.py
View file

@ -1,6 +1,4 @@
import binascii
import distutils.version
import locale
import logging.handlers
import mimetypes
import os
@ -14,7 +12,6 @@ import sys
import base58
import requests
import simplejson as json
import pkg_resources
from urllib2 import urlopen
from appdirs import user_data_dir
@ -25,7 +22,7 @@ from twisted.internet import defer, threads, error, reactor
from twisted.internet.task import LoopingCall
from txjsonrpc import jsonrpclib
from txjsonrpc.web import jsonrpc
from txjsonrpc.web.jsonrpc import Handler, Proxy
from txjsonrpc.web.jsonrpc import Handler
from lbrynet import __version__ as lbrynet_version
from lbryum.version import LBRYUM_VERSION as lbryum_version
@ -43,14 +40,17 @@ from lbrynet.lbrynet_daemon.LBRYPublisher import Publisher
from lbrynet.lbrynet_daemon.LBRYExchangeRateManager import ExchangeRateManager
from lbrynet.lbrynet_daemon.Lighthouse import LighthouseClient
from lbrynet.core.LBRYMetadata import Metadata
from lbrynet.core import log_support
from lbrynet.core import utils
from lbrynet.core.LBRYMetadata import verify_name_characters
from lbrynet.core.utils import generate_id
from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings
from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, DEFAULT_MAX_SEARCH_RESULTS, KNOWN_DHT_NODES, DEFAULT_MAX_KEY_FEE, \
DEFAULT_WALLET, DEFAULT_SEARCH_TIMEOUT, DEFAULT_CACHE_TIME, DEFAULT_UI_BRANCH, LOG_POST_URL, LOG_FILE_NAME, SOURCE_TYPES
from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, DEFAULT_MAX_SEARCH_RESULTS, \
KNOWN_DHT_NODES, DEFAULT_MAX_KEY_FEE, DEFAULT_WALLET, \
DEFAULT_SEARCH_TIMEOUT, DEFAULT_CACHE_TIME, DEFAULT_UI_BRANCH, \
LOG_POST_URL, LOG_FILE_NAME
from lbrynet.conf import DEFAULT_SD_DOWNLOAD_TIMEOUT
from lbrynet.conf import DEFAULT_TIMEOUT, WALLET_TYPES
from lbrynet.conf import DEFAULT_TIMEOUT
from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob, BlobStreamDescriptorReader
from lbrynet.core.Session import LBRYSession
from lbrynet.core.PTCWallet import PTCWallet
@ -320,14 +320,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
else:
self.wallet_dir = os.path.join(get_path(FOLDERID.RoamingAppData, UserHandle.current), "lbryum")
elif sys.platform == "darwin":
# use the path from the bundle if its available.
try:
import Foundation
bundle = Foundation.NSBundle.mainBundle()
self.lbrycrdd_path = bundle.pathForResource_ofType_('lbrycrdd', None)
except Exception:
log.exception('Failed to get path from bundle, falling back to default')
self.lbrycrdd_path = "./lbrycrdd"
self.lbrycrdd_path = get_darwin_lbrycrdd_path()
if self.wallet_type == "lbrycrd":
self.wallet_dir = user_data_dir("lbrycrd")
else:
@ -486,8 +479,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
log.info("Scheduling scripts")
reactor.callLater(3, self._run_scripts)
# self.lbrynet_connection_checker.start(3600)
if self.first_run:
d = self._upload_log(log_type="first_run")
elif self.upload_log:
@ -495,11 +486,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
else:
d = defer.succeed(None)
# if float(self.session.wallet.wallet_balance) == 0.0:
# d.addCallback(lambda _: self._check_first_run())
# d.addCallback(self._show_first_run_result)
# d.addCallback(lambda _: _wait_for_credits() if self.requested_first_run_credits else _announce())
d.addCallback(lambda _: _announce())
return d
@ -934,6 +920,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
d = self.settings.start()
d.addCallback(lambda _: self.settings.get_lbryid())
d.addCallback(self._set_lbryid)
d.addCallback(lambda _: self._modify_loggly_formatter())
return d
def _set_lbryid(self, lbryid):
@ -949,6 +936,14 @@ class LBRYDaemon(jsonrpc.JSONRPC):
d = self.settings.save_lbryid(self.lbryid)
return d
def _modify_loggly_formatter(self):
session_id = base58.b58encode(generate_id())
log_support.configure_loggly_handler(
lbry_id=base58.b58encode(self.lbryid),
session_id=session_id
)
def _setup_lbry_file_manager(self):
self.startup_status = STARTUP_STAGES[3]
self.lbry_file_metadata_manager = DBLBRYFileMetadataManager(self.db_dir)
@ -1015,62 +1010,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
return dl
# def _check_first_run(self):
# def _set_first_run_false():
# log.info("Not first run")
# self.first_run = False
# self.session_settings['requested_first_run_credits'] = True
# f = open(self.daemon_conf, "w")
# f.write(json.dumps(self.session_settings))
# f.close()
# return 0.0
#
# if self.wallet_type == 'lbryum':
# d = self.session.wallet.is_first_run()
# d.addCallback(lambda is_first_run: self._do_first_run() if is_first_run or not self.requested_first_run_credits
# else _set_first_run_false())
# else:
# d = defer.succeed(None)
# d.addCallback(lambda _: _set_first_run_false())
# return d
#
# def _do_first_run(self):
# def send_request(url, data):
# log.info("Requesting first run credits")
# r = requests.post(url, json=data)
# if r.status_code == 200:
# self.requested_first_run_credits = True
# self.session_settings['requested_first_run_credits'] = True
# f = open(self.daemon_conf, "w")
# f.write(json.dumps(self.session_settings))
# f.close()
# return r.json()['credits_sent']
# return 0.0
#
# def log_error(err):
# log.warning("unable to request free credits. %s", err.getErrorMessage())
# return 0.0
#
# def request_credits(address):
# url = "http://credreq.lbry.io/requestcredits"
# data = {"address": address}
# d = threads.deferToThread(send_request, url, data)
# d.addErrback(log_error)
# return d
#
# self.first_run = True
# d = self.session.wallet.get_new_address()
# d.addCallback(request_credits)
#
# return d
#
# def _show_first_run_result(self, credits_received):
# if credits_received != 0.0:
# points_string = locale.format_string("%.2f LBC", (round(credits_received, 2),), grouping=True)
# self.startup_message = "Thank you for testing the alpha version of LBRY! You have been given %s for free because we love you. Please hang on for a few minutes for the next block to be mined. When you refresh this page and see your credits you're ready to go!." % points_string
# else:
# self.startup_message = None
def _setup_stream_identifier(self):
file_saver_factory = LBRYFileSaverFactory(self.session.peer_finder, self.session.rate_limiter,
self.session.blob_manager, self.stream_info_manager,
@ -2423,6 +2362,24 @@ def get_output_callback(params):
return callback
def get_darwin_lbrycrdd_path():
# use the path from the bundle if its available.
default = "./lbrycrdd"
try:
import Foundation
except ImportError:
log.warning('Foundation module not installed, falling back to default lbrycrdd path')
return default
else:
try:
bundle = Foundation.NSBundle.mainBundle()
return bundle.pathForResource_ofType_('lbrycrdd', None)
except Exception:
log.exception('Failed to get path from bundle, falling back to default')
return default
class _DownloadNameHelper(object):
def __init__(self, daemon, name, timeout=DEFAULT_TIMEOUT, download_directory=None,
file_name=None, wait_for_write=True):

2
lbrynet/lbrynet_daemon/LBRYDaemonCLI.py
View file

@ -1,7 +1,7 @@
import sys
import json
from lbrynet.conf import API_CONNECTION_STRING, LOG_FILE_NAME
from lbrynet.conf import API_CONNECTION_STRING
from jsonrpc.proxy import JSONRPCProxy
help_msg = "Useage: lbrynet-cli method json-args\n" \

13
lbrynet/lbrynet_daemon/LBRYDaemonControl.py
View file

@ -5,7 +5,6 @@ import os
import webbrowser
import sys
import socket
import platform
from appdirs import user_data_dir
from twisted.web import server
@ -14,8 +13,8 @@ from jsonrpc.proxy import JSONRPCProxy
from lbrynet.core import log_support
from lbrynet.lbrynet_daemon.LBRYDaemonServer import LBRYDaemonServer, LBRYDaemonRequest
from lbrynet.conf import API_CONNECTION_STRING, API_INTERFACE, API_ADDRESS, API_PORT, \
DEFAULT_WALLET, UI_ADDRESS, DEFAULT_UI_BRANCH, LOG_FILE_NAME
from lbrynet.conf import API_CONNECTION_STRING, API_INTERFACE, API_PORT, \
UI_ADDRESS, DEFAULT_UI_BRANCH, LOG_FILE_NAME
# TODO: stop it!
if sys.platform != "darwin":
@ -74,11 +73,11 @@ def start():
parser.set_defaults(branch=False, launchui=True, logtoconsole=False, quiet=False)
args = parser.parse_args()
log_support.configureFileHandler(lbrynet_log)
log_support.disable_noisy_loggers()
log_support.configure_file_handler(lbrynet_log)
log_support.configure_loggly_handler()
if args.logtoconsole:
log_support.configureConsole()
log_support.configure_console()
try:
JSONRPCProxy.from_url(API_CONNECTION_STRING).is_running()

5
lbrynet/lbrynet_daemon/LBRYDaemonServer.py
View file

@ -9,15 +9,14 @@ import tempfile
import time
import cgi
from datetime import datetime
from appdirs import user_data_dir
from twisted.web import server, static, resource
from twisted.internet import defer, interfaces, error, reactor, task, threads
from twisted.internet import defer, interfaces, error, reactor, threads
from zope.interface import implements
from lbrynet.lbrynet_daemon.LBRYDaemon import LBRYDaemon
from lbrynet.conf import API_CONNECTION_STRING, API_ADDRESS, DEFAULT_WALLET, UI_ADDRESS, DEFAULT_UI_BRANCH, LOG_FILE_NAME
from lbrynet.conf import API_ADDRESS, UI_ADDRESS, DEFAULT_UI_BRANCH, LOG_FILE_NAME
# TODO: omg, this code is essentially duplicated in LBRYDaemon

5
lbrynet/lbrynet_daemon/LBRYDownloader.py
View file

@ -5,14 +5,13 @@ import sys
from copy import deepcopy
from appdirs import user_data_dir
from datetime import datetime
from twisted.internet import defer
from twisted.internet.task import LoopingCall
from lbrynet.core.Error import InvalidStreamInfoError, InsufficientFundsError, KeyFeeAboveMaxAllowed
from lbrynet.core.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed
from lbrynet.core.PaymentRateManager import PaymentRateManager
from lbrynet.core.StreamDescriptor import download_sd_blob
from lbrynet.core.LBRYMetadata import Metadata, LBRYFeeValidator
from lbrynet.core.LBRYMetadata import LBRYFeeValidator
from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloaderFactory
from lbrynet.conf import DEFAULT_TIMEOUT, LOG_FILE_NAME

1
lbrynet/lbrynet_daemon/LBRYPublisher.py
View file

@ -4,7 +4,6 @@ import os
import sys
from appdirs import user_data_dir
from datetime import datetime
from lbrynet.core.Error import InsufficientFundsError
from lbrynet.lbryfilemanager.LBRYFileCreator import create_lbry_file

1
lbrynet/lbrynet_daemon/Lighthouse.py
View file

@ -1,7 +1,6 @@
import logging
import random
from txjsonrpc.web.jsonrpc import Proxy
from twisted.internet import defer
from lbrynet.conf import SEARCH_SERVERS
log = logging.getLogger(__name__)

3
lbrynet/node_rpc_cli.py
View file

@ -6,7 +6,6 @@ CLI for sending rpc commands to a DHT node
from twisted.internet import reactor
from txjsonrpc.web.jsonrpc import Proxy
import argparse
import sys
def print_value(value):
@ -39,4 +38,4 @@ def main():
d = proxy.callRemote(args.rpc_command)
d.addCallbacks(print_value, print_error)
d.addBoth(lambda _: shut_down())
reactor.run()
reactor.run()

3
packaging/travis/install_dependencies_and_run_tests.sh
View file

@ -44,4 +44,5 @@ trial tests
# Ignoring distutils because: https://github.com/PyCQA/pylint/issues/73
# TODO: as code quality improves, make pylint be more strict
pylint -E --disable=inherit-non-class --disable=no-member --ignored-modules=distutils lbrynet
pylint -E --disable=inherit-non-class --disable=no-member --ignored-modules=distutils \
--enable=unused-import lbrynet

1
requirements.txt
View file

@ -9,6 +9,7 @@ gmpy==1.17
jsonrpc==1.2
jsonrpclib==0.1.7
https://github.com/lbryio/lbryum/tarball/master/#egg=lbryum
loggly-python-handler==1.0.0
miniupnpc==1.9
pbkdf2==1.3
protobuf==3.0.0b3