LBRYCommunity/lbry-sdk
1
0
Fork 1
Code Issues Pull requests Projects Releases 282 Packages Wiki Activity

Merge pull request #113 from lbryio/log-to-loggly

Browse source 
Log to loggly
This commit is contained in:
Jack Robison 2016-08-09 02:30:55 -04:00 • committed by GitHub
commit 8d14fe6a90
6 changed files with 103 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
lbrynet/conf.py
View file

@ -58,3 +58,5 @@ CURRENCIES = {
'LBC': {'type': 'crypto'},
'USD': {'type': 'fiat'},
}
LOGGLY_TOKEN = 'YWRmNGU4NmEtNjkwNC00YjM2LTk3ZjItMGZhODM3ZDhkYzBi'

73
lbrynet/core/log_support.py
View file

@ -1,24 +1,81 @@
import base64
import json
import logging
import logging.handlers
import sys
import loggly.handlers
import lbrynet
from lbrynet import conf
DEFAULT_FORMAT = "%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s"
DEFAULT_FORMATTER = logging.Formatter(DEFAULT_FORMAT)
LOGGLY_URL = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}"
def configureConsole(log=None, level=logging.INFO):
def remove_handlers(log, handler_name):
for handler in log.handlers:
if handler.name == handler_name:
log.removeHandler(handler)
def _log_decorator(fn):
def helper(*args, **kwargs):
log = kwargs.pop('log', logging.getLogger())
level = kwargs.pop('level', logging.INFO)
handler = fn(*args, **kwargs)
if handler.name:
remove_handlers(log, handler.name)
log.addHandler(handler)
log.setLevel(level)
return helper
def disable_noisy_loggers():
logging.getLogger('requests').setLevel(logging.WARNING)
@_log_decorator
def configure_console(**kwargs):
"""Convenience function to configure a logger that outputs to stdout"""
log = log or logging.getLogger()
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(DEFAULT_FORMATTER)
log.addHandler(handler)
log.setLevel(level=level)
handler.name = 'console'
return handler
def configureFileHandler(file_name, log=None, level=logging.INFO):
log = log or logging.getLogger()
@_log_decorator
def configure_file_handler(file_name, **kwargs):
handler = logging.handlers.RotatingFileHandler(file_name, maxBytes=2097152, backupCount=5)
handler.setFormatter(DEFAULT_FORMATTER)
log.addHandler(handler)
log.setLevel(level=level)
handler.name = 'file'
return handler
def get_loggly_url(token=None, version=None):
token = token or base64.b64decode(conf.LOGGLY_TOKEN)
version = version or lbrynet.__version__
return LOGGLY_URL.format(token=token, tag='lbrynet-' + version)
@_log_decorator
def configure_loggly_handler(url=None, **kwargs):
url = url or get_loggly_url()
json_format = {
"loggerName": "%(name)s",
"asciTime": "%(asctime)s",
"fileName": "%(filename)s",
"functionName": "%(funcName)s",
"levelNo": "%(levelno)s",
"lineNo": "%(lineno)d",
"levelName": "%(levelname)s",
"message": "%(message)s",
}
json_format.update(kwargs)
formatter = logging.Formatter(json.dumps(json_format))
handler = loggly.handlers.HTTPSHandler(url)
handler.setFormatter(formatter)
handler.name = 'loggly'
return handler

4
lbrynet/core/server/ServerRequestHandler.py
View file

@ -99,8 +99,8 @@ class ServerRequestHandler(object):
d.addCallback(lambda _: self.blob_sender.send_blob_if_requested(self))
d.addCallbacks(lambda _: self.finished_response(), self.request_failure_handler)
else:
log.info("Request buff not a valid json message")
log.info("Request buff: %s", str(self.request_buff))
log.debug("Request buff not a valid json message")
log.debug("Request buff: %s", str(self.request_buff))
else:
log.warning("The client sent data when we were uploading a file. This should not happen")

100
lbrynet/lbrynet_daemon/LBRYDaemon.py
View file

@ -39,6 +39,7 @@ from lbrynet.lbrynet_daemon.LBRYDownloader import GetStream
from lbrynet.lbrynet_daemon.LBRYPublisher import Publisher
from lbrynet.lbrynet_daemon.LBRYExchangeRateManager import ExchangeRateManager
from lbrynet.lbrynet_daemon.Lighthouse import LighthouseClient
from lbrynet.core import log_support
from lbrynet.core import utils
from lbrynet.core.LBRYMetadata import verify_name_characters
from lbrynet.core.utils import generate_id
@ -318,14 +319,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
else:
self.wallet_dir = os.path.join(get_path(FOLDERID.RoamingAppData, UserHandle.current), "lbryum")
elif sys.platform == "darwin":
# use the path from the bundle if its available.
try:
import Foundation
bundle = Foundation.NSBundle.mainBundle()
self.lbrycrdd_path = bundle.pathForResource_ofType_('lbrycrdd', None)
except Exception:
log.exception('Failed to get path from bundle, falling back to default')
self.lbrycrdd_path = "./lbrycrdd"
self.lbrycrdd_path = get_darwin_lbrycrdd_path()
if self.wallet_type == "lbrycrd":
self.wallet_dir = user_data_dir("lbrycrd")
else:
@ -483,8 +477,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
log.info("Scheduling scripts")
reactor.callLater(3, self._run_scripts)
# self.lbrynet_connection_checker.start(3600)
if self.first_run:
d = self._upload_log(log_type="first_run")
elif self.upload_log:
@ -492,11 +484,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
else:
d = defer.succeed(None)
# if float(self.session.wallet.wallet_balance) == 0.0:
# d.addCallback(lambda _: self._check_first_run())
# d.addCallback(self._show_first_run_result)
# d.addCallback(lambda _: _wait_for_credits() if self.requested_first_run_credits else _announce())
d.addCallback(lambda _: _announce())
return d
@ -931,6 +918,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
d = self.settings.start()
d.addCallback(lambda _: self.settings.get_lbryid())
d.addCallback(self._set_lbryid)
d.addCallback(lambda _: self._modify_loggly_formatter())
return d
def _set_lbryid(self, lbryid):
@ -946,6 +934,14 @@ class LBRYDaemon(jsonrpc.JSONRPC):
d = self.settings.save_lbryid(self.lbryid)
return d
def _modify_loggly_formatter(self):
session_id = base58.b58encode(generate_id())
log_support.configure_loggly_handler(
lbry_id=base58.b58encode(self.lbryid),
session_id=session_id
)
def _setup_lbry_file_manager(self):
self.startup_status = STARTUP_STAGES[3]
self.lbry_file_metadata_manager = DBLBRYFileMetadataManager(self.db_dir)
@ -1012,62 +1008,6 @@ class LBRYDaemon(jsonrpc.JSONRPC):
return dl
# def _check_first_run(self):
# def _set_first_run_false():
# log.info("Not first run")
# self.first_run = False
# self.session_settings['requested_first_run_credits'] = True
# f = open(self.daemon_conf, "w")
# f.write(json.dumps(self.session_settings))
# f.close()
# return 0.0
#
# if self.wallet_type == 'lbryum':
# d = self.session.wallet.is_first_run()
# d.addCallback(lambda is_first_run: self._do_first_run() if is_first_run or not self.requested_first_run_credits
# else _set_first_run_false())
# else:
# d = defer.succeed(None)
# d.addCallback(lambda _: _set_first_run_false())
# return d
#
# def _do_first_run(self):
# def send_request(url, data):
# log.info("Requesting first run credits")
# r = requests.post(url, json=data)
# if r.status_code == 200:
# self.requested_first_run_credits = True
# self.session_settings['requested_first_run_credits'] = True
# f = open(self.daemon_conf, "w")
# f.write(json.dumps(self.session_settings))
# f.close()
# return r.json()['credits_sent']
# return 0.0
#
# def log_error(err):
# log.warning("unable to request free credits. %s", err.getErrorMessage())
# return 0.0
#
# def request_credits(address):
# url = "http://credreq.lbry.io/requestcredits"
# data = {"address": address}
# d = threads.deferToThread(send_request, url, data)
# d.addErrback(log_error)
# return d
#
# self.first_run = True
# d = self.session.wallet.get_new_address()
# d.addCallback(request_credits)
#
# return d
#
# def _show_first_run_result(self, credits_received):
# if credits_received != 0.0:
# points_string = locale.format_string("%.2f LBC", (round(credits_received, 2),), grouping=True)
# self.startup_message = "Thank you for testing the alpha version of LBRY! You have been given %s for free because we love you. Please hang on for a few minutes for the next block to be mined. When you refresh this page and see your credits you're ready to go!." % points_string
# else:
# self.startup_message = None
def _setup_stream_identifier(self):
file_saver_factory = LBRYFileSaverFactory(self.session.peer_finder, self.session.rate_limiter,
self.session.blob_manager, self.stream_info_manager,
@ -2373,6 +2313,24 @@ def get_output_callback(params):
return callback
def get_darwin_lbrycrdd_path():
# use the path from the bundle if its available.
default = "./lbrycrdd"
try:
import Foundation
except ImportError:
log.warning('Foundation module not installed, falling back to default lbrycrdd path')
return default
else:
try:
bundle = Foundation.NSBundle.mainBundle()
return bundle.pathForResource_ofType_('lbrycrdd', None)
except Exception:
log.exception('Failed to get path from bundle, falling back to default')
return default
class _DownloadNameHelper(object):
def __init__(self, daemon, name, timeout=DEFAULT_TIMEOUT, download_directory=None,
file_name=None, wait_for_write=True):

8
lbrynet/lbrynet_daemon/LBRYDaemonControl.py
View file

@ -73,11 +73,11 @@ def start():
parser.set_defaults(branch=False, launchui=True, logtoconsole=False, quiet=False)
args = parser.parse_args()
log_support.configureFileHandler(lbrynet_log)
log_support.disable_noisy_loggers()
log_support.configure_file_handler(lbrynet_log)
log_support.configure_loggly_handler()
if args.logtoconsole:
log_support.configureConsole()
log_support.configure_console()
try:
JSONRPCProxy.from_url(API_CONNECTION_STRING).is_running()

1
requirements.txt
View file

@ -9,6 +9,7 @@ gmpy==1.17
jsonrpc==1.2
jsonrpclib==0.1.7
https://github.com/lbryio/lbryum/tarball/master/#egg=lbryum
loggly-python-handler==1.0.0
miniupnpc==1.9
pbkdf2==1.3
protobuf==3.0.0b3