Merge branch 'master' into content_type

This commit is contained in:
Jack 2016-08-17 03:04:14 -04:00
commit 68b8112af0
23 changed files with 325 additions and 124 deletions

View file

@ -1,5 +1,5 @@
[bumpversion]
current_version = 0.3.17
current_version = 0.3.18
commit = True
tag = True
message = Bump version: {current_version} -> {new_version}

2
.gitignore vendored
View file

@ -28,3 +28,5 @@ lbrynet.egg-info/PKG-INFO
# temporary files from the twisted.trial test runner
_trial_temp/
.DS_Store

View file

@ -1,8 +1,2 @@
import logging
log = logging.getLogger(__name__)
logging.getLogger(__name__).addHandler(logging.NullHandler())
log.setLevel(logging.INFO)
__version__ = "0.3.17"
__version__ = "0.3.18"
version = tuple(__version__.split('.'))

View file

@ -0,0 +1,2 @@
from events import *
from api import AnalyticsApi as Api

71
lbrynet/analytics/api.py Normal file
View file

@ -0,0 +1,71 @@
import functools
import json
import logging
from requests import auth
from requests_futures import sessions
from lbrynet import conf
from lbrynet.analytics import utils
log = logging.getLogger(__name__)
def log_response(fn):
def _log(future):
if future.cancelled():
log.warning('Request was unexpectedly cancelled')
else:
response = future.result()
log.debug('Response (%s): %s', response.status_code, response.content)
@functools.wraps(fn)
def wrapper(*args, **kwargs):
future = fn(*args, **kwargs)
future.add_done_callback(_log)
return future
return wrapper
class AnalyticsApi(object):
def __init__(self, session, url, write_key):
self.session = session
self.url = url
self.write_key = write_key
@property
def auth(self):
return auth.HTTPBasicAuth(self.write_key, '')
@log_response
def batch(self, events):
"""Send multiple events in one request.
Each event needs to have its type specified.
"""
data = json.dumps({
'batch': events,
'sentAt': utils.now(),
})
log.debug('sending %s events', len(events))
log.debug('Data: %s', data)
return self.session.post(self.url + '/batch', json=data, auth=self.auth)
@log_response
def track(self, event):
"""Send a single tracking event"""
log.debug('Sending track event: %s', event)
import base64
return self.session.post(self.url + '/track', json=event, auth=self.auth)
@classmethod
def load(cls, session=None):
"""Initialize an instance using values from lbry.io."""
if not session:
session = sessions.FuturesSession()
return cls(
session,
conf.ANALYTICS_ENDPOINT,
utils.deobfuscate(conf.ANALYTICS_TOKEN)
)

View file

@ -0,0 +1,47 @@
from lbrynet.analytics import utils
class Events(object):
def __init__(self, context, lbry_id, session_id):
self.context = context
self.lbry_id = lbry_id
self.session_id = session_id
def heartbeat(self):
return {
'userId': 'lbry',
'event': 'Heartbeat',
'properties': {
'lbry_id': self.lbry_id,
'session_id': self.session_id
},
'context': self.context,
'timestamp': utils.now()
}
def make_context(platform, wallet, is_dev=False):
# TODO: distinguish between developer and release instances
return {
'is_dev': is_dev,
'app': {
'name': 'lbrynet',
'version': platform['lbrynet_version'],
'ui_version': platform['ui_version'],
'python_version': platform['python_version'],
'wallet': {
'name': wallet,
# TODO: add in version info for lbrycrdd
'version': platform['lbryum_version'] if wallet == 'lbryum' else None
},
},
# TODO: expand os info to give linux/osx specific info
'os': {
'name': platform['os_system'],
'version': platform['os_release']
},
'library': {
'name': 'lbrynet-analytics',
'version': '1.0.0'
},
}

View file

@ -0,0 +1,8 @@
import datetime
from lbrynet.core.utils import *
def now():
"""Return utc now in isoformat with timezone"""
return datetime.datetime.utcnow().isoformat() + 'Z'

View file

@ -59,4 +59,7 @@ CURRENCIES = {
'USD': {'type': 'fiat'},
}
LOGGLY_TOKEN = 'YWRmNGU4NmEtNjkwNC00YjM2LTk3ZjItMGZhODM3ZDhkYzBi'
LOGGLY_TOKEN = 'LJEzATH4AzRgAwxjAP00LwZ2YGx3MwVgZTMuBQZ3MQuxLmOv'
ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='

View file

@ -1,12 +1,14 @@
import base64
import json
import logging
import logging.handlers
import sys
import traceback
from requests_futures.sessions import FuturesSession
import lbrynet
from lbrynet import conf
from requests_futures.sessions import FuturesSession
from lbrynet.core import utils
session = FuturesSession()
@ -55,18 +57,28 @@ def _log_decorator(fn):
def helper(*args, **kwargs):
log = kwargs.pop('log', logging.getLogger())
level = kwargs.pop('level', logging.INFO)
if not isinstance(level, int):
# despite the name, getLevelName returns
# the numeric level when passed a text level
level = logging.getLevelName(level)
handler = fn(*args, **kwargs)
if handler.name:
remove_handlers(log, handler.name)
handler.setLevel(level)
log.addHandler(handler)
if log.level > level:
log.setLevel(level)
return helper
def disable_noisy_loggers():
def disable_third_party_loggers():
logging.getLogger('requests').setLevel(logging.WARNING)
def disable_noisy_loggers():
logging.getLogger('lbrynet.dht').setLevel(logging.INFO)
@_log_decorator
def configure_console(**kwargs):
"""Convenience function to configure a logger that outputs to stdout"""
@ -85,7 +97,7 @@ def configure_file_handler(file_name, **kwargs):
def get_loggly_url(token=None, version=None):
token = token or base64.b64decode(conf.LOGGLY_TOKEN)
token = token or utils.deobfuscate(conf.LOGGLY_TOKEN)
version = version or lbrynet.__version__
return LOGGLY_URL.format(token=token, tag='lbrynet-' + version)

View file

@ -1,3 +1,4 @@
import base64
import distutils.version
import random
@ -37,3 +38,11 @@ def version_is_greater_than(a, b):
return distutils.version.StrictVersion(a) > distutils.version.StrictVersion(b)
except ValueError:
return distutils.version.LooseVersion(a) > distutils.version.LooseVersion(b)
def deobfuscate(obfustacated):
return base64.b64decode(obfustacated.decode('rot13'))
def obfuscate(plain):
return base64.b64encode(plain).encode('rot13')

View file

@ -240,9 +240,9 @@ class Node(object):
known_nodes = {}
def log_error(err, n):
log.error("error storing blob_hash %s at %s", binascii.hexlify(blob_hash), str(n))
log.error(err.getErrorMessage())
log.error(err.getTraceback())
log.debug("error storing blob_hash %s at %s", binascii.hexlify(blob_hash), str(n))
log.debug(err.getErrorMessage())
log.debug(err.getTraceback())
def log_success(res):
log.debug("Response to store request: %s", str(res))

View file

@ -1,15 +1,25 @@
import binascii
import functools
import json
import unqlite
import logging
import os
from twisted.internet import threads, defer
import unqlite
log = logging.getLogger(__name__)
def run_in_thread(fn):
@functools.wraps(fn)
def wrapped(*args, **kwargs):
return threads.deferToThread(fn, *args, **kwargs)
return wrapped
class LBRYSettings(object):
NAME = "settings.db"
def __init__(self, db_dir):
self.db_dir = db_dir
self.db = None
@ -18,47 +28,39 @@ class LBRYSettings(object):
return self._open_db()
def stop(self):
self.db.close()
self.db = None
return defer.succeed(True)
def _open_db(self):
log.debug("Opening %s as the settings database", str(os.path.join(self.db_dir, "settings.db")))
self.db = unqlite.UnQLite(os.path.join(self.db_dir, "settings.db"))
filename = os.path.join(self.db_dir, self.NAME)
log.debug("Opening %s as the settings database", filename)
self.db = unqlite.UnQLite(filename)
return defer.succeed(True)
@run_in_thread
def save_lbryid(self, lbryid):
def save_lbryid():
self.db['lbryid'] = binascii.hexlify(lbryid)
self.db.commit()
return threads.deferToThread(save_lbryid)
@run_in_thread
def get_lbryid(self):
def get_lbryid():
if 'lbryid' in self.db:
return binascii.unhexlify(self.db['lbryid'])
else:
return None
return threads.deferToThread(get_lbryid)
@run_in_thread
def get_server_running_status(self):
def get_status():
if 'server_running' in self.db:
return json.loads(self.db['server_running'])
else:
return True
return threads.deferToThread(get_status)
@run_in_thread
def save_server_running_status(self, running):
def save_status():
self.db['server_running'] = json.dumps(running)
return threads.deferToThread(save_status)
self.db.commit()
def get_default_data_payment_rate(self):
return self._get_payment_rate("default_data_payment_rate")
@ -78,43 +80,35 @@ class LBRYSettings(object):
def save_server_crypt_info_payment_rate(self, rate):
return self._save_payment_rate("server_crypt_info_payment_rate", rate)
@run_in_thread
def _get_payment_rate(self, rate_type):
def get_rate():
if rate_type in self.db:
return json.loads(self.db[rate_type])
else:
return None
return threads.deferToThread(get_rate)
@run_in_thread
def _save_payment_rate(self, rate_type, rate):
def save_rate():
if rate is not None:
self.db[rate_type] = json.dumps(rate)
elif rate_type in self.db:
del self.db[rate_type]
self.db.commit()
return threads.deferToThread(save_rate)
@run_in_thread
def get_query_handler_status(self, query_identifier):
def get_status():
if json.dumps(('q_h', query_identifier)) in self.db:
return json.loads(self.db[(json.dumps(('q_h', query_identifier)))])
else:
return True
return threads.deferToThread(get_status)
def enable_query_handler(self, query_identifier):
return self._set_query_handler_status(query_identifier, True)
def disable_query_handler(self, query_identifier):
return self._set_query_handler_status(query_identifier, False)
@run_in_thread
def _set_query_handler_status(self, query_identifier, status):
def set_status():
self.db[json.dumps(('q_h', query_identifier))] = json.dumps(status)
return threads.deferToThread(set_status)
self.db.commit()

View file

@ -18,7 +18,7 @@ from appdirs import user_data_dir
from datetime import datetime
from decimal import Decimal
from twisted.web import server
from twisted.internet import defer, threads, error, reactor
from twisted.internet import defer, threads, error, reactor, task
from twisted.internet.task import LoopingCall
from txjsonrpc import jsonrpclib
from txjsonrpc.web import jsonrpc
@ -26,6 +26,7 @@ from txjsonrpc.web.jsonrpc import Handler
from lbrynet import __version__ as lbrynet_version
from lbryum.version import LBRYUM_VERSION as lbryum_version
from lbrynet import analytics
from lbrynet.core.PaymentRateManager import PaymentRateManager
from lbrynet.core.server.BlobAvailabilityHandler import BlobAvailabilityHandlerFactory
from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory
@ -174,6 +175,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
self.known_dht_nodes = KNOWN_DHT_NODES
self.first_run_after_update = False
self.uploaded_temp_files = []
self._session_id = base58.b58encode(generate_id())
if os.name == "nt":
from lbrynet.winhelpers.knownpaths import get_path, FOLDERID, UserHandle
@ -510,6 +512,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
d.addCallback(lambda _: threads.deferToThread(self._setup_data_directory))
d.addCallback(lambda _: self._check_db_migration())
d.addCallback(lambda _: self._get_settings())
d.addCallback(lambda _: self._set_events())
d.addCallback(lambda _: self._get_session())
d.addCallback(lambda _: add_lbry_file_to_sd_identifier(self.sd_identifier))
d.addCallback(lambda _: self._setup_stream_identifier())
@ -519,19 +522,31 @@ class LBRYDaemon(jsonrpc.JSONRPC):
d.addCallback(lambda _: self._setup_server())
d.addCallback(lambda _: _log_starting_vals())
d.addCallback(lambda _: _announce_startup())
d.addCallback(lambda _: self._load_analytics_api())
# TODO: handle errors here
d.callback(None)
return defer.succeed(None)
def _load_analytics_api(self):
self.analytics_api = analytics.Api.load()
self.send_heartbeat = LoopingCall(self._send_heartbeat)
self.send_heartbeat.start(60)
def _send_heartbeat(self):
log.debug('Sending heartbeat')
heartbeat = self._events.heartbeat()
self.analytics_api.track(heartbeat)
def _get_platform(self):
r = {
"processor": platform.processor(),
"python_version: ": platform.python_version(),
"python_version": platform.python_version(),
"platform": platform.platform(),
"os_release": platform.release(),
"os_system": platform.system(),
"lbrynet_version: ": lbrynet_version,
"lbryum_version: ": lbryum_version,
"lbrynet_version": lbrynet_version,
"lbryum_version": lbryum_version,
"ui_version": self.lbry_ui_manager.loaded_git_version,
}
if not self.ip:
@ -545,13 +560,16 @@ class LBRYDaemon(jsonrpc.JSONRPC):
def _initial_setup(self):
def _log_platform():
log.info("Platform: " + json.dumps(self._get_platform()))
log.info("Platform: %s", json.dumps(self._get_platform()))
return defer.succeed(None)
d = _log_platform()
return d
def _set_events(self):
context = analytics.make_context(self._get_platform(), self.wallet_type)
self._events = analytics.Events(context, base58.b58encode(self.lbryid), self._session_id)
def _check_network_connection(self):
try:
host = socket.gethostbyname(REMOTE_SERVER)
@ -939,10 +957,9 @@ class LBRYDaemon(jsonrpc.JSONRPC):
return d
def _modify_loggly_formatter(self):
session_id = base58.b58encode(generate_id())
log_support.configure_loggly_handler(
lbry_id=base58.b58encode(self.lbryid),
session_id=session_id
session_id=self._session_id
)
@ -1768,7 +1785,13 @@ class LBRYDaemon(jsonrpc.JSONRPC):
"""
name = p['name']
force = p.get('force', False)
if force:
d = self._get_est_cost(name)
else:
d = self._search(name)
d.addCallback(lambda r: [i['cost'] for i in r][0])
d.addCallback(lambda r: self._render_response(r, OK_CODE))
return d
@ -2412,8 +2435,10 @@ class _DownloadNameHelper(object):
def wait_or_get_stream(self, args):
stream_info, lbry_file = args
if lbry_file:
log.debug('Wait on lbry_file')
return self._wait_on_lbry_file(lbry_file)
else:
log.debug('No lbry_file, need to get stream')
return self._get_stream(stream_info)
def _get_stream(self, stream_info):
@ -2440,9 +2465,7 @@ class _DownloadNameHelper(object):
written_bytes = self.get_written_bytes(f.file_name)
if written_bytes:
return defer.succeed(self._disp_file(f))
d = defer.succeed(None)
d.addCallback(lambda _: reactor.callLater(1, self._wait_on_lbry_file, f))
return d
return task.deferLater(reactor, 1, self._wait_on_lbry_file, f)
def get_written_bytes(self, file_name):
"""Returns the number of bytes written to `file_name`.
@ -2523,4 +2546,3 @@ class _ResolveNameHelper(object):
def is_cached_name_expired(self):
time_in_cache = self.now() - self.name_data['timestamp']
return time_in_cache >= self.daemon.cache_time

View file

@ -73,11 +73,12 @@ def start():
parser.set_defaults(branch=False, launchui=True, logtoconsole=False, quiet=False)
args = parser.parse_args()
log_support.disable_noisy_loggers()
log_support.configure_file_handler(lbrynet_log)
log_support.configure_loggly_handler()
if args.logtoconsole:
log_support.configure_console()
log_support.configure_console(level='DEBUG')
log_support.disable_third_party_loggers()
log_support.disable_noisy_loggers()
try:
JSONRPCProxy.from_url(API_CONNECTION_STRING).is_running()

View file

@ -10,3 +10,11 @@
*.iml
id.conf
lbrycrd-cli
lbrycrd-osx.zip
lbrycrd-tx
lbrycrdd
lbrynet.*.dmg
LBRY.app

Binary file not shown.

Binary file not shown.

After

Width:  |  Height:  |  Size: 8.5 KiB

View file

@ -0,0 +1,11 @@
badge_icon = 'app.icns'
icon_locations = {
'LBRY.app': (115, 164),
'Applications': (387, 164)
}
background='dmg_background.png'
default_view='icon-view'
symlinks = { 'Applications': '/Applications' }
window_rect=((200, 200), (500, 320))
files = [ 'LBRY.app' ]
icon_size=128

View file

@ -9,17 +9,11 @@ ON_TRAVIS=false
rm -rf build dist LBRY.app
pip install wheel
# the default py2app (v0.9) has a bug that is fixed in the head of /metachris/py2app
pip install git+https://github.com/metachris/py2app
pip install jsonrpc
mkdir -p $tmp
cd $tmp
echo "Updating lbrynet"
if [ -z ${TRAVIS_BUILD_DIR+x} ]; then
# building locally
mkdir -p $tmp
cd $tmp
git clone --depth 1 http://github.com/lbryio/lbry.git
cd lbry
LBRY="${tmp}/lbry"
@ -29,6 +23,39 @@ else
cd ${TRAVIS_BUILD_DIR}
LBRY=${TRAVIS_BUILD_DIR}
fi
pip install wheel
MODULES="pyobjc-core pyobjc-framework-Cocoa pyobjc-framework-CFNetwork pyobjc-framework-Quartz"
if [ ${ON_TRAVIS} = true ]; then
WHEEL_DIR="${TRAVIS_BUILD_DIR}/cache/wheel"
mkdir -p "${WHEEL_DIR}"
# mapping from the package name to the
# actual built wheel file is surprisingly
# hard so instead of checking for the existance
# of each wheel, we mark with a file when they've all been
# built and skip when that file exists
for MODULE in ${MODULES}; do
if [ ! -f "${WHEEL_DIR}"/${MODULE}.finished ]; then
pip wheel -w "${WHEEL_DIR}" ${MODULE}
touch "${WHEEL_DIR}"/${MODULE}.finished
fi
done
pip install "${WHEEL_DIR}"/*.whl
else
pip install $MODULES
fi
pip install dmgbuild
pip show dmgbuild
export PATH=${PATH}:/Library/Frameworks/Python.framework/Versions/2.7/bin
dmgbuild --help
pip install jsonrpc certifi
# the default py2app (v0.9) has a bug that is fixed in the head of /metachris/py2app
pip install git+https://github.com/metachris/py2app
NAME=`python setup.py --name`
VERSION=`python setup.py -V`
pip install -r requirements.txt
@ -48,32 +75,12 @@ codesign -s "${LBRY_DEVELOPER_ID}" -f "${DEST}/dist/LBRYURIHandler.app/Contents/
codesign --deep -s "${LBRY_DEVELOPER_ID}" -f "${DEST}/dist/LBRYURIHandler.app/Contents/MacOS/LBRYURIHandler"
codesign -vvvv "${DEST}/dist/LBRYURIHandler.app"
pip install certifi
MODULES="pyobjc-core pyobjc-framework-Cocoa pyobjc-framework-CFNetwork"
if [ ${ON_TRAVIS} = true ]; then
WHEEL_DIR="${TRAVIS_BUILD_DIR}/cache/wheel"
mkdir -p "${WHEEL_DIR}"
# mapping from the package name to the
# actual built wheel file is surprisingly
# hard so instead of checking for the existance
# of each wheel, we mark with a file when they've all been
# built and skip when that file exists
if [ ! -f "${WHEEL_DIR}"/finished ]; then
pip wheel -w "${WHEEL_DIR}" ${MODULES}
touch "${WHEEL_DIR}"/finished
fi
pip install "${WHEEL_DIR}"/*.whl
else
pip install $MODULES
fi
# add lbrycrdd as a resource. Following
# http://stackoverflow.com/questions/11370012/can-executables-made-with-py2app-include-other-terminal-scripts-and-run-them
# LBRYCRDD_URL="$(curl https://api.github.com/repos/lbryio/lbrycrd/releases/latest | grep 'browser_download_url' | grep osx | cut -d'"' -f4)"
LBRYCRDD_URL="https://github.com/lbryio/lbrycrd/releases/download/v0.3.15/lbrycrd-osx.zip"
wget "${LBRYCRDD_URL}" --output-document lbrycrd-osx.zip
unzip lbrycrd-osx.zip
unzip -o lbrycrd-osx.zip
python setup_app.py py2app --resources lbrycrdd
chmod +x "${DEST}/dist/LBRY.app/Contents/Resources/lbrycrdd"
@ -105,5 +112,4 @@ codesign -vvvv "${DEST}/dist/LBRY.app"
rm -rf $tmp
mv dist/LBRY.app LBRY.app
rm -rf dist "${NAME}.${VERSION}.dmg"
# TODO: make this pretty!
hdiutil create "${NAME}.${VERSION}.dmg" -volname lbry -srcfolder LBRY.app
dmgbuild -s dmg_settings.py "LBRY" "${NAME}.${VERSION}.dmg"

View file

@ -1,5 +1,5 @@
[Desktop Entry]
Version=0.3.17
Version=0.3.18
Name=LBRY
Comment=The world's first user-owned content marketplace
Icon=lbry

View file

@ -12,7 +12,7 @@ https://github.com/lbryio/lbryum/tarball/master/#egg=lbryum
loggly-python-handler==1.0.0
miniupnpc==1.9
pbkdf2==1.3
protobuf==3.0.0b3
protobuf==3.0.0
pycrypto==2.6.1
python-bitcoinrpc==0.1
qrcode==5.2.2

View file

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
from lbrynet.core import utils
from twisted.trial import unittest
@ -17,3 +18,13 @@ class CompareVersionTest(unittest.TestCase):
self.assertTrue(utils.version_is_greater_than('1.3.9.1', '1.3.9'))
class ObfuscationTest(unittest.TestCase):
def test_deobfuscation_reverses_obfuscation(self):
plain = "my_test_string"
obf = utils.obfuscate(plain)
self.assertEqual(plain, utils.deobfuscate(obf))
def test_can_use_unicode(self):
plain = ''
obf = utils.obfuscate(plain)
self.assertEqual(plain, utils.deobfuscate(obf))