Merge branch 'master' into reflector

# Conflicts:
#	lbrynet/__init__.py

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.17
+current_version = 0.3.18
 commit = True
 tag = True
 message = Bump version: {current_version} -> {new_version}

.gitignore

@@ -28,3 +28,5 @@ lbrynet.egg-info/PKG-INFO
 
 # temporary files from the twisted.trial test runner
 _trial_temp/
+
+.DS_Store

lbrynet/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.3.17"
+__version__ = "0.3.18"
 version = tuple(__version__.split('.'))

lbrynet/analytics/__init__.py

@@ -0,0 +1,2 @@
+from events import *
+from api import AnalyticsApi as Api

lbrynet/analytics/api.py

@@ -0,0 +1,71 @@
+import functools
+import json
+import logging
+
+from requests import auth
+from requests_futures import sessions
+
+from lbrynet import conf
+from lbrynet.analytics import utils
+
+
+log = logging.getLogger(__name__)
+
+
+def log_response(fn):
+    def _log(future):
+        if future.cancelled():
+            log.warning('Request was unexpectedly cancelled')
+        else:
+            response = future.result()
+            log.debug('Response (%s): %s', response.status_code, response.content)
+
+    @functools.wraps(fn)
+    def wrapper(*args, **kwargs):
+        future = fn(*args, **kwargs)
+        future.add_done_callback(_log)
+        return future
+    return wrapper
+
+
+class AnalyticsApi(object):
+    def __init__(self, session, url, write_key):
+        self.session = session
+        self.url = url
+        self.write_key = write_key
+
+    @property
+    def auth(self):
+        return auth.HTTPBasicAuth(self.write_key, '')
+
+    @log_response
+    def batch(self, events):
+        """Send multiple events in one request.
+
+        Each event needs to have its type specified.
+        """
+        data = json.dumps({
+            'batch': events,
+            'sentAt': utils.now(),
+        })
+        log.debug('sending %s events', len(events))
+        log.debug('Data: %s', data)
+        return self.session.post(self.url + '/batch', json=data, auth=self.auth)
+
+    @log_response
+    def track(self, event):
+        """Send a single tracking event"""
+        log.debug('Sending track event: %s', event)
+        import base64
+        return self.session.post(self.url + '/track', json=event, auth=self.auth)
+
+    @classmethod
+    def load(cls, session=None):
+        """Initialize an instance using values from lbry.io."""
+        if not session:
+            session = sessions.FuturesSession()
+        return cls(
+            session,
+            conf.ANALYTICS_ENDPOINT,
+            utils.deobfuscate(conf.ANALYTICS_TOKEN)
+        )

lbrynet/analytics/events.py

@@ -0,0 +1,47 @@
+from lbrynet.analytics import utils
+
+
+class Events(object):
+    def __init__(self, context, lbry_id, session_id):
+        self.context = context
+        self.lbry_id = lbry_id
+        self.session_id = session_id
+
+    def heartbeat(self):
+        return {
+            'userId': 'lbry',
+            'event': 'Heartbeat',
+            'properties': {
+                'lbry_id': self.lbry_id,
+                'session_id': self.session_id
+            },
+            'context': self.context,
+            'timestamp': utils.now()
+        }
+
+
+def make_context(platform, wallet, is_dev=False):
+    # TODO: distinguish between developer and release instances
+    return {
+        'is_dev': is_dev,
+        'app': {
+            'name': 'lbrynet',
+            'version': platform['lbrynet_version'],
+            'ui_version': platform['ui_version'],
+            'python_version': platform['python_version'],
+            'wallet': {
+                'name': wallet,
+                # TODO: add in version info for lbrycrdd
+                'version': platform['lbryum_version'] if wallet == 'lbryum' else None
+            },
+        },
+        # TODO: expand os info to give linux/osx specific info
+        'os': {
+            'name': platform['os_system'],
+            'version': platform['os_release']
+        },
+        'library': {
+            'name': 'lbrynet-analytics',
+            'version': '1.0.0'
+        },
+    }

lbrynet/analytics/utils.py

@@ -0,0 +1,8 @@
+import datetime
+
+from lbrynet.core.utils import *
+
+
+def now():
+    """Return utc now in isoformat with timezone"""
+    return datetime.datetime.utcnow().isoformat() + 'Z'

lbrynet/conf.py

@@ -61,4 +61,7 @@ CURRENCIES = {
                 'USD': {'type': 'fiat'},
              }
 
-LOGGLY_TOKEN = 'YWRmNGU4NmEtNjkwNC00YjM2LTk3ZjItMGZhODM3ZDhkYzBi'
+LOGGLY_TOKEN = 'LJEzATH4AzRgAwxjAP00LwZ2YGx3MwVgZTMuBQZ3MQuxLmOv'
+
+ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
+ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='

lbrynet/core/log_support.py

@@ -1,12 +1,14 @@
-import base64
 import json
 import logging
 import logging.handlers
 import sys
 import traceback
+
+from requests_futures.sessions import FuturesSession
+
 import lbrynet
 from lbrynet import conf
-from requests_futures.sessions import FuturesSession
+from lbrynet.core import utils
 
 session = FuturesSession()
 
@@ -95,7 +97,7 @@ def configure_file_handler(file_name, **kwargs):
 
 
 def get_loggly_url(token=None, version=None):
-    token = token or base64.b64decode(conf.LOGGLY_TOKEN)
+    token = token or utils.deobfuscate(conf.LOGGLY_TOKEN)
     version = version or lbrynet.__version__
     return LOGGLY_URL.format(token=token, tag='lbrynet-' + version)
 

lbrynet/core/utils.py

@@ -1,3 +1,4 @@
+import base64
 import distutils.version
 import random
 
@@ -37,3 +38,11 @@ def version_is_greater_than(a, b):
         return distutils.version.StrictVersion(a) > distutils.version.StrictVersion(b)
     except ValueError:
         return distutils.version.LooseVersion(a) > distutils.version.LooseVersion(b)
+
+
+def deobfuscate(obfustacated):
+    return base64.b64decode(obfustacated.decode('rot13'))
+
+
+def obfuscate(plain):
+    return base64.b64encode(plain).encode('rot13')

lbrynet/lbrynet_console/LBRYSettings.py

@@ -1,15 +1,25 @@
 import binascii
+import functools
 import json
-import unqlite
 import logging
 import os
+
 from twisted.internet import threads, defer
+import unqlite
 
 
 log = logging.getLogger(__name__)
 
 
+def run_in_thread(fn):
+    @functools.wraps(fn)
+    def wrapped(*args, **kwargs):
+        return threads.deferToThread(fn, *args, **kwargs)
+    return wrapped
+
+
 class LBRYSettings(object):
+    NAME = "settings.db"
     def __init__(self, db_dir):
         self.db_dir = db_dir
         self.db = None
@@ -18,47 +28,39 @@ class LBRYSettings(object):
         return self._open_db()
 
     def stop(self):
+        self.db.close()
         self.db = None
         return defer.succeed(True)
 
     def _open_db(self):
-        log.debug("Opening %s as the settings database", str(os.path.join(self.db_dir, "settings.db")))
+        filename = os.path.join(self.db_dir, self.NAME)
-        self.db = unqlite.UnQLite(os.path.join(self.db_dir, "settings.db"))
+        log.debug("Opening %s as the settings database", filename)
+        self.db = unqlite.UnQLite(filename)
         return defer.succeed(True)
 
+    @run_in_thread
     def save_lbryid(self, lbryid):
+        self.db['lbryid'] = binascii.hexlify(lbryid)
+        self.db.commit()
 
-        def save_lbryid():
+    @run_in_thread
-            self.db['lbryid'] = binascii.hexlify(lbryid)
-
-        return threads.deferToThread(save_lbryid)
-
     def get_lbryid(self):
+        if 'lbryid' in self.db:
+            return binascii.unhexlify(self.db['lbryid'])
+        else:
+            return None
 
-        def get_lbryid():
+    @run_in_thread
-            if 'lbryid' in self.db:
-                return binascii.unhexlify(self.db['lbryid'])
-            else:
-                return None
-
-        return threads.deferToThread(get_lbryid)
-
     def get_server_running_status(self):
+        if 'server_running' in self.db:
+            return json.loads(self.db['server_running'])
+        else:
+            return True
 
-        def get_status():
+    @run_in_thread
-            if 'server_running' in self.db:
-                return json.loads(self.db['server_running'])
-            else:
-                return True
-
-        return threads.deferToThread(get_status)
-
     def save_server_running_status(self, running):
-
+        self.db['server_running'] = json.dumps(running)
-        def save_status():
+        self.db.commit()
-            self.db['server_running'] = json.dumps(running)
-
-        return threads.deferToThread(save_status)
 
     def get_default_data_payment_rate(self):
         return self._get_payment_rate("default_data_payment_rate")
@@ -78,35 +80,27 @@ class LBRYSettings(object):
     def save_server_crypt_info_payment_rate(self, rate):
         return self._save_payment_rate("server_crypt_info_payment_rate", rate)
 
+    @run_in_thread
     def _get_payment_rate(self, rate_type):
+        if rate_type in self.db:
+            return json.loads(self.db[rate_type])
+        else:
+            return None
 
-        def get_rate():
+    @run_in_thread
-            if rate_type in self.db:
-                return json.loads(self.db[rate_type])
-            else:
-                return None
-
-        return threads.deferToThread(get_rate)
-
     def _save_payment_rate(self, rate_type, rate):
+        if rate is not None:
+            self.db[rate_type] = json.dumps(rate)
+        elif rate_type in self.db:
+            del self.db[rate_type]
+        self.db.commit()
 
-        def save_rate():
+    @run_in_thread
-            if rate is not None:
-                self.db[rate_type] = json.dumps(rate)
-            elif rate_type in self.db:
-                del self.db[rate_type]
-
-        return threads.deferToThread(save_rate)
-
     def get_query_handler_status(self, query_identifier):
-
+        if json.dumps(('q_h', query_identifier)) in self.db:
-        def get_status():
+            return json.loads(self.db[(json.dumps(('q_h', query_identifier)))])
-            if json.dumps(('q_h', query_identifier)) in self.db:
+        else:
-                return json.loads(self.db[(json.dumps(('q_h', query_identifier)))])
+            return True
-            else:
-                return True
-
-        return threads.deferToThread(get_status)
 
     def enable_query_handler(self, query_identifier):
         return self._set_query_handler_status(query_identifier, True)
@@ -114,7 +108,7 @@ class LBRYSettings(object):
     def disable_query_handler(self, query_identifier):
         return self._set_query_handler_status(query_identifier, False)
 
+    @run_in_thread
     def _set_query_handler_status(self, query_identifier, status):
-        def set_status():
+        self.db[json.dumps(('q_h', query_identifier))] = json.dumps(status)
-            self.db[json.dumps(('q_h', query_identifier))] = json.dumps(status)
+        self.db.commit()
-        return threads.deferToThread(set_status)

lbrynet/lbrynet_daemon/LBRYDaemon.py

@@ -18,7 +18,7 @@ from appdirs import user_data_dir
 from datetime import datetime
 from decimal import Decimal
 from twisted.web import server
-from twisted.internet import defer, threads, error, reactor
+from twisted.internet import defer, threads, error, reactor, task
 from twisted.internet.task import LoopingCall
 from txjsonrpc import jsonrpclib
 from txjsonrpc.web import jsonrpc
@@ -26,6 +26,7 @@ from txjsonrpc.web.jsonrpc import Handler
 
 from lbrynet import __version__ as lbrynet_version
 from lbryum.version import LBRYUM_VERSION as lbryum_version
+from lbrynet import analytics
 from lbrynet.core.PaymentRateManager import PaymentRateManager
 from lbrynet.core.server.BlobAvailabilityHandler import BlobAvailabilityHandlerFactory
 from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory
@@ -174,6 +175,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         self.known_dht_nodes = KNOWN_DHT_NODES
         self.first_run_after_update = False
         self.uploaded_temp_files = []
+        self._session_id = base58.b58encode(generate_id())
 
         if os.name == "nt":
             from lbrynet.winhelpers.knownpaths import get_path, FOLDERID, UserHandle
@@ -514,6 +516,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         d.addCallback(lambda _: threads.deferToThread(self._setup_data_directory))
         d.addCallback(lambda _: self._check_db_migration())
         d.addCallback(lambda _: self._get_settings())
+        d.addCallback(lambda _: self._set_events())
         d.addCallback(lambda _: self._get_session())
         d.addCallback(lambda _: add_lbry_file_to_sd_identifier(self.sd_identifier))
         d.addCallback(lambda _: self._setup_stream_identifier())
@@ -523,21 +526,33 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         d.addCallback(lambda _: self._setup_server())
         d.addCallback(lambda _: _log_starting_vals())
         d.addCallback(lambda _: _announce_startup())
+        d.addCallback(lambda _: self._load_analytics_api())
+        # TODO: handle errors here
         d.callback(None)
 
         return defer.succeed(None)
 
+    def _load_analytics_api(self):
+        self.analytics_api = analytics.Api.load()
+        self.send_heartbeat = LoopingCall(self._send_heartbeat)
+        self.send_heartbeat.start(60)
+
+    def _send_heartbeat(self):
+        log.debug('Sending heartbeat')
+        heartbeat = self._events.heartbeat()
+        self.analytics_api.track(heartbeat)
+
     def _get_platform(self):
         r =  {
             "processor": platform.processor(),
-            "python_version: ": platform.python_version(),
+            "python_version": platform.python_version(),
             "platform": platform.platform(),
             "os_release": platform.release(),
             "os_system": platform.system(),
-            "lbrynet_version: ": lbrynet_version,
+            "lbrynet_version": lbrynet_version,
-            "lbryum_version: ": lbryum_version,
+            "lbryum_version": lbryum_version,
             "ui_version": self.lbry_ui_manager.loaded_git_version,
-            }
+        }
         if not self.ip:
             try:
                 r['ip'] = json.load(urlopen('http://jsonip.com'))['ip']
@@ -549,13 +564,16 @@ class LBRYDaemon(jsonrpc.JSONRPC):
 
     def _initial_setup(self):
         def _log_platform():
-            log.info("Platform: " + json.dumps(self._get_platform()))
+            log.info("Platform: %s", json.dumps(self._get_platform()))
             return defer.succeed(None)
 
         d = _log_platform()
-
         return d
 
+    def _set_events(self):
+        context = analytics.make_context(self._get_platform(), self.wallet_type)
+        self._events = analytics.Events(context, base58.b58encode(self.lbryid), self._session_id)
+
     def _check_network_connection(self):
         try:
             host = socket.gethostbyname(REMOTE_SERVER)
@@ -972,10 +990,9 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         return d
 
     def _modify_loggly_formatter(self):
-        session_id = base58.b58encode(generate_id())
         log_support.configure_loggly_handler(
             lbry_id=base58.b58encode(self.lbryid),
-            session_id=session_id
+            session_id=self._session_id
         )
 
 
@@ -1801,7 +1818,13 @@ class LBRYDaemon(jsonrpc.JSONRPC):
         """
 
         name = p['name']
-        d = self._get_est_cost(name)
+        force = p.get('force', False)
+
+        if force:
+            d = self._get_est_cost(name)
+        else:
+            d = self._search(name)
+            d.addCallback(lambda r: [i['cost'] for i in r][0])
         d.addCallback(lambda r: self._render_response(r, OK_CODE))
         return d
 
@@ -2485,8 +2508,10 @@ class _DownloadNameHelper(object):
     def wait_or_get_stream(self, args):
         stream_info, lbry_file = args
         if lbry_file:
+            log.debug('Wait on lbry_file')
             return self._wait_on_lbry_file(lbry_file)
         else:
+            log.debug('No lbry_file, need to get stream')
             return self._get_stream(stream_info)
 
     def _get_stream(self, stream_info):
@@ -2513,9 +2538,7 @@ class _DownloadNameHelper(object):
         written_bytes = self.get_written_bytes(f.file_name)
         if written_bytes:
             return defer.succeed(self._disp_file(f))
-        d = defer.succeed(None)
+        return task.deferLater(reactor, 1, self._wait_on_lbry_file, f)
-        d.addCallback(lambda _: reactor.callLater(1, self._wait_on_lbry_file, f))
-        return d
 
     def get_written_bytes(self, file_name):
         """Returns the number of bytes written to `file_name`.
@@ -2596,4 +2619,3 @@ class _ResolveNameHelper(object):
     def is_cached_name_expired(self):
         time_in_cache = self.now() - self.name_data['timestamp']
         return time_in_cache >= self.daemon.cache_time
-

packaging/osx/lbry-osx-app/.gitignore

@@ -10,3 +10,11 @@
 *.iml
 
 id.conf
+
+lbrycrd-cli
+lbrycrd-osx.zip
+lbrycrd-tx
+lbrycrdd
+
+lbrynet.*.dmg
+LBRY.app

packaging/osx/lbry-osx-app/dmg_settings.py

@@ -0,0 +1,11 @@
+badge_icon = 'app.icns'
+icon_locations = {
+    'LBRY.app':     (115, 164),
+    'Applications': (387, 164)
+}
+background='dmg_background.png'
+default_view='icon-view'
+symlinks = { 'Applications': '/Applications' }
+window_rect=((200, 200), (500, 320))
+files = [ 'LBRY.app' ]
+icon_size=128

packaging/osx/lbry-osx-app/setup_app.sh

@@ -9,17 +9,11 @@ ON_TRAVIS=false
 
 rm -rf build dist LBRY.app
 
-pip install wheel
-# the default py2app (v0.9) has a bug that is fixed in the head of /metachris/py2app
-pip install git+https://github.com/metachris/py2app
-pip install jsonrpc
-
-mkdir -p $tmp
-cd $tmp
-
 echo "Updating lbrynet"
 if [ -z ${TRAVIS_BUILD_DIR+x} ]; then
     # building locally
+    mkdir -p $tmp
+    cd $tmp
     git clone --depth 1 http://github.com/lbryio/lbry.git
     cd lbry
     LBRY="${tmp}/lbry"
@@ -29,6 +23,39 @@ else
     cd ${TRAVIS_BUILD_DIR}
     LBRY=${TRAVIS_BUILD_DIR}
 fi
+
+pip install wheel
+MODULES="pyobjc-core pyobjc-framework-Cocoa pyobjc-framework-CFNetwork pyobjc-framework-Quartz"
+if [ ${ON_TRAVIS} = true ]; then
+    WHEEL_DIR="${TRAVIS_BUILD_DIR}/cache/wheel"
+    mkdir -p "${WHEEL_DIR}"
+    # mapping from the package name to the
+    # actual built wheel file is surprisingly
+    # hard so instead of checking for the existance
+    # of each wheel, we mark with a file when they've all been
+    # built and skip when that file exists
+    for MODULE in ${MODULES}; do
+	if [ ! -f "${WHEEL_DIR}"/${MODULE}.finished ]; then
+	    pip wheel -w "${WHEEL_DIR}" ${MODULE}
+	    touch "${WHEEL_DIR}"/${MODULE}.finished
+	fi
+    done
+    pip install "${WHEEL_DIR}"/*.whl
+else
+    pip install $MODULES
+fi
+
+pip install dmgbuild
+pip show dmgbuild
+
+export PATH=${PATH}:/Library/Frameworks/Python.framework/Versions/2.7/bin
+dmgbuild --help
+
+pip install jsonrpc certifi
+
+# the default py2app (v0.9) has a bug that is fixed in the head of /metachris/py2app
+pip install git+https://github.com/metachris/py2app
+
 NAME=`python setup.py --name`
 VERSION=`python setup.py -V`
 pip install -r requirements.txt
@@ -48,32 +75,12 @@ codesign -s "${LBRY_DEVELOPER_ID}" -f "${DEST}/dist/LBRYURIHandler.app/Contents/
 codesign --deep -s "${LBRY_DEVELOPER_ID}" -f "${DEST}/dist/LBRYURIHandler.app/Contents/MacOS/LBRYURIHandler"
 codesign -vvvv "${DEST}/dist/LBRYURIHandler.app"
 
-pip install certifi
-MODULES="pyobjc-core pyobjc-framework-Cocoa pyobjc-framework-CFNetwork"
-if [ ${ON_TRAVIS} = true ]; then
-    WHEEL_DIR="${TRAVIS_BUILD_DIR}/cache/wheel"
-    mkdir -p "${WHEEL_DIR}"
-    # mapping from the package name to the
-    # actual built wheel file is surprisingly
-    # hard so instead of checking for the existance
-    # of each wheel, we mark with a file when they've all been
-    # built and skip when that file exists
-    if [ ! -f "${WHEEL_DIR}"/finished ]; then
-	pip wheel -w "${WHEEL_DIR}" ${MODULES}
-	touch "${WHEEL_DIR}"/finished
-    fi
-    pip install "${WHEEL_DIR}"/*.whl
-else
-    pip install $MODULES
-fi
-
-
 # add lbrycrdd as a resource. Following
 # http://stackoverflow.com/questions/11370012/can-executables-made-with-py2app-include-other-terminal-scripts-and-run-them
 # LBRYCRDD_URL="$(curl https://api.github.com/repos/lbryio/lbrycrd/releases/latest | grep 'browser_download_url' | grep osx | cut -d'"' -f4)"
 LBRYCRDD_URL="https://github.com/lbryio/lbrycrd/releases/download/v0.3.15/lbrycrd-osx.zip"
 wget "${LBRYCRDD_URL}" --output-document lbrycrd-osx.zip
-unzip lbrycrd-osx.zip
+unzip -o lbrycrd-osx.zip
 python setup_app.py py2app --resources lbrycrdd
 
 chmod +x "${DEST}/dist/LBRY.app/Contents/Resources/lbrycrdd"
@@ -105,5 +112,4 @@ codesign -vvvv "${DEST}/dist/LBRY.app"
 rm -rf $tmp
 mv dist/LBRY.app LBRY.app
 rm -rf dist "${NAME}.${VERSION}.dmg"
-# TODO: make this pretty!
+dmgbuild -s dmg_settings.py "LBRY" "${NAME}.${VERSION}.dmg"
-hdiutil create "${NAME}.${VERSION}.dmg" -volname lbry -srcfolder LBRY.app

packaging/ubuntu/lbry

@@ -42,7 +42,7 @@ DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 
 if [ -z "$(pgrep lbrynet-daemon)" ]; then
   echo "running lbrynet-daemon..."
-  $DIR/lbrynet-daemon --branch="$WEB_UI_BRANCH" &
+  $DIR/lbrynet-daemon --no-launch --branch="$WEB_UI_BRANCH" &
   sleep 3 # let the daemon load before connecting
 fi
 

packaging/ubuntu/lbry.desktop

@@ -1,5 +1,5 @@
 [Desktop Entry]
-Version=0.3.17
+Version=0.3.18
 Name=LBRY
 Comment=The world's first user-owned content marketplace
 Icon=lbry

requirements.txt

@@ -12,7 +12,7 @@ https://github.com/lbryio/lbryum/tarball/master/#egg=lbryum
 loggly-python-handler==1.0.0
 miniupnpc==1.9
 pbkdf2==1.3
-protobuf==3.0.0b3
+protobuf==3.0.0
 pycrypto==2.6.1
 python-bitcoinrpc==0.1
 qrcode==5.2.2

tests/unit/core/test_utils.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from lbrynet.core import utils
 
 from twisted.trial import unittest
@@ -15,5 +16,15 @@ class CompareVersionTest(unittest.TestCase):
 
     def test_version_can_have_four_parts(self):
         self.assertTrue(utils.version_is_greater_than('1.3.9.1', '1.3.9'))
-        
+
-    
+
+class ObfuscationTest(unittest.TestCase):
+    def test_deobfuscation_reverses_obfuscation(self):
+        plain = "my_test_string"
+        obf = utils.obfuscate(plain)
+        self.assertEqual(plain, utils.deobfuscate(obf))
+
+    def test_can_use_unicode(self):
+        plain = '☃'
+        obf = utils.obfuscate(plain)
+        self.assertEqual(plain, utils.deobfuscate(obf))