lbry-sdk/lbrynet/core/utils.py

250 lines
7.4 KiB
Python
Raw Normal View History

import base64
import codecs
import datetime
2015-08-20 17:27:15 +02:00
import random
import socket
2016-12-30 19:35:17 +01:00
import string
2018-07-21 20:12:29 +02:00
import json
2018-06-07 18:18:07 +02:00
import traceback
import functools
import logging
2016-11-19 23:58:40 +01:00
import pkg_resources
2018-06-07 18:18:07 +02:00
from twisted.python.failure import Failure
2018-02-28 20:59:12 +01:00
from twisted.internet import defer
from lbryschema.claim import ClaimDict
from lbrynet.core.cryptoutils import get_lbry_hash_obj
2018-06-07 18:18:07 +02:00
log = logging.getLogger(__name__)
2016-09-30 06:06:07 +02:00
# digest_size is in bytes, and blob hashes are hex encoded
blobhash_length = get_lbry_hash_obj().digest_size * 2
2015-08-20 17:27:15 +02:00
2016-10-05 21:16:20 +02:00
# defining these time functions here allows for easier overriding in testing
2016-09-30 06:06:07 +02:00
def now():
return datetime.datetime.now()
2016-10-05 21:16:20 +02:00
2016-09-30 06:06:07 +02:00
def utcnow():
return datetime.datetime.utcnow()
2016-10-05 21:16:20 +02:00
2016-09-30 06:06:07 +02:00
def isonow():
"""Return utc now in isoformat with timezone"""
return utcnow().isoformat() + 'Z'
2016-09-30 06:06:07 +02:00
def today():
return datetime.datetime.today()
2017-01-02 20:52:24 +01:00
def timedelta(**kwargs):
return datetime.timedelta(**kwargs)
def datetime_obj(*args, **kwargs):
return datetime.datetime(*args, **kwargs)
def call_later(delay, func, *args, **kwargs):
2017-04-10 16:51:49 +02:00
# Import here to ensure that it gets called after installing a reactor
# see: http://twistedmatrix.com/documents/current/core/howto/choosing-reactor.html
from twisted.internet import reactor
return reactor.callLater(delay, func, *args, **kwargs)
def safe_start_looping_call(looping_call, interval_sec):
if not looping_call.running:
looping_call.start(interval_sec)
def safe_stop_looping_call(looping_call):
if looping_call.running:
looping_call.stop()
2015-08-20 17:27:15 +02:00
def generate_id(num=None):
h = get_lbry_hash_obj()
if num is not None:
2018-06-12 17:54:01 +02:00
h.update(str(num).encode())
2015-08-20 17:27:15 +02:00
else:
2018-06-12 17:54:01 +02:00
h.update(str(random.getrandbits(512)).encode())
2015-08-20 17:27:15 +02:00
return h.digest()
2016-10-05 21:16:20 +02:00
def is_valid_hashcharacter(char):
return char in "0123456789abcdef"
2015-08-20 17:27:15 +02:00
def is_valid_blobhash(blobhash):
2016-10-19 00:25:16 +02:00
"""Checks whether the blobhash is the correct length and contains only
valid characters (0-9, a-f)
2015-08-20 17:27:15 +02:00
@param blobhash: string, the blobhash to check
2016-10-19 00:25:16 +02:00
@return: True/False
2015-08-20 17:27:15 +02:00
"""
2016-10-19 00:25:16 +02:00
return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
def version_is_greater_than(a, b):
"""Returns True if version a is more recent than version b"""
2016-11-19 23:58:40 +01:00
return pkg_resources.parse_version(a) > pkg_resources.parse_version(b)
def rot13(some_str):
return codecs.encode(some_str, 'rot_13')
def deobfuscate(obfustacated):
return base64.b64decode(rot13(obfustacated))
def obfuscate(plain):
return rot13(base64.b64encode(plain).decode())
def check_connection(server="lbry.io", port=80, timeout=5):
"""Attempts to open a socket to server:port and returns True if successful."""
log.debug('Checking connection to %s:%s', server, port)
try:
2017-10-31 17:18:47 +01:00
server = socket.gethostbyname(server)
2018-10-16 17:03:56 +02:00
conn = socket.create_connection((server, port), timeout)
conn.close()
2016-11-11 17:10:00 +01:00
log.debug('Connection successful')
return True
except (socket.gaierror, socket.herror) as ex:
2017-10-31 17:18:55 +01:00
log.warning("Failed to connect to %s:%s. Unable to resolve domain. Trying to bypass DNS",
server, port)
try:
server = "8.8.8.8"
port = 53
socket.create_connection((server, port), timeout)
log.debug('Connection successful')
return True
except Exception as ex:
2017-10-31 17:18:55 +01:00
log.error("Failed to connect to %s:%s. Maybe the internet connection is not working",
server, port)
return False
except Exception as ex:
2017-10-31 17:18:55 +01:00
log.error("Failed to connect to %s:%s. Maybe the internet connection is not working",
server, port)
return False
2016-12-30 19:35:17 +01:00
def random_string(length=10, chars=string.ascii_lowercase):
return ''.join([random.choice(chars) for _ in range(length)])
def short_hash(hash_str):
return hash_str[:6]
2017-03-09 16:39:17 +01:00
def get_sd_hash(stream_info):
if not stream_info:
return None
if isinstance(stream_info, ClaimDict):
return stream_info.source_hash
result = stream_info.get('claim', {}).\
get('value', {}).\
get('stream', {}).\
get('source', {}).\
get('source')
if not result:
2018-07-22 01:08:28 +02:00
log.warning("Unable to get sd_hash")
return result
def json_dumps_pretty(obj, **kwargs):
return json.dumps(obj, sort_keys=True, indent=2, separators=(',', ': '), **kwargs)
2018-02-28 20:59:12 +01:00
class DeferredLockContextManager:
def __init__(self, lock):
self._lock = lock
def __enter__(self):
2018-10-18 13:40:37 +02:00
yield self._lock.acquire()
def __exit__(self, exc_type, exc_val, exc_tb):
yield self._lock.release()
2018-02-28 20:59:12 +01:00
@defer.inlineCallbacks
def DeferredDict(d, consumeErrors=False):
keys = []
dl = []
response = {}
2018-07-06 07:17:20 +02:00
for k, v in d.items():
2018-02-28 20:59:12 +01:00
keys.append(k)
dl.append(v)
results = yield defer.DeferredList(dl, consumeErrors=consumeErrors)
for k, (success, result) in zip(keys, results):
if success:
response[k] = result
defer.returnValue(response)
2018-06-06 23:21:56 +02:00
class DeferredProfiler:
2018-06-06 23:21:56 +02:00
def __init__(self):
self.profile_results = {}
def add_result(self, fn, start_time, finished_time, stack, success):
self.profile_results[fn].append((start_time, finished_time, stack, success))
def show_profile_results(self, fn):
profile_results = list(self.profile_results[fn])
call_counts = {
caller: [(start, finished, finished - start, success)
for (start, finished, _caller, success) in profile_results
if _caller == caller]
for caller in set(result[2] for result in profile_results)
}
log.info("called %s %i times from %i sources\n", fn.__name__, len(profile_results), len(call_counts))
for caller in sorted(list(call_counts.keys()), key=lambda c: len(call_counts[c]), reverse=True):
call_info = call_counts[caller]
times = [r[2] for r in call_info]
own_time = sum(times)
times.sort()
longest = 0 if not times else times[-1]
shortest = 0 if not times else times[0]
log.info(
"%i successes and %i failures\nlongest %f, shortest %f, avg %f\ncaller:\n%s",
len([r for r in call_info if r[3]]),
len([r for r in call_info if not r[3]]),
longest, shortest, own_time / float(len(call_info)), caller
)
def profiled_deferred(self, reactor=None):
if not reactor:
from twisted.internet import reactor
def _cb(result, fn, start, caller_info):
if isinstance(result, (Failure, Exception)):
error = result
result = None
else:
error = None
self.add_result(fn, start, reactor.seconds(), caller_info, error is None)
if error is None:
return result
raise error
def _profiled_deferred(fn):
reactor.addSystemEventTrigger("after", "shutdown", self.show_profile_results, fn)
self.profile_results[fn] = []
@functools.wraps(fn)
def _wrapper(*args, **kwargs):
caller_info = "".join(traceback.format_list(traceback.extract_stack()[-3:-1]))
start = reactor.seconds()
d = defer.maybeDeferred(fn, *args, **kwargs)
d.addBoth(_cb, fn, start, caller_info)
return d
return _wrapper
return _profiled_deferred
_profiler = DeferredProfiler()
profile_deferred = _profiler.profiled_deferred