lbry-sdk/lbry/torrent/tracker.py

import random
import socket
import string
import struct
import asyncio
import logging
import time
from collections import namedtuple
from functools import reduce
from typing import Optional

from lbry.utils import resolve_host, async_timed_cache, cache_concurrent
from lbry.wallet.stream import StreamController
from lbry import version

log = logging.getLogger(__name__)
CONNECTION_EXPIRES_AFTER_SECONDS = 50
PREFIX = 'LB'  # todo: PR BEP20 to add ourselves
# see: http://bittorrent.org/beps/bep_0015.html and http://xbtt.sourceforge.net/udp_tracker_protocol.html
ConnectRequest = namedtuple("ConnectRequest", ["connection_id", "action", "transaction_id"])
ConnectResponse = namedtuple("ConnectResponse", ["action", "transaction_id", "connection_id"])
AnnounceRequest = namedtuple("AnnounceRequest",
                             ["connection_id", "action", "transaction_id", "info_hash", "peer_id", "downloaded", "left",
                              "uploaded", "event", "ip_addr", "key", "num_want", "port"])
AnnounceResponse = namedtuple("AnnounceResponse",
                              ["action", "transaction_id", "interval", "leechers", "seeders", "peers"])
CompactIPv4Peer = namedtuple("CompactPeer", ["address", "port"])
ScrapeRequest = namedtuple("ScrapeRequest", ["connection_id", "action", "transaction_id", "infohashes"])
ScrapeResponse = namedtuple("ScrapeResponse", ["action", "transaction_id", "items"])
ScrapeResponseItem = namedtuple("ScrapeResponseItem", ["seeders", "completed", "leechers"])
ErrorResponse = namedtuple("ErrorResponse", ["action", "transaction_id", "message"])
STRUCTS = {
    ConnectRequest: struct.Struct(">QII"),
    ConnectResponse: struct.Struct(">IIQ"),
    AnnounceRequest: struct.Struct(">QII20s20sQQQIIIiH"),
    AnnounceResponse: struct.Struct(">IIIII"),
    CompactIPv4Peer: struct.Struct(">IH"),
    ScrapeRequest: struct.Struct(">QII"),
    ScrapeResponse: struct.Struct(">II"),
    ScrapeResponseItem: struct.Struct(">III"),
    ErrorResponse: struct.Struct(">II")
}


def decode(cls, data, offset=0):
    decoder = STRUCTS[cls]
    if cls == AnnounceResponse:
        return AnnounceResponse(*decoder.unpack_from(data, offset),
                                peers=[decode(CompactIPv4Peer, data, index) for index in range(20, len(data), 6)])
    elif cls == ScrapeResponse:
        return ScrapeResponse(*decoder.unpack_from(data, offset),
                              items=[decode(ScrapeResponseItem, data, index) for index in range(8, len(data), 12)])
    elif cls == ErrorResponse:
        return ErrorResponse(*decoder.unpack_from(data, offset), data[decoder.size:])
    return cls(*decoder.unpack_from(data, offset))


def encode(obj):
    if isinstance(obj, ScrapeRequest):
        return STRUCTS[ScrapeRequest].pack(*obj[:-1]) + b''.join(obj.infohashes)
    elif isinstance(obj, ErrorResponse):
        return STRUCTS[ErrorResponse].pack(*obj[:-1]) + obj.message
    elif isinstance(obj, AnnounceResponse):
        return STRUCTS[AnnounceResponse].pack(*obj[:-1]) + b''.join([encode(peer) for peer in obj.peers])
    return STRUCTS[type(obj)].pack(*obj)


def make_peer_id(random_part: Optional[str] = None) -> bytes:
    # see https://wiki.theory.org/BitTorrentSpecification#peer_id and https://www.bittorrent.org/beps/bep_0020.html
    # not to confuse with node id; peer id identifies uniquely the software, version and instance
    random_part = random_part or ''.join(random.choice(string.ascii_letters) for _ in range(20))
    return f"{PREFIX}-{'-'.join(map(str, version))}-{random_part}"[:20].encode()


class UDPTrackerClientProtocol(asyncio.DatagramProtocol):
    def __init__(self, timeout: float = 10.0):
        self.transport = None
        self.data_queue = {}
        self.timeout = timeout
        self.semaphore = asyncio.Semaphore(10)

    def connection_made(self, transport: asyncio.DatagramTransport) -> None:
        self.transport = transport

    async def request(self, obj, tracker_ip, tracker_port):
        self.data_queue[obj.transaction_id] = asyncio.get_running_loop().create_future()
        try:
            async with self.semaphore:
                self.transport.sendto(encode(obj), (tracker_ip, tracker_port))
                return await asyncio.wait_for(self.data_queue[obj.transaction_id], self.timeout)
        finally:
            self.data_queue.pop(obj.transaction_id, None)

    async def connect(self, tracker_ip, tracker_port):
        transaction_id = random.getrandbits(32)
        return decode(ConnectResponse,
                      await self.request(ConnectRequest(0x41727101980, 0, transaction_id), tracker_ip, tracker_port))

    @cache_concurrent
    @async_timed_cache(CONNECTION_EXPIRES_AFTER_SECONDS)
    async def ensure_connection_id(self, peer_id, tracker_ip, tracker_port):
        # peer_id is just to ensure cache coherency
        return (await self.connect(tracker_ip, tracker_port)).connection_id

    async def announce(self, info_hash, peer_id, port, tracker_ip, tracker_port, stopped=False):
        connection_id = await self.ensure_connection_id(peer_id, tracker_ip, tracker_port)
        # this should make the key deterministic but unique per info hash + peer id
        key = int.from_bytes(info_hash[:4], "big") ^ int.from_bytes(peer_id[:4], "big") ^ port
        transaction_id = random.getrandbits(32)
        req = AnnounceRequest(
            connection_id, 1, transaction_id, info_hash, peer_id, 0, 0, 0, 3 if stopped else 1, 0, key, -1, port)
        return decode(AnnounceResponse, await self.request(req, tracker_ip, tracker_port))

    async def scrape(self, infohashes, tracker_ip, tracker_port, connection_id=None):
        connection_id = await self.ensure_connection_id(None, tracker_ip, tracker_port)
        transaction_id = random.getrandbits(32)
        reply = await self.request(
            ScrapeRequest(connection_id, 2, transaction_id, infohashes), tracker_ip, tracker_port)
        return decode(ScrapeResponse, reply), connection_id

    def datagram_received(self, data: bytes, addr: (str, int)) -> None:
        if len(data) < 8:
            return
        transaction_id = int.from_bytes(data[4:8], byteorder="big", signed=False)
        if transaction_id in self.data_queue:
            if not self.data_queue[transaction_id].done():
                if data[3] == 3:
                    return self.data_queue[transaction_id].set_exception(Exception(decode(ErrorResponse, data).message))
                return self.data_queue[transaction_id].set_result(data)
        log.debug("unexpected packet (can be a response for a previously timed out request): %s", data.hex())

    def connection_lost(self, exc: Exception = None) -> None:
        self.transport = None


class TrackerClient:
    EVENT_CONTROLLER = StreamController()

    def __init__(self, node_id, announce_port, get_servers, timeout=10.0):
        self.client = UDPTrackerClientProtocol(timeout=timeout)
        self.transport = None
        self.peer_id = make_peer_id(node_id.hex() if node_id else None)
        self.announce_port = announce_port
        self._get_servers = get_servers
        self.results = {}  # we can't probe the server before the interval, so we keep the result here until it expires
        self.tasks = {}

    async def start(self):
        self.transport, _ = await asyncio.get_running_loop().create_datagram_endpoint(
            lambda: self.client, local_addr=("0.0.0.0", 0))
        self.EVENT_CONTROLLER.stream.listen(
            lambda request: self.on_hash(request[1], request[2]) if request[0] == 'search' else None)

    def stop(self):
        while self.tasks:
            self.tasks.popitem()[1].cancel()
        if self.transport is not None:
            self.transport.close()
        self.client = None
        self.transport = None
        self.EVENT_CONTROLLER.close()

    def on_hash(self, info_hash, on_announcement=None):
        if info_hash not in self.tasks:
            task = asyncio.create_task(self.get_peer_list(info_hash, on_announcement=on_announcement))
            task.add_done_callback(lambda *_: self.tasks.pop(info_hash, None))
            self.tasks[info_hash] = task

    async def announce_many(self, *info_hashes, stopped=False):
        await asyncio.gather(
            *[self._announce_many(server, info_hashes, stopped=stopped) for server in self._get_servers()],
            return_exceptions=True)

    async def _announce_many(self, server, info_hashes, stopped=False):
        tracker_ip = await resolve_host(*server, 'udp')
        still_good_info_hashes = {
            info_hash for (info_hash, (next_announcement, _)) in self.results.get(tracker_ip, {}).items()
            if time.time() < next_announcement
        }
        results = await asyncio.gather(
            *[self._probe_server(info_hash, tracker_ip, server[1], stopped=stopped)
              for info_hash in info_hashes if info_hash not in still_good_info_hashes],
            return_exceptions=True)
        if results:
            errors = sum([1 for result in results if result is None or isinstance(result, Exception)])
            log.info("Tracker: finished announcing %d files to %s:%d, %d errors", len(results), *server, errors)

    async def get_peer_list(self, info_hash, stopped=False, on_announcement=None):
        found = []
        servers = self._get_servers()
        for done in asyncio.as_completed([self._probe_server(info_hash, *server, stopped) for server in servers]):
            result = await done
            if result is not None:
                await asyncio.gather(*filter(asyncio.iscoroutine, [on_announcement(result)] if on_announcement else []))
                found.append(result)
        return found

    async def _probe_server(self, info_hash, tracker_host, tracker_port, stopped=False):
        result = None
        try:
            tracker_host = await resolve_host(tracker_host, tracker_port, 'udp')
        except socket.error:
            log.warning("DNS failure while resolving tracker host: %s, skipping.", tracker_host)
            return
        self.results.setdefault(tracker_host, {})
        if info_hash in self.results[tracker_host]:
            next_announcement, result = self.results[tracker_host][info_hash]
            if time.time() < next_announcement:
                return result
        try:
            result = await self.client.announce(
                info_hash, self.peer_id, self.announce_port, tracker_host, tracker_port, stopped)
            self.results[tracker_host][info_hash] = (time.time() + result.interval, result)
        except asyncio.TimeoutError:  # todo: this is UDP, timeout is common, we need a better metric for failures
            self.results[tracker_host][info_hash] = (time.time() + 60.0, result)
            log.debug("Tracker timed out: %s:%d", tracker_host, tracker_port)
            return None
        log.debug("Announced: %s found %d peers for %s", tracker_host, len(result.peers), info_hash.hex()[:8])
        return result


def subscribe_hash(info_hash: bytes, on_data):
    TrackerClient.EVENT_CONTROLLER.add(('search', info_hash, on_data))


class UDPTrackerServerProtocol(asyncio.DatagramProtocol):  # for testing. Not suitable for production
    def __init__(self):
        self.transport = None
        self.known_conns = set()
        self.peers = {}

    def connection_made(self, transport: asyncio.DatagramTransport) -> None:
        self.transport = transport

    def add_peer(self, info_hash, ip_address: str, port: int):
        self.peers.setdefault(info_hash, [])
        self.peers[info_hash].append(encode_peer(ip_address, port))

    def datagram_received(self, data: bytes, addr: (str, int)) -> None:
        if len(data) < 16:
            return
        action = int.from_bytes(data[8:12], "big", signed=False)
        if action == 0:
            req = decode(ConnectRequest, data)
            connection_id = random.getrandbits(32)
            self.known_conns.add(connection_id)
            return self.transport.sendto(encode(ConnectResponse(0, req.transaction_id, connection_id)), addr)
        elif action == 1:
            req = decode(AnnounceRequest, data)
            if req.connection_id not in self.known_conns:
                resp = encode(ErrorResponse(3, req.transaction_id, b'Connection ID missmatch.\x00'))
            else:
                compact_address = encode_peer(addr[0], req.port)
                if req.event != 3:
                    self.add_peer(req.info_hash, addr[0], req.port)
                elif compact_address in self.peers.get(req.info_hash, []):
                    self.peers[req.info_hash].remove(compact_address)
                peers = [decode(CompactIPv4Peer, peer) for peer in self.peers[req.info_hash]]
                resp = encode(AnnounceResponse(1, req.transaction_id, 1700, 0, len(peers), peers))
            return self.transport.sendto(resp, addr)


def encode_peer(ip_address: str, port: int):
    compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), ip_address.split('.'), bytearray())
    return compact_ip + port.to_bytes(2, "big", signed=False)