LBRYCommunity/lbry-sdk
1
0
Fork 1
Code Issues Pull requests Projects Releases 282 Packages Wiki Activity

dht_crawler: save a set of connections to avoid dupes, enable initial crawl

Browse source
This commit is contained in:
Victor Shyba 2022-06-11 17:40:42 -03:00 committed by Victor Shyba
parent 90c2a58470
commit 443a1c32fa
1 changed files with 4 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
scripts/dht_crawler.py
View file

@ -142,12 +142,12 @@ class Crawler:
def associate_peers(self, target_peer, peers): def associate_peers(self, target_peer, peers):
db_peer = self.get_from_peer(target_peer) db_peer = self.get_from_peer(target_peer)
connections = [ connections = {
DHTConnection( DHTConnection(
from_peer_id=db_peer.peer_id, from_peer_id=db_peer.peer_id,
to_peer_id=self.get_from_peer(peer).peer_id) to_peer_id=self.get_from_peer(peer).peer_id)
for peer in peers for peer in peers
] }
for peer in peers: for peer in peers:
self.db.query(DHTPeer).filter(DHTPeer.address == peer.address, DHTPeer.udp_port == peer.udp_port).update( self.db.query(DHTPeer).filter(DHTPeer.address == peer.address, DHTPeer.udp_port == peer.udp_port).update(
{DHTPeer.last_seen: datetime.datetime.utcnow()}) {DHTPeer.last_seen: datetime.datetime.utcnow()})
@ -261,8 +261,8 @@ async def test():
crawler = Crawler("/tmp/a.db") crawler = Crawler("/tmp/a.db")
await crawler.node.start_listening() await crawler.node.start_listening()
conf = Config() conf = Config()
#for (host, port) in conf.known_dht_nodes: for (host, port) in conf.known_dht_nodes:
# await crawler.crawl_routing_table(host, port) await crawler.crawl_routing_table(host, port)
await crawler.process() await crawler.process()
if __name__ == '__main__': if __name__ == '__main__':