LBRYCommunity/lbry-sdk
1
0
Fork 1
Code Issues Pull requests Projects Releases 282 Packages Wiki Activity

use multiple clients on sync script indexing

Browse source
This commit is contained in:
Victor Shyba 2021-01-24 23:19:28 -03:00
parent ee7b37d3f3
commit 82eec3d8d7
1 changed files with 9 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
scripts/sync.py
View file

@ -8,7 +8,6 @@ from elasticsearch.helpers import async_bulk
from lbry.wallet.server.db.elastic_search import extract_doc, SearchIndex from lbry.wallet.server.db.elastic_search import extract_doc, SearchIndex
es = AsyncElasticsearch()
INDEX = 'claims' INDEX = 'claims'
@ -37,15 +36,23 @@ FROM claim LEFT JOIN claimtrie USING (claim_hash)
yield extract_doc(claim, INDEX) yield extract_doc(claim, INDEX)
async def consume(producer):
es = AsyncElasticsearch()
await async_bulk(es, producer)
await es.close()
async def main(): async def main():
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("db_path", type=str) parser.add_argument("db_path", type=str)
parser.add_argument("-c", "--clients", type=int, default=16)
args = parser.parse_args() args = parser.parse_args()
db = apsw.Connection(args.db_path, flags=apsw.SQLITE_OPEN_READONLY | apsw.SQLITE_OPEN_URI) db = apsw.Connection(args.db_path, flags=apsw.SQLITE_OPEN_READONLY | apsw.SQLITE_OPEN_URI)
index = SearchIndex('') index = SearchIndex('')
await index.start() await index.start()
await index.stop() await index.stop()
await async_bulk(es, get_all(db.cursor())) producer = get_all(db.cursor())
await asyncio.gather(*(consume(producer) for _ in range(args.clients)))
if __name__ == '__main__': if __name__ == '__main__':