From 82eec3d8d782af2a028a4107d0f009060cfb3a43 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sun, 24 Jan 2021 23:19:28 -0300 Subject: [PATCH] use multiple clients on sync script indexing --- scripts/sync.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/sync.py b/scripts/sync.py index 16668e81b..3e2da2504 100644 --- a/scripts/sync.py +++ b/scripts/sync.py @@ -8,7 +8,6 @@ from elasticsearch.helpers import async_bulk from lbry.wallet.server.db.elastic_search import extract_doc, SearchIndex -es = AsyncElasticsearch() INDEX = 'claims' @@ -37,15 +36,23 @@ FROM claim LEFT JOIN claimtrie USING (claim_hash) yield extract_doc(claim, INDEX) +async def consume(producer): + es = AsyncElasticsearch() + await async_bulk(es, producer) + await es.close() + + async def main(): parser = argparse.ArgumentParser() parser.add_argument("db_path", type=str) + parser.add_argument("-c", "--clients", type=int, default=16) args = parser.parse_args() db = apsw.Connection(args.db_path, flags=apsw.SQLITE_OPEN_READONLY | apsw.SQLITE_OPEN_URI) index = SearchIndex('') await index.start() await index.stop() - await async_bulk(es, get_all(db.cursor())) + producer = get_all(db.cursor()) + await asyncio.gather(*(consume(producer) for _ in range(args.clients))) if __name__ == '__main__':