LBRYCommunity/lbry-sdk
1
0
Fork 1
Code Issues Pull requests Projects Releases 282 Packages Wiki Activity

sqlite performance script measures sql time inside each process individually

Browse source
This commit is contained in:
Lex Berezhny 2019-07-11 20:10:28 -04:00
parent b81c801f2d
commit c5aeac6898
1 changed files with 53 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
lbry/scripts/sqlite_perf_test.py
View file

@ -1,4 +1,4 @@
import asyncio, sqlite3, time import uvloop, asyncio, sqlite3, time, sys
from concurrent.futures import ProcessPoolExecutor from concurrent.futures import ProcessPoolExecutor
from contextvars import ContextVar from contextvars import ContextVar
@ -14,7 +14,7 @@ def init():
def reader(): def reader():
conn = db.get() conn = db.get()
for _ in range(1): start = time.time()
conn.execute(""" conn.execute("""
SELECT SELECT
claimtrie.claim_hash as is_controlling, claimtrie.claim_hash as is_controlling,
@ -45,28 +45,39 @@ def reader():
ORDER BY claim.height DESC, claim.normalized ASC ORDER BY claim.height DESC, claim.normalized ASC
LIMIT 20 OFFSET 100 LIMIT 20 OFFSET 100
""").fetchall() """).fetchall()
elapsed = time.time() - start
return elapsed
async def run_times(executor, iterations, show=True): async def run_times(executor, iterations, show=True):
start = time.time() start = time.time()
await asyncio.gather(*(asyncio.get_running_loop().run_in_executor(executor, reader) for _ in range(iterations))) timings = await asyncio.gather(*(
elapsed = time.time() - start asyncio.get_running_loop().run_in_executor(executor, reader) for _ in range(iterations)
))
total = time.time() - start
if show: if show:
print(f"{iterations:3}: {elapsed:.5f}ms total, {elapsed/iterations:.5f}ms/query") avg = sum(timings)/len(timings)
print(f"{iterations:4}: {total:.5f}ms total concurrent, {len(timings)*avg:.5f}ms total sequential (avg*runs)")
print(f" {total/len(timings):.5f}ms/query concurrent (total/runs)")
print(f" {avg:.5f}ms/query actual average (sum(queries)/runs)")
sys.stdout.write(' sample:')
for i, t in zip(range(10), timings[::-1]):
sys.stdout.write(f' {t:.5f}ms')
print(' ...\n' if len(timings) > 10 else '\n')
async def main(): async def main():
executor = ProcessPoolExecutor(4, initializer=init) executor = ProcessPoolExecutor(4, initializer=init)
await run_times(executor, 4, show=False) await run_times(executor, 4, show=False)
await run_times(executor, 1) await run_times(executor, 1)
await run_times(executor, 4) await run_times(executor, 2**3)
await run_times(executor, 8) await run_times(executor, 2**5)
await run_times(executor, 16) await run_times(executor, 2**7)
await run_times(executor, 32) await run_times(executor, 2**9)
await run_times(executor, 64) await run_times(executor, 2**11)
await run_times(executor, 128) await run_times(executor, 2**13)
await run_times(executor, 256)
executor.shutdown(True) executor.shutdown(True)
if __name__ == '__main__': if __name__ == '__main__':
uvloop.install()
asyncio.run(main()) asyncio.run(main())