LBRYCommunity/lbry-sdk
1
0
Fork 1
Code Issues Pull requests Projects Releases 282 Packages Wiki Activity

batch under sqlite variable limit

Browse source
This commit is contained in:
Victor Shyba 2020-11-25 15:07:24 -03:00 • committed by Lex Berezhny
parent 435a80d4c3
commit ea691e78b8
3 changed files with 20 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
lbry/db/queries/address.py
View file

@ -60,5 +60,7 @@ def get_all_addresses(self):
def add_keys(pubkeys): def add_keys(pubkeys):
c = context() c = context()
c.execute(c.insert_or_ignore(PubkeyAddress).values([{'address': k['address']} for k in pubkeys])) for start in range(0, len(pubkeys), c.variable_limit - 1):
c.execute(c.insert_or_ignore(AccountAddress).values(pubkeys)) batch = pubkeys[start:(start + c.variable_limit - 1)]
c.execute(c.insert_or_ignore(PubkeyAddress).values([{'address': k['address']} for k in batch]))
c.execute(c.insert_or_ignore(AccountAddress).values(batch))

14
lbry/db/query_context.py
View file

@ -57,6 +57,7 @@ class QueryContext:
current_progress: Optional['ProgressContext'] = None current_progress: Optional['ProgressContext'] = None
copy_managers: Dict[str, CopyManager] = field(default_factory=dict) copy_managers: Dict[str, CopyManager] = field(default_factory=dict)
_variable_limit: Optional[int] = None
@property @property
def is_postgres(self): def is_postgres(self):
@ -66,6 +67,19 @@ class QueryContext:
def is_sqlite(self): def is_sqlite(self):
return self.engine.dialect.name == 'sqlite' return self.engine.dialect.name == 'sqlite'
@property
def variable_limit(self):
if self._variable_limit is not None:
return self._variable_limit
if self.is_sqlite:
for result in self.fetchall('PRAGMA COMPILE_OPTIONS;'):
for _, value in result.items():
if value.startswith('MAX_VARIABLE_NUMBER'):
self._variable_limit = int(value.split('=')[1])
return self._variable_limit
self._variable_limit = 32766 # default for 3.32.0 and large enough for other cases
return self._variable_limit
def raise_unsupported_dialect(self): def raise_unsupported_dialect(self):
raise RuntimeError(f'Unsupported database dialect: {self.engine.dialect.name}.') raise RuntimeError(f'Unsupported database dialect: {self.engine.dialect.name}.')

4
tests/unit/wallet/test_account.py
View file

@ -82,9 +82,9 @@ class TestHierarchicalDeterministicAccount(AccountTestCase):
async def test_generate_keys_over_batch_threshold_saves_it_properly(self): async def test_generate_keys_over_batch_threshold_saves_it_properly(self):
account = await Account.generate(self.db) account = await Account.generate(self.db)
async with account.receiving.address_generator_lock: async with account.receiving.address_generator_lock:
await account.receiving._generate_keys(0, 200) await account.receiving._generate_keys(0, 1000)
records = await account.receiving.get_address_records() records = await account.receiving.get_address_records()
self.assertEqual(len(records), 201) self.assertEqual(len(records), 1001)
async def test_get_or_create_usable_address(self): async def test_get_or_create_usable_address(self):
account = await Account.generate(self.db) account = await Account.generate(self.db)