batch under sqlite variable limit

lbry/db/queries/address.py

@@ -60,5 +60,7 @@ def get_all_addresses(self):
 
 def add_keys(pubkeys):
     c = context()
-    c.execute(c.insert_or_ignore(PubkeyAddress).values([{'address': k['address']} for k in pubkeys]))
-    c.execute(c.insert_or_ignore(AccountAddress).values(pubkeys))
+    for start in range(0, len(pubkeys), c.variable_limit - 1):
+        batch = pubkeys[start:(start + c.variable_limit - 1)]
+        c.execute(c.insert_or_ignore(PubkeyAddress).values([{'address': k['address']} for k in batch]))
+        c.execute(c.insert_or_ignore(AccountAddress).values(batch))

lbry/db/query_context.py

@@ -57,6 +57,7 @@ class QueryContext:
     current_progress: Optional['ProgressContext'] = None
 
     copy_managers: Dict[str, CopyManager] = field(default_factory=dict)
+    _variable_limit: Optional[int] = None
 
     @property
     def is_postgres(self):
@@ -66,6 +67,19 @@ class QueryContext:
     def is_sqlite(self):
         return self.engine.dialect.name == 'sqlite'
 
+    @property
+    def variable_limit(self):
+        if self._variable_limit is not None:
+            return self._variable_limit
+        if self.is_sqlite:
+            for result in self.fetchall('PRAGMA COMPILE_OPTIONS;'):
+                for _, value in result.items():
+                    if value.startswith('MAX_VARIABLE_NUMBER'):
+                        self._variable_limit = int(value.split('=')[1])
+                        return self._variable_limit
+        self._variable_limit = 32766  # default for 3.32.0 and large enough for other cases
+        return self._variable_limit
+
     def raise_unsupported_dialect(self):
         raise RuntimeError(f'Unsupported database dialect: {self.engine.dialect.name}.')
 

tests/unit/wallet/test_account.py

@@ -82,9 +82,9 @@ class TestHierarchicalDeterministicAccount(AccountTestCase):
     async def test_generate_keys_over_batch_threshold_saves_it_properly(self):
         account = await Account.generate(self.db)
         async with account.receiving.address_generator_lock:
-            await account.receiving._generate_keys(0, 200)
+            await account.receiving._generate_keys(0, 1000)
         records = await account.receiving.get_address_records()
-        self.assertEqual(len(records), 201)
+        self.assertEqual(len(records), 1001)
 
     async def test_get_or_create_usable_address(self):
         account = await Account.generate(self.db)