From ea691e78b8d9fd679ef344236bc8de28f137efdc Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Wed, 25 Nov 2020 15:07:24 -0300
Subject: [PATCH] batch under sqlite variable limit

---
 lbry/db/queries/address.py        |  6 ++++--
 lbry/db/query_context.py          | 14 ++++++++++++++
 tests/unit/wallet/test_account.py |  4 ++--
 3 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/lbry/db/queries/address.py b/lbry/db/queries/address.py
index 5873384e2..070d6c080 100644
--- a/lbry/db/queries/address.py
+++ b/lbry/db/queries/address.py
@@ -60,5 +60,7 @@ def get_all_addresses(self):
 
 def add_keys(pubkeys):
     c = context()
-    c.execute(c.insert_or_ignore(PubkeyAddress).values([{'address': k['address']} for k in pubkeys]))
-    c.execute(c.insert_or_ignore(AccountAddress).values(pubkeys))
+    for start in range(0, len(pubkeys), c.variable_limit - 1):
+        batch = pubkeys[start:(start + c.variable_limit - 1)]
+        c.execute(c.insert_or_ignore(PubkeyAddress).values([{'address': k['address']} for k in batch]))
+        c.execute(c.insert_or_ignore(AccountAddress).values(batch))
diff --git a/lbry/db/query_context.py b/lbry/db/query_context.py
index 444417871..28384f182 100644
--- a/lbry/db/query_context.py
+++ b/lbry/db/query_context.py
@@ -57,6 +57,7 @@ class QueryContext:
     current_progress: Optional['ProgressContext'] = None
 
     copy_managers: Dict[str, CopyManager] = field(default_factory=dict)
+    _variable_limit: Optional[int] = None
 
     @property
     def is_postgres(self):
@@ -66,6 +67,19 @@ class QueryContext:
     def is_sqlite(self):
         return self.engine.dialect.name == 'sqlite'
 
+    @property
+    def variable_limit(self):
+        if self._variable_limit is not None:
+            return self._variable_limit
+        if self.is_sqlite:
+            for result in self.fetchall('PRAGMA COMPILE_OPTIONS;'):
+                for _, value in result.items():
+                    if value.startswith('MAX_VARIABLE_NUMBER'):
+                        self._variable_limit = int(value.split('=')[1])
+                        return self._variable_limit
+        self._variable_limit = 32766  # default for 3.32.0 and large enough for other cases
+        return self._variable_limit
+
     def raise_unsupported_dialect(self):
         raise RuntimeError(f'Unsupported database dialect: {self.engine.dialect.name}.')
 
diff --git a/tests/unit/wallet/test_account.py b/tests/unit/wallet/test_account.py
index 4c0fc1c71..ff39912f6 100644
--- a/tests/unit/wallet/test_account.py
+++ b/tests/unit/wallet/test_account.py
@@ -82,9 +82,9 @@ class TestHierarchicalDeterministicAccount(AccountTestCase):
     async def test_generate_keys_over_batch_threshold_saves_it_properly(self):
         account = await Account.generate(self.db)
         async with account.receiving.address_generator_lock:
-            await account.receiving._generate_keys(0, 200)
+            await account.receiving._generate_keys(0, 1000)
         records = await account.receiving.get_address_records()
-        self.assertEqual(len(records), 201)
+        self.assertEqual(len(records), 1001)
 
     async def test_get_or_create_usable_address(self):
         account = await Account.generate(self.db)