closes 2281 - fix too many sql variables + tests

torba/tests/client_tests/unit/test_database.py

@@ -1,5 +1,6 @@
 import unittest
 import sqlite3
+from functools import wraps
 
 from torba.client.wallet import Wallet
 from torba.client.constants import COIN
@@ -224,6 +225,21 @@ class TestQueries(AsyncioTestCase):
     def txi(self, txo):
         return ledger_class.transaction_class.input_class.spend(txo)
 
+    async def test_large_tx_doesnt_hit_variable_limits(self):
+        # SQLite is usually compiled with 999 variables limit: https://www.sqlite.org/limits.html
+        # This can be removed when there is a better way. See: https://github.com/lbryio/lbry-sdk/issues/2281
+        fetchall = self.ledger.db.db.execute_fetchall
+        def check_parameters_length(sql, parameters):
+            self.assertLess(len(parameters or []), 999)
+            return fetchall(sql, parameters)
+
+        self.ledger.db.db.execute_fetchall = check_parameters_length
+        account = await self.create_account()
+        tx = await self.create_tx_from_nothing(account, 1)
+        for _ in range(1200):
+            tx = await self.create_tx_from_txo(tx.outputs[0], account, 1)
+        await self.ledger.db.get_transactions()
+
     async def test_queries(self):
         self.assertEqual(0, await self.ledger.db.get_address_count())
         account1 = await self.create_account()

torba/torba/client/basedatabase.py

@@ -412,21 +412,25 @@ class BaseDatabase(SQLiteMixin):
             for txi in txs[-1].inputs:
                 txi_txoids.append(txi.txo_ref.id)
 
-        annotated_txos = {
+        annotated_txos = {}
+        for offset in range(0, len(txids), 900):
+            annotated_txos.update({
                 txo.id: txo for txo in
                 (await self.get_txos(
                     my_account=my_account,
-                txid__in=txids
+                    txid__in=txids[offset:offset+900],
                 ))
-        }
+            })
 
-        referenced_txos = {
+        referenced_txos = {}
+        for offset in range(0, len(txi_txoids), 900):
+            referenced_txos.update({
                 txo.id: txo for txo in
                 (await self.get_txos(
                     my_account=my_account,
-                txoid__in=txi_txoids
+                    txoid__in=txi_txoids[offset:offset+900],
                 ))
-        }
+            })
 
         for tx in txs:
             for txi in tx.inputs: