re-enable postgres integration tests

.github/workflows/main.yml

@@ -51,7 +51,6 @@ jobs:
           HOME: /tmp
         run: coverage run -m unittest -vv tests.unit.test_conf tests.unit.blockchain tests.unit.test_event_controller tests.unit.crypto tests.unit.schema tests.unit.db
 #        run: coverage run -m unittest discover -vv tests.unit
-#      - if: startsWith(runner.os, 'linux') || startsWith(runner.os, 'mac')
       - env:
           COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           COVERALLS_PARALLEL: true
@@ -71,17 +70,17 @@ jobs:
 #          - other
         db:
           - sqlite
-#          - postgres
+          - postgres
-#    services:
+    services:
-#      postgres:
+      postgres:
-#        image: postgres:12
+        image: postgres:12
-#        env:
+        env:
-#          POSTGRES_USER: postgres
+          POSTGRES_USER: postgres
-#          POSTGRES_PASSWORD: postgres
+          POSTGRES_PASSWORD: postgres
-#          POSTGRES_DB: postgres
+          POSTGRES_DB: postgres
-#        ports:
+        ports:
-#        - 5432:5432
+        - 5432:5432
-#        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
     steps:
       - uses: actions/checkout@v1
       - uses: actions/setup-python@v1

lbry/blockchain/sync/queries.py

@@ -33,6 +33,7 @@ def select_unvalidated_signables(signable, pk, include_urls=False, include_previ
                     (TXO.c.height <= signable.c.height)
                 )
                 .order_by(desc(TXO.c.height))
+                .limit(1)
                 .scalar_subquery().label('public_key')
             ),
         )
@@ -47,7 +48,7 @@ def select_unvalidated_signables(signable, pk, include_urls=False, include_previ
                 (TXO.c.txo_type.in_(CLAIM_TYPE_CODES)) &
                 (TXO.c.height <= signable.c.height)
             )
-            .order_by(desc(TXO.c.height)).offset(1)
+            .order_by(desc(TXO.c.height)).offset(1).limit(1)
             .scalar_subquery().label('previous_channel_hash')
         )
     if include_urls:

lbry/db/database.py

@@ -12,7 +12,7 @@ from lbry.event import EventController
 from lbry.crypto.bip32 import PubKey
 from lbry.blockchain.transaction import Transaction, Output
 from .constants import TXO_TYPES, CLAIM_TYPE_CODES
-from .query_context import initialize, ProgressPublisher
+from .query_context import initialize, uninitialize, ProgressPublisher
 from . import queries as q
 from . import sync
 
@@ -99,6 +99,16 @@ class Database:
             return processes
         return 1
 
+    @classmethod
+    def temp_from_url_regtest(cls, db_url, lbrycrd_dir=None):
+        from lbry import Config, RegTestLedger  # pylint: disable=import-outside-toplevel
+        directory = tempfile.mkdtemp()
+        conf = Config.with_same_dir(directory).set(db_url=db_url)
+        if lbrycrd_dir is not None:
+            conf.lbrycrd_dir = lbrycrd_dir
+        ledger = RegTestLedger(conf)
+        return cls(ledger)
+
     @classmethod
     def temp_sqlite_regtest(cls, lbrycrd_dir=None):
         from lbry import Config, RegTestLedger  # pylint: disable=import-outside-toplevel
@@ -116,11 +126,13 @@ class Database:
         return cls(Ledger(conf))
 
     @classmethod
-    def in_memory(cls):
+    def from_url(cls, db_url):
         from lbry import Config, Ledger  # pylint: disable=import-outside-toplevel
-        conf = Config.with_same_dir('/dev/null')
+        return cls(Ledger(Config.with_same_dir('/dev/null').set(db_url=db_url)))
-        conf.db_url = 'sqlite:///:memory:'
+
-        return cls(Ledger(conf))
+    @classmethod
+    def in_memory(cls):
+        return cls.from_url('sqlite:///:memory:')
 
     def sync_create(self, name):
         engine = create_engine(self.url)
@@ -159,6 +171,8 @@ class Database:
     async def close(self):
         self.progress_publisher.stop()
         if self.executor is not None:
+            if isinstance(self.executor, ThreadPoolExecutor):
+                await self.run_in_executor(uninitialize)
             self.executor.shutdown()
             self.executor = None
 

lbry/db/queries.py

@@ -835,9 +835,9 @@ def resolve_url(raw_url):
         if channel is not None:
             q['order_by'] = ['^creation_height']
             q['channel_hash'] = channel.claim_hash
-            q['is_signature_valid'] = 1
+            q['is_signature_valid'] = True
         elif set(q) == {'name'}:
-            q['is_controlling'] = 1
+            q['is_controlling'] = True
         # matches = search_claims(censor, **q, limit=1)
         matches = search_claims(**q, limit=1)[0]
         if matches:

lbry/db/query_context.py

@@ -155,6 +155,8 @@ def uninitialize():
     if ctx is not None:
         if ctx.connection:
             ctx.connection.close()
+        if ctx.engine:
+            ctx.engine.dispose()
         _context.set(None)
 
 

setup.py

@@ -62,7 +62,7 @@ setup(
     ],
     extras_require={
         'ui': ['pyside2'],
-        'postgresql': ['psycopg2'],
+        'postgres': ['psycopg2'],
         'lint': ['mypy', 'pylint'],
         'test': ['coverage'],
     },

tests/integration/blockchain/test_blockchain.py

@@ -37,7 +37,17 @@ class BasicBlockchainTestCase(AsyncioTestCase):
         return Lbrycrd.temp_regtest()
 
     async def make_db(self, chain):
-        db = Database.temp_sqlite_regtest(chain.ledger.conf.lbrycrd_dir)
+        db_driver = os.environ.get('TEST_DB', 'sqlite')
+        if db_driver == 'sqlite':
+            db = Database.temp_sqlite_regtest(chain.ledger.conf.lbrycrd_dir)
+        elif db_driver.startswith('postgres') or db_driver.startswith('psycopg'):
+            db_name = f'lbry_test_chain'
+            meta_db = Database.from_url(f'postgresql:///postgres')
+            await meta_db.drop(db_name)
+            await meta_db.create(db_name)
+            db = Database.temp_from_url_regtest(f'postgresql:///{db_name}', chain.ledger.conf.lbrycrd_dir)
+        else:
+            raise RuntimeError(f"Unsupported database driver: {db_driver}")
         self.addCleanup(remove_tree, db.ledger.conf.data_dir)
         await db.open()
         self.addCleanup(db.close)

tox.ini

@@ -2,7 +2,7 @@
 deps =
   coverage
 
-extras = test
+extras = test,postgres
 changedir = {toxinidir}/tests
 setenv =
   HOME=/tmp