diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6a27e8f87..063224207 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -51,7 +51,6 @@ jobs: HOME: /tmp run: coverage run -m unittest -vv tests.unit.test_conf tests.unit.blockchain tests.unit.test_event_controller tests.unit.crypto tests.unit.schema tests.unit.db # run: coverage run -m unittest discover -vv tests.unit -# - if: startsWith(runner.os, 'linux') || startsWith(runner.os, 'mac') - env: COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_PARALLEL: true @@ -71,17 +70,17 @@ jobs: # - other db: - sqlite -# - postgres -# services: -# postgres: -# image: postgres:12 -# env: -# POSTGRES_USER: postgres -# POSTGRES_PASSWORD: postgres -# POSTGRES_DB: postgres -# ports: -# - 5432:5432 -# options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + - postgres + services: + postgres: + image: postgres:12 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 steps: - uses: actions/checkout@v1 - uses: actions/setup-python@v1 diff --git a/lbry/blockchain/sync/queries.py b/lbry/blockchain/sync/queries.py index f04a224d6..89472e5a8 100644 --- a/lbry/blockchain/sync/queries.py +++ b/lbry/blockchain/sync/queries.py @@ -33,6 +33,7 @@ def select_unvalidated_signables(signable, pk, include_urls=False, include_previ (TXO.c.height <= signable.c.height) ) .order_by(desc(TXO.c.height)) + .limit(1) .scalar_subquery().label('public_key') ), ) @@ -47,7 +48,7 @@ def select_unvalidated_signables(signable, pk, include_urls=False, include_previ (TXO.c.txo_type.in_(CLAIM_TYPE_CODES)) & (TXO.c.height <= signable.c.height) ) - .order_by(desc(TXO.c.height)).offset(1) + .order_by(desc(TXO.c.height)).offset(1).limit(1) .scalar_subquery().label('previous_channel_hash') ) if include_urls: diff --git a/lbry/db/database.py b/lbry/db/database.py index c3f14264c..df1470b46 100644 --- a/lbry/db/database.py +++ b/lbry/db/database.py @@ -12,7 +12,7 @@ from lbry.event import EventController from lbry.crypto.bip32 import PubKey from lbry.blockchain.transaction import Transaction, Output from .constants import TXO_TYPES, CLAIM_TYPE_CODES -from .query_context import initialize, ProgressPublisher +from .query_context import initialize, uninitialize, ProgressPublisher from . import queries as q from . import sync @@ -99,6 +99,16 @@ class Database: return processes return 1 + @classmethod + def temp_from_url_regtest(cls, db_url, lbrycrd_dir=None): + from lbry import Config, RegTestLedger # pylint: disable=import-outside-toplevel + directory = tempfile.mkdtemp() + conf = Config.with_same_dir(directory).set(db_url=db_url) + if lbrycrd_dir is not None: + conf.lbrycrd_dir = lbrycrd_dir + ledger = RegTestLedger(conf) + return cls(ledger) + @classmethod def temp_sqlite_regtest(cls, lbrycrd_dir=None): from lbry import Config, RegTestLedger # pylint: disable=import-outside-toplevel @@ -116,11 +126,13 @@ class Database: return cls(Ledger(conf)) @classmethod - def in_memory(cls): + def from_url(cls, db_url): from lbry import Config, Ledger # pylint: disable=import-outside-toplevel - conf = Config.with_same_dir('/dev/null') - conf.db_url = 'sqlite:///:memory:' - return cls(Ledger(conf)) + return cls(Ledger(Config.with_same_dir('/dev/null').set(db_url=db_url))) + + @classmethod + def in_memory(cls): + return cls.from_url('sqlite:///:memory:') def sync_create(self, name): engine = create_engine(self.url) @@ -159,6 +171,8 @@ class Database: async def close(self): self.progress_publisher.stop() if self.executor is not None: + if isinstance(self.executor, ThreadPoolExecutor): + await self.run_in_executor(uninitialize) self.executor.shutdown() self.executor = None diff --git a/lbry/db/queries.py b/lbry/db/queries.py index a58d6f4a4..344e0d92b 100644 --- a/lbry/db/queries.py +++ b/lbry/db/queries.py @@ -835,9 +835,9 @@ def resolve_url(raw_url): if channel is not None: q['order_by'] = ['^creation_height'] q['channel_hash'] = channel.claim_hash - q['is_signature_valid'] = 1 + q['is_signature_valid'] = True elif set(q) == {'name'}: - q['is_controlling'] = 1 + q['is_controlling'] = True # matches = search_claims(censor, **q, limit=1) matches = search_claims(**q, limit=1)[0] if matches: diff --git a/lbry/db/query_context.py b/lbry/db/query_context.py index bf8c5dc6f..da9515a47 100644 --- a/lbry/db/query_context.py +++ b/lbry/db/query_context.py @@ -155,6 +155,8 @@ def uninitialize(): if ctx is not None: if ctx.connection: ctx.connection.close() + if ctx.engine: + ctx.engine.dispose() _context.set(None) diff --git a/setup.py b/setup.py index 462e30fc3..413b4b227 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,7 @@ setup( ], extras_require={ 'ui': ['pyside2'], - 'postgresql': ['psycopg2'], + 'postgres': ['psycopg2'], 'lint': ['mypy', 'pylint'], 'test': ['coverage'], }, diff --git a/tests/integration/blockchain/test_blockchain.py b/tests/integration/blockchain/test_blockchain.py index d07f624b1..307310d76 100644 --- a/tests/integration/blockchain/test_blockchain.py +++ b/tests/integration/blockchain/test_blockchain.py @@ -37,7 +37,17 @@ class BasicBlockchainTestCase(AsyncioTestCase): return Lbrycrd.temp_regtest() async def make_db(self, chain): - db = Database.temp_sqlite_regtest(chain.ledger.conf.lbrycrd_dir) + db_driver = os.environ.get('TEST_DB', 'sqlite') + if db_driver == 'sqlite': + db = Database.temp_sqlite_regtest(chain.ledger.conf.lbrycrd_dir) + elif db_driver.startswith('postgres') or db_driver.startswith('psycopg'): + db_name = f'lbry_test_chain' + meta_db = Database.from_url(f'postgresql:///postgres') + await meta_db.drop(db_name) + await meta_db.create(db_name) + db = Database.temp_from_url_regtest(f'postgresql:///{db_name}', chain.ledger.conf.lbrycrd_dir) + else: + raise RuntimeError(f"Unsupported database driver: {db_driver}") self.addCleanup(remove_tree, db.ledger.conf.data_dir) await db.open() self.addCleanup(db.close) diff --git a/tox.ini b/tox.ini index 73b77a476..645884df3 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ deps = coverage -extras = test +extras = test,postgres changedir = {toxinidir}/tests setenv = HOME=/tmp