From 10c262a095b10ecf09a295d5ee2a5d642df2b4f0 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Sun, 12 Apr 2020 11:06:05 -0400 Subject: [PATCH] postgres --- .github/workflows/main.yml | 10 +++++ lbry/db/database.py | 22 +++++---- lbry/db/tables.py | 86 +++++++++++++++++++----------------- lbry/wallet/orchstr8/node.py | 5 ++- 4 files changed, 72 insertions(+), 51 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fc5ee876c..4c4a6d8c3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -39,6 +39,16 @@ jobs: db: - postgres - sqlite + services: + postgres: + image: postgres:12 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 steps: - uses: actions/checkout@v1 - uses: actions/setup-python@v1 diff --git a/lbry/db/database.py b/lbry/db/database.py index 0fe7dbe59..21f0304a3 100644 --- a/lbry/db/database.py +++ b/lbry/db/database.py @@ -21,7 +21,13 @@ from lbry.wallet import PubKey from lbry.wallet.transaction import Transaction, Output, OutputScript, TXRefImmutable from lbry.wallet.constants import TXO_TYPES, CLAIM_TYPES -from .tables import metadata, Version, TX, TXI, TXO, PubkeyAddress, AccountAddress +from .tables import ( + metadata, Version, + PubkeyAddress, AccountAddress, + TX, + TXO, txo_join_account, + TXI, txi_join_account, +) log = logging.getLogger(__name__) @@ -229,8 +235,8 @@ class Database: def sync_create(self, name): engine = sqlalchemy.create_engine(self.url) db = engine.connect() - db.execute('commit') - db.execute(f'create database {name}') + db.execute(text('commit')) + db.execute(text(f'create database {name}')) async def create(self, name): await asyncio.get_event_loop().run_in_executor( @@ -240,8 +246,8 @@ class Database: def sync_drop(self, name): engine = sqlalchemy.create_engine(self.url) db = engine.connect() - db.execute('commit') - db.execute(f'drop database if exists {name}') + db.execute(text('commit')) + db.execute(text(f'drop database if exists {name}')) async def drop(self, name): await asyncio.get_event_loop().run_in_executor( @@ -366,8 +372,8 @@ class Database: assert accounts, "'accounts' argument required when no 'tx_hash' constraint is present" where = in_account(accounts) tx_hashes = union( - select(TXO.c.tx_hash).select_from(TXO.join(AccountAddress)).where(where), - select(TXI.c.tx_hash).select_from(TXI.join(AccountAddress)).where(where) + select(TXO.c.tx_hash).select_from(txo_join_account).where(where), + select(TXI.c.tx_hash).select_from(txi_join_account).where(where) ) s = s.where(TX.c.tx_hash.in_(tx_hashes)) return await self.execute_fetchall(query2([TX], s, **constraints)) @@ -753,7 +759,7 @@ class Database: if not {'purchased_claim_hash', 'purchased_claim_hash__in'}.intersection(constraints): constraints['purchased_claim_hash__is_not_null'] = True constraints['tx_hash__in'] = ( - select(TXI.c.tx_hash).select_from(TXI.join(AccountAddress)).where(in_account(accounts)) + select(TXI.c.tx_hash).select_from(txi_join_account).where(in_account(accounts)) ) async def get_purchases(self, **constraints): diff --git a/lbry/db/tables.py b/lbry/db/tables.py index eca336f52..a9eb09067 100644 --- a/lbry/db/tables.py +++ b/lbry/db/tables.py @@ -2,7 +2,7 @@ from sqlalchemy import ( MetaData, Table, Column, ForeignKey, - BINARY, TEXT, SMALLINT, INTEGER, BOOLEAN + LargeBinary, Text, SmallInteger, Integer, Boolean ) @@ -11,74 +11,78 @@ metadata = MetaData() Version = Table( 'version', metadata, - Column('version', TEXT, primary_key=True), + Column('version', Text, primary_key=True), ) PubkeyAddress = Table( 'pubkey_address', metadata, - Column('address', TEXT, primary_key=True), - Column('history', TEXT, nullable=True), - Column('used_times', INTEGER, server_default='0'), + Column('address', Text, primary_key=True), + Column('history', Text, nullable=True), + Column('used_times', Integer, server_default='0'), ) AccountAddress = Table( 'account_address', metadata, - Column('account', TEXT, primary_key=True), - Column('address', TEXT, ForeignKey(PubkeyAddress.columns.address), primary_key=True), - Column('chain', INTEGER), - Column('pubkey', BINARY), - Column('chain_code', BINARY), - Column('n', INTEGER), - Column('depth', INTEGER), + Column('account', Text, primary_key=True), + Column('address', Text, ForeignKey(PubkeyAddress.columns.address), primary_key=True), + Column('chain', Integer), + Column('pubkey', LargeBinary), + Column('chain_code', LargeBinary), + Column('n', Integer), + Column('depth', Integer), ) Block = Table( 'block', metadata, - Column('block_hash', BINARY, primary_key=True), - Column('previous_hash', BINARY), - Column('file_number', SMALLINT), - Column('height', INTEGER), + Column('block_hash', LargeBinary, primary_key=True), + Column('previous_hash', LargeBinary), + Column('file_number', SmallInteger), + Column('height', Integer), ) TX = Table( 'tx', metadata, - Column('block_hash', BINARY, nullable=True), - Column('tx_hash', BINARY, primary_key=True), - Column('raw', BINARY), - Column('height', INTEGER), - Column('position', SMALLINT), - Column('is_verified', BOOLEAN, server_default='FALSE'), - Column('purchased_claim_hash', BINARY, nullable=True), - Column('day', INTEGER, nullable=True), + Column('block_hash', LargeBinary, nullable=True), + Column('tx_hash', LargeBinary, primary_key=True), + Column('raw', LargeBinary), + Column('height', Integer), + Column('position', SmallInteger), + Column('is_verified', Boolean, server_default='FALSE'), + Column('purchased_claim_hash', LargeBinary, nullable=True), + Column('day', Integer, nullable=True), ) TXO = Table( 'txo', metadata, - Column('tx_hash', BINARY, ForeignKey(TX.columns.tx_hash)), - Column('txo_hash', BINARY, primary_key=True), - Column('address', TEXT, ForeignKey(AccountAddress.columns.address)), - Column('position', INTEGER), - Column('amount', INTEGER), - Column('script', BINARY), - Column('is_reserved', BOOLEAN, server_default='0'), - Column('txo_type', INTEGER, server_default='0'), - Column('claim_id', TEXT, nullable=True), - Column('claim_hash', BINARY, nullable=True), - Column('claim_name', TEXT, nullable=True), - Column('channel_hash', BINARY, nullable=True), - Column('reposted_claim_hash', BINARY, nullable=True), + Column('tx_hash', LargeBinary, ForeignKey(TX.columns.tx_hash)), + Column('txo_hash', LargeBinary, primary_key=True), + Column('address', Text), + Column('position', Integer), + Column('amount', Integer), + Column('script', LargeBinary), + Column('is_reserved', Boolean, server_default='0'), + Column('txo_type', Integer, server_default='0'), + Column('claim_id', Text, nullable=True), + Column('claim_hash', LargeBinary, nullable=True), + Column('claim_name', Text, nullable=True), + Column('channel_hash', LargeBinary, nullable=True), + Column('reposted_claim_hash', LargeBinary, nullable=True), ) +txo_join_account = TXO.join(AccountAddress, TXO.columns.address == AccountAddress.columns.address) + TXI = Table( 'txi', metadata, - Column('tx_hash', BINARY, ForeignKey(TX.columns.tx_hash)), - Column('txo_hash', BINARY, ForeignKey(TXO.columns.txo_hash), primary_key=True), - Column('address', TEXT, ForeignKey(AccountAddress.columns.address)), - Column('position', INTEGER), + Column('tx_hash', LargeBinary, ForeignKey(TX.columns.tx_hash)), + Column('txo_hash', LargeBinary, ForeignKey(TXO.columns.txo_hash), primary_key=True), + Column('address', Text), + Column('position', Integer), ) + +txi_join_account = TXI.join(AccountAddress, TXI.columns.address == AccountAddress.columns.address) diff --git a/lbry/wallet/orchstr8/node.py b/lbry/wallet/orchstr8/node.py index 8c6973e22..039fffbee 100644 --- a/lbry/wallet/orchstr8/node.py +++ b/lbry/wallet/orchstr8/node.py @@ -130,11 +130,12 @@ class WalletNode: if db_driver == 'sqlite': db = 'sqlite:///'+os.path.join(self.data_path, self.ledger_class.get_id(), 'blockchain.db') elif db_driver == 'postgres': + db_connection = 'postgres:postgres@localhost:5432' db_name = f'lbry_test_{self.port}' - meta_db = Database(f'postgresql+psycopg2:///postgres') + meta_db = Database(f'postgresql+psycopg2://{db_connection}/postgres') await meta_db.drop(db_name) await meta_db.create(db_name) - db = f'postgresql+psycopg2:///{db_name}' + db = f'postgresql+psycopg2://{db_connection}/{db_name}' else: raise RuntimeError(f"Unsupported database driver: {db_driver}") self.manager = self.manager_class.from_config({