From 0eeab397cfc40c9e6455367d80a958600698765d Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 18 May 2022 10:44:42 -0400 Subject: [PATCH 01/13] move files to docs folder --- blocking.md => docs/blocking.md | 0 cluster_guide.md => docs/cluster_guide.md | 0 diagram.png => docs/diagram.png | Bin .../elasticsearch => docs}/trending algorithm.pdf | Bin 4 files changed, 0 insertions(+), 0 deletions(-) rename blocking.md => docs/blocking.md (100%) rename cluster_guide.md => docs/cluster_guide.md (100%) rename diagram.png => docs/diagram.png (100%) rename {scribe/elasticsearch => docs}/trending algorithm.pdf (100%) diff --git a/blocking.md b/docs/blocking.md similarity index 100% rename from blocking.md rename to docs/blocking.md diff --git a/cluster_guide.md b/docs/cluster_guide.md similarity index 100% rename from cluster_guide.md rename to docs/cluster_guide.md diff --git a/diagram.png b/docs/diagram.png similarity index 100% rename from diagram.png rename to docs/diagram.png diff --git a/scribe/elasticsearch/trending algorithm.pdf b/docs/trending algorithm.pdf similarity index 100% rename from scribe/elasticsearch/trending algorithm.pdf rename to docs/trending algorithm.pdf -- 2.45.2 From 27e09d7aa7906f54ebeae2a1f19e372c877655be Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 18 May 2022 10:50:20 -0400 Subject: [PATCH 02/13] rename repo to `lbry-hub` and module to `hub` --- docker/set_build.py | 2 +- {scribe => hub}/__init__.py | 0 hub/blockchain/__init__.py | 1 + {scribe => hub}/blockchain/__main__.py | 6 ++-- {scribe => hub}/blockchain/daemon.py | 4 +-- {scribe => hub}/blockchain/env.py | 2 +- {scribe => hub}/blockchain/mempool.py | 4 +-- {scribe => hub}/blockchain/network.py | 12 ++++---- {scribe => hub}/blockchain/prefetcher.py | 4 +-- {scribe => hub}/blockchain/service.py | 30 +++++++++---------- .../blockchain/transaction/__init__.py | 4 +-- .../blockchain/transaction/deserializer.py | 6 ++-- .../blockchain/transaction/script.py | 4 +-- {scribe => hub}/build_info.py | 0 {scribe => hub}/common.py | 0 {scribe => hub}/db/__init__.py | 0 {scribe => hub}/db/common.py | 2 +- {scribe => hub}/db/db.py | 26 ++++++++-------- {scribe => hub}/db/interface.py | 4 +-- {scribe => hub}/db/merkle.py | 2 +- {scribe => hub}/db/migrators/__init__.py | 0 {scribe => hub}/db/migrators/migrate7to8.py | 4 +-- {scribe => hub}/db/migrators/migrate8to9.py | 0 {scribe => hub}/db/prefixes.py | 6 ++-- {scribe => hub}/db/revertable.py | 4 +-- hub/elasticsearch/__init__.py | 2 ++ {scribe => hub}/elasticsearch/__main__.py | 6 ++-- {scribe => hub}/elasticsearch/constants.py | 0 {scribe => hub}/elasticsearch/env.py | 2 +- .../elasticsearch/fast_ar_trending.py | 0 .../elasticsearch/notifier_protocol.py | 0 {scribe => hub}/elasticsearch/search.py | 18 +++++------ {scribe => hub}/elasticsearch/service.py | 18 +++++------ {scribe => hub}/env.py | 2 +- {scribe => hub}/error/Makefile | 0 {scribe => hub}/error/README.md | 0 {scribe => hub}/error/__init__.py | 0 {scribe => hub}/error/base.py | 0 {scribe => hub}/error/generate.py | 0 {scribe => hub}/hub/__init__.py | 0 {scribe => hub}/hub/__main__.py | 6 ++-- {scribe => hub}/hub/common.py | 2 +- {scribe => hub}/hub/env.py | 2 +- {scribe => hub}/hub/framer.py | 0 {scribe => hub}/hub/jsonrpc.py | 4 +-- {scribe => hub}/hub/mempool.py | 12 ++++---- {scribe => hub}/hub/service.py | 14 ++++----- {scribe => hub}/hub/session.py | 30 +++++++++---------- {scribe => hub}/hub/udp.py | 4 +-- {scribe => hub}/metrics.py | 0 {scribe => hub}/schema/Makefile | 0 {scribe => hub}/schema/README.md | 0 {scribe => hub}/schema/__init__.py | 0 {scribe => hub}/schema/attrs.py | 12 ++++---- {scribe => hub}/schema/base.py | 0 {scribe => hub}/schema/base58.py | 0 {scribe => hub}/schema/bip32.py | 2 +- {scribe => hub}/schema/claim.py | 12 ++++---- {scribe => hub}/schema/compat.py | 6 ++-- {scribe => hub}/schema/mime_types.py | 0 {scribe => hub}/schema/purchase.py | 2 +- {scribe => hub}/schema/result.py | 8 ++--- {scribe => hub}/schema/support.py | 4 +-- {scribe => hub}/schema/tags.py | 0 {scribe => hub}/schema/types/__init__.py | 0 {scribe => hub}/schema/types/v1/__init__.py | 0 .../schema/types/v1/certificate_pb2.py | 0 {scribe => hub}/schema/types/v1/fee_pb2.py | 0 .../schema/types/v1/legacy_claim_pb2.py | 0 .../schema/types/v1/metadata_pb2.py | 0 .../schema/types/v1/signature_pb2.py | 0 {scribe => hub}/schema/types/v1/source_pb2.py | 0 {scribe => hub}/schema/types/v1/stream_pb2.py | 0 {scribe => hub}/schema/types/v2/__init__.py | 0 {scribe => hub}/schema/types/v2/claim_pb2.py | 0 {scribe => hub}/schema/types/v2/hub_pb2.py | 0 .../schema/types/v2/hub_pb2_grpc.py | 0 .../schema/types/v2/purchase_pb2.py | 0 {scribe => hub}/schema/types/v2/result_pb2.py | 0 .../schema/types/v2/result_pb2_grpc.py | 0 .../schema/types/v2/support_pb2.py | 0 {scribe => hub}/schema/url.py | 0 {scribe => hub}/service.py | 12 ++++---- scribe/blockchain/__init__.py | 1 - scribe/elasticsearch/__init__.py | 2 -- setup.py | 2 +- tests/test_resolve_command.py | 2 +- tests/test_revertable.py | 4 +-- tests/testcase.py | 2 +- 89 files changed, 160 insertions(+), 160 deletions(-) rename {scribe => hub}/__init__.py (100%) create mode 100644 hub/blockchain/__init__.py rename {scribe => hub}/blockchain/__main__.py (80%) rename {scribe => hub}/blockchain/daemon.py (98%) rename {scribe => hub}/blockchain/env.py (99%) rename {scribe => hub}/blockchain/mempool.py (97%) rename {scribe => hub}/blockchain/network.py (96%) rename {scribe => hub}/blockchain/prefetcher.py (97%) rename {scribe => hub}/blockchain/service.py (98%) rename {scribe => hub}/blockchain/transaction/__init__.py (98%) rename {scribe => hub}/blockchain/transaction/deserializer.py (97%) rename {scribe => hub}/blockchain/transaction/script.py (97%) rename {scribe => hub}/build_info.py (100%) rename {scribe => hub}/common.py (100%) rename {scribe => hub}/db/__init__.py (100%) rename {scribe => hub}/db/common.py (99%) rename {scribe => hub}/db/db.py (98%) rename {scribe => hub}/db/interface.py (98%) rename {scribe => hub}/db/merkle.py (99%) rename {scribe => hub}/db/migrators/__init__.py (100%) rename {scribe => hub}/db/migrators/migrate7to8.py (97%) rename {scribe => hub}/db/migrators/migrate8to9.py (100%) rename {scribe => hub}/db/prefixes.py (99%) rename {scribe => hub}/db/revertable.py (99%) create mode 100644 hub/elasticsearch/__init__.py rename {scribe => hub}/elasticsearch/__main__.py (81%) rename {scribe => hub}/elasticsearch/constants.py (100%) rename {scribe => hub}/elasticsearch/env.py (99%) rename {scribe => hub}/elasticsearch/fast_ar_trending.py (100%) rename {scribe => hub}/elasticsearch/notifier_protocol.py (100%) rename {scribe => hub}/elasticsearch/search.py (98%) rename {scribe => hub}/elasticsearch/service.py (97%) rename {scribe => hub}/env.py (99%) rename {scribe => hub}/error/Makefile (100%) rename {scribe => hub}/error/README.md (100%) rename {scribe => hub}/error/__init__.py (100%) rename {scribe => hub}/error/base.py (100%) rename {scribe => hub}/error/generate.py (100%) rename {scribe => hub}/hub/__init__.py (100%) rename {scribe => hub}/hub/__main__.py (82%) rename {scribe => hub}/hub/common.py (99%) rename {scribe => hub}/hub/env.py (99%) rename {scribe => hub}/hub/framer.py (100%) rename {scribe => hub}/hub/jsonrpc.py (99%) rename {scribe => hub}/hub/mempool.py (97%) rename {scribe => hub}/hub/service.py (93%) rename {scribe => hub}/hub/session.py (98%) rename {scribe => hub}/hub/udp.py (98%) rename {scribe => hub}/metrics.py (100%) rename {scribe => hub}/schema/Makefile (100%) rename {scribe => hub}/schema/README.md (100%) rename {scribe => hub}/schema/__init__.py (100%) rename {scribe => hub}/schema/attrs.py (97%) rename {scribe => hub}/schema/base.py (100%) rename {scribe => hub}/schema/base58.py (100%) rename {scribe => hub}/schema/bip32.py (99%) rename {scribe => hub}/schema/claim.py (97%) rename {scribe => hub}/schema/compat.py (94%) rename {scribe => hub}/schema/mime_types.py (100%) rename {scribe => hub}/schema/purchase.py (94%) rename {scribe => hub}/schema/result.py (97%) rename {scribe => hub}/schema/support.py (78%) rename {scribe => hub}/schema/tags.py (100%) rename {scribe => hub}/schema/types/__init__.py (100%) rename {scribe => hub}/schema/types/v1/__init__.py (100%) rename {scribe => hub}/schema/types/v1/certificate_pb2.py (100%) rename {scribe => hub}/schema/types/v1/fee_pb2.py (100%) rename {scribe => hub}/schema/types/v1/legacy_claim_pb2.py (100%) rename {scribe => hub}/schema/types/v1/metadata_pb2.py (100%) rename {scribe => hub}/schema/types/v1/signature_pb2.py (100%) rename {scribe => hub}/schema/types/v1/source_pb2.py (100%) rename {scribe => hub}/schema/types/v1/stream_pb2.py (100%) rename {scribe => hub}/schema/types/v2/__init__.py (100%) rename {scribe => hub}/schema/types/v2/claim_pb2.py (100%) rename {scribe => hub}/schema/types/v2/hub_pb2.py (100%) rename {scribe => hub}/schema/types/v2/hub_pb2_grpc.py (100%) rename {scribe => hub}/schema/types/v2/purchase_pb2.py (100%) rename {scribe => hub}/schema/types/v2/result_pb2.py (100%) rename {scribe => hub}/schema/types/v2/result_pb2_grpc.py (100%) rename {scribe => hub}/schema/types/v2/support_pb2.py (100%) rename {scribe => hub}/schema/url.py (100%) rename {scribe => hub}/service.py (97%) delete mode 100644 scribe/blockchain/__init__.py delete mode 100644 scribe/elasticsearch/__init__.py diff --git a/docker/set_build.py b/docker/set_build.py index 4cefcce..af89f93 100644 --- a/docker/set_build.py +++ b/docker/set_build.py @@ -2,7 +2,7 @@ import sys import os import re import logging -import scribe.build_info as build_info_mod +import hub.build_info as build_info_mod log = logging.getLogger() log.addHandler(logging.StreamHandler()) diff --git a/scribe/__init__.py b/hub/__init__.py similarity index 100% rename from scribe/__init__.py rename to hub/__init__.py diff --git a/hub/blockchain/__init__.py b/hub/blockchain/__init__.py new file mode 100644 index 0000000..32c9bea --- /dev/null +++ b/hub/blockchain/__init__.py @@ -0,0 +1 @@ +from hub.blockchain.network import LBCTestNet, LBCRegTest, LBCMainNet diff --git a/scribe/blockchain/__main__.py b/hub/blockchain/__main__.py similarity index 80% rename from scribe/blockchain/__main__.py rename to hub/blockchain/__main__.py index 32f851a..6f04bd6 100644 --- a/scribe/blockchain/__main__.py +++ b/hub/blockchain/__main__.py @@ -2,9 +2,9 @@ import os import logging import traceback import argparse -from scribe.common import setup_logging -from scribe.blockchain.env import BlockchainEnv -from scribe.blockchain.service import BlockchainProcessorService +from hub.common import setup_logging +from hub.blockchain.env import BlockchainEnv +from hub.blockchain.service import BlockchainProcessorService def main(): diff --git a/scribe/blockchain/daemon.py b/hub/blockchain/daemon.py similarity index 98% rename from scribe/blockchain/daemon.py rename to hub/blockchain/daemon.py index 745eaa3..b5af795 100644 --- a/scribe/blockchain/daemon.py +++ b/hub/blockchain/daemon.py @@ -7,8 +7,8 @@ from functools import wraps import aiohttp from prometheus_client import Gauge, Histogram -from scribe import PROMETHEUS_NAMESPACE -from scribe.common import LRUCacheWithMetrics, RPCError, DaemonError, WarmingUpError, WorkQueueFullError +from hub import PROMETHEUS_NAMESPACE +from hub.common import LRUCacheWithMetrics, RPCError, DaemonError, WarmingUpError, WorkQueueFullError log = logging.getLogger(__name__) diff --git a/scribe/blockchain/env.py b/hub/blockchain/env.py similarity index 99% rename from scribe/blockchain/env.py rename to hub/blockchain/env.py index ea5c560..e96669a 100644 --- a/scribe/blockchain/env.py +++ b/hub/blockchain/env.py @@ -1,4 +1,4 @@ -from scribe.env import Env +from hub.env import Env class BlockchainEnv(Env): diff --git a/scribe/blockchain/mempool.py b/hub/blockchain/mempool.py similarity index 97% rename from scribe/blockchain/mempool.py rename to hub/blockchain/mempool.py index 848f929..f516702 100644 --- a/scribe/blockchain/mempool.py +++ b/hub/blockchain/mempool.py @@ -2,10 +2,10 @@ import itertools import attr import typing from collections import defaultdict -from scribe.blockchain.transaction.deserializer import Deserializer +from hub.blockchain.transaction.deserializer import Deserializer if typing.TYPE_CHECKING: - from scribe.db import HubDB + from hub.db import HubDB @attr.s(slots=True) diff --git a/scribe/blockchain/network.py b/hub/blockchain/network.py similarity index 96% rename from scribe/blockchain/network.py rename to hub/blockchain/network.py index eb63ecf..fc794db 100644 --- a/scribe/blockchain/network.py +++ b/hub/blockchain/network.py @@ -4,12 +4,12 @@ import typing from typing import List from hashlib import sha256 from decimal import Decimal -from scribe.schema.base58 import Base58 -from scribe.schema.bip32 import PublicKey -from scribe.common import hash160, hash_to_hex_str, double_sha256 -from scribe.blockchain.transaction import TxOutput, TxInput, Block -from scribe.blockchain.transaction.deserializer import Deserializer -from scribe.blockchain.transaction.script import OpCodes, P2PKH_script, P2SH_script, txo_script_parser +from hub.schema.base58 import Base58 +from hub.schema.bip32 import PublicKey +from hub.common import hash160, hash_to_hex_str, double_sha256 +from hub.blockchain.transaction import TxOutput, TxInput, Block +from hub.blockchain.transaction.deserializer import Deserializer +from hub.blockchain.transaction.script import OpCodes, P2PKH_script, P2SH_script, txo_script_parser HASHX_LEN = 11 diff --git a/scribe/blockchain/prefetcher.py b/hub/blockchain/prefetcher.py similarity index 97% rename from scribe/blockchain/prefetcher.py rename to hub/blockchain/prefetcher.py index ab4eaff..ec569a0 100644 --- a/scribe/blockchain/prefetcher.py +++ b/hub/blockchain/prefetcher.py @@ -2,8 +2,8 @@ import asyncio import logging import typing if typing.TYPE_CHECKING: - from scribe.blockchain.network import LBCMainNet - from scribe.blockchain.daemon import LBCDaemon + from hub.blockchain.network import LBCMainNet + from hub.blockchain.daemon import LBCDaemon def chunks(items, size): diff --git a/scribe/blockchain/service.py b/hub/blockchain/service.py similarity index 98% rename from scribe/blockchain/service.py rename to hub/blockchain/service.py index 078f06b..3a75442 100644 --- a/scribe/blockchain/service.py +++ b/hub/blockchain/service.py @@ -7,20 +7,20 @@ from typing import Optional, List, Tuple, Set, DefaultDict, Dict from prometheus_client import Gauge, Histogram from collections import defaultdict -from scribe import PROMETHEUS_NAMESPACE -from scribe.db.prefixes import ACTIVATED_SUPPORT_TXO_TYPE, ACTIVATED_CLAIM_TXO_TYPE -from scribe.db.prefixes import PendingActivationKey, PendingActivationValue, ClaimToTXOValue -from scribe.error.base import ChainError -from scribe.common import hash_to_hex_str, hash160, RPCError, HISTOGRAM_BUCKETS, StagedClaimtrieItem, sha256, LRUCache -from scribe.blockchain.daemon import LBCDaemon -from scribe.blockchain.transaction import Tx, TxOutput, TxInput, Block -from scribe.blockchain.prefetcher import Prefetcher -from scribe.blockchain.mempool import MemPool -from scribe.schema.url import normalize_name -from scribe.service import BlockchainService +from hub import PROMETHEUS_NAMESPACE +from hub.db.prefixes import ACTIVATED_SUPPORT_TXO_TYPE, ACTIVATED_CLAIM_TXO_TYPE +from hub.db.prefixes import PendingActivationKey, PendingActivationValue, ClaimToTXOValue +from hub.error.base import ChainError +from hub.common import hash_to_hex_str, hash160, RPCError, HISTOGRAM_BUCKETS, StagedClaimtrieItem, sha256, LRUCache +from hub.blockchain.daemon import LBCDaemon +from hub.blockchain.transaction import Tx, TxOutput, TxInput, Block +from hub.blockchain.prefetcher import Prefetcher +from hub.blockchain.mempool import MemPool +from hub.schema.url import normalize_name +from hub.service import BlockchainService if typing.TYPE_CHECKING: - from scribe.blockchain.env import BlockchainEnv - from scribe.db.revertable import RevertableOpStack + from hub.blockchain.env import BlockchainEnv + from hub.db.revertable import RevertableOpStack NAMESPACE = f"{PROMETHEUS_NAMESPACE}_writer" @@ -1725,9 +1725,9 @@ class BlockchainProcessorService(BlockchainService): def _iter_start_tasks(self): while self.db.db_version < max(self.db.DB_VERSIONS): if self.db.db_version == 7: - from scribe.db.migrators.migrate7to8 import migrate, FROM_VERSION, TO_VERSION + from hub.db.migrators.migrate7to8 import migrate, FROM_VERSION, TO_VERSION elif self.db.db_version == 8: - from scribe.db.migrators.migrate8to9 import migrate, FROM_VERSION, TO_VERSION + from hub.db.migrators.migrate8to9 import migrate, FROM_VERSION, TO_VERSION self.db._index_address_status = self.env.index_address_status else: raise RuntimeError("unknown db version") diff --git a/scribe/blockchain/transaction/__init__.py b/hub/blockchain/transaction/__init__.py similarity index 98% rename from scribe/blockchain/transaction/__init__.py rename to hub/blockchain/transaction/__init__.py index 88d5ce9..3d894fd 100644 --- a/scribe/blockchain/transaction/__init__.py +++ b/hub/blockchain/transaction/__init__.py @@ -3,8 +3,8 @@ import functools import typing from dataclasses import dataclass from struct import Struct -from scribe.schema.claim import Claim -from scribe.common import double_sha256 +from hub.schema.claim import Claim +from hub.common import double_sha256 if (sys.version_info.major, sys.version_info.minor) > (3, 7): cachedproperty = functools.cached_property diff --git a/scribe/blockchain/transaction/deserializer.py b/hub/blockchain/transaction/deserializer.py similarity index 97% rename from scribe/blockchain/transaction/deserializer.py rename to hub/blockchain/transaction/deserializer.py index c9e027b..073f743 100644 --- a/scribe/blockchain/transaction/deserializer.py +++ b/hub/blockchain/transaction/deserializer.py @@ -1,9 +1,9 @@ -from scribe.common import double_sha256 -from scribe.blockchain.transaction import ( +from hub.common import double_sha256 +from hub.blockchain.transaction import ( unpack_le_int32_from, unpack_le_int64_from, unpack_le_uint16_from, unpack_le_uint32_from, unpack_le_uint64_from, Tx, TxInput, TxOutput ) -from scribe.blockchain.transaction.script import txo_script_parser +from hub.blockchain.transaction.script import txo_script_parser class Deserializer: diff --git a/scribe/blockchain/transaction/script.py b/hub/blockchain/transaction/script.py similarity index 97% rename from scribe/blockchain/transaction/script.py rename to hub/blockchain/transaction/script.py index 4be83be..3edd72e 100644 --- a/scribe/blockchain/transaction/script.py +++ b/hub/blockchain/transaction/script.py @@ -1,6 +1,6 @@ import typing -from scribe.blockchain.transaction import NameClaim, ClaimUpdate, ClaimSupport -from scribe.blockchain.transaction import unpack_le_uint16_from, unpack_le_uint32_from, pack_le_uint16, pack_le_uint32 +from hub.blockchain.transaction import NameClaim, ClaimUpdate, ClaimSupport +from hub.blockchain.transaction import unpack_le_uint16_from, unpack_le_uint32_from, pack_le_uint16, pack_le_uint32 class _OpCodes(typing.NamedTuple): diff --git a/scribe/build_info.py b/hub/build_info.py similarity index 100% rename from scribe/build_info.py rename to hub/build_info.py diff --git a/scribe/common.py b/hub/common.py similarity index 100% rename from scribe/common.py rename to hub/common.py diff --git a/scribe/db/__init__.py b/hub/db/__init__.py similarity index 100% rename from scribe/db/__init__.py rename to hub/db/__init__.py diff --git a/scribe/db/common.py b/hub/db/common.py similarity index 99% rename from scribe/db/common.py rename to hub/db/common.py index 23f9164..5c6c438 100644 --- a/scribe/db/common.py +++ b/hub/db/common.py @@ -1,7 +1,7 @@ import typing import enum from typing import Optional -from scribe.error import ResolveCensoredError +from hub.error import ResolveCensoredError @enum.unique diff --git a/scribe/db/db.py b/hub/db/db.py similarity index 98% rename from scribe/db/db.py rename to hub/db/db.py index bcb3cc5..ded2e70 100644 --- a/scribe/db/db.py +++ b/hub/db/db.py @@ -12,19 +12,19 @@ from functools import partial from bisect import bisect_right from collections import defaultdict from concurrent.futures.thread import ThreadPoolExecutor -from scribe import PROMETHEUS_NAMESPACE -from scribe.error import ResolveCensoredError -from scribe.schema.url import URL, normalize_name -from scribe.schema.claim import guess_stream_type -from scribe.schema.result import Censor -from scribe.blockchain.transaction import TxInput -from scribe.common import hash_to_hex_str, hash160, LRUCacheWithMetrics, sha256 -from scribe.db.merkle import Merkle, MerkleCache, FastMerkleCacheItem -from scribe.db.common import ResolveResult, STREAM_TYPES, CLAIM_TYPES, ExpandedResolveResult, DBError, UTXO -from scribe.db.prefixes import PendingActivationValue, ClaimTakeoverValue, ClaimToTXOValue, PrefixDB -from scribe.db.prefixes import ACTIVATED_CLAIM_TXO_TYPE, ACTIVATED_SUPPORT_TXO_TYPE, EffectiveAmountKey -from scribe.db.prefixes import PendingActivationKey, TXOToClaimValue, DBStatePrefixRow, MempoolTXPrefixRow -from scribe.db.prefixes import HashXMempoolStatusPrefixRow +from hub import PROMETHEUS_NAMESPACE +from hub.error import ResolveCensoredError +from hub.schema.url import URL, normalize_name +from hub.schema.claim import guess_stream_type +from hub.schema.result import Censor +from hub.blockchain.transaction import TxInput +from hub.common import hash_to_hex_str, hash160, LRUCacheWithMetrics, sha256 +from hub.db.merkle import Merkle, MerkleCache, FastMerkleCacheItem +from hub.db.common import ResolveResult, STREAM_TYPES, CLAIM_TYPES, ExpandedResolveResult, DBError, UTXO +from hub.db.prefixes import PendingActivationValue, ClaimTakeoverValue, ClaimToTXOValue, PrefixDB +from hub.db.prefixes import ACTIVATED_CLAIM_TXO_TYPE, ACTIVATED_SUPPORT_TXO_TYPE, EffectiveAmountKey +from hub.db.prefixes import PendingActivationKey, TXOToClaimValue, DBStatePrefixRow, MempoolTXPrefixRow +from hub.db.prefixes import HashXMempoolStatusPrefixRow TXO_STRUCT = struct.Struct(b'>LH') diff --git a/scribe/db/interface.py b/hub/db/interface.py similarity index 98% rename from scribe/db/interface.py rename to hub/db/interface.py index 5045705..f3b7094 100644 --- a/scribe/db/interface.py +++ b/hub/db/interface.py @@ -2,8 +2,8 @@ import struct import typing import rocksdb from typing import Optional -from scribe.db.common import DB_PREFIXES, COLUMN_SETTINGS -from scribe.db.revertable import RevertableOpStack, RevertablePut, RevertableDelete +from hub.db.common import DB_PREFIXES, COLUMN_SETTINGS +from hub.db.revertable import RevertableOpStack, RevertablePut, RevertableDelete ROW_TYPES = {} diff --git a/scribe/db/merkle.py b/hub/db/merkle.py similarity index 99% rename from scribe/db/merkle.py rename to hub/db/merkle.py index 3321fa1..2541bf6 100644 --- a/scribe/db/merkle.py +++ b/hub/db/merkle.py @@ -29,7 +29,7 @@ import typing from asyncio import Event from math import ceil, log -from scribe.common import double_sha256 +from hub.common import double_sha256 class Merkle: diff --git a/scribe/db/migrators/__init__.py b/hub/db/migrators/__init__.py similarity index 100% rename from scribe/db/migrators/__init__.py rename to hub/db/migrators/__init__.py diff --git a/scribe/db/migrators/migrate7to8.py b/hub/db/migrators/migrate7to8.py similarity index 97% rename from scribe/db/migrators/migrate7to8.py rename to hub/db/migrators/migrate7to8.py index 58e1627..cfeaae5 100644 --- a/scribe/db/migrators/migrate7to8.py +++ b/hub/db/migrators/migrate7to8.py @@ -3,9 +3,9 @@ import time import array import typing from bisect import bisect_right -from scribe.common import sha256 +from hub.common import sha256 if typing.TYPE_CHECKING: - from scribe.db.db import HubDB + from hub.db.db import HubDB FROM_VERSION = 7 TO_VERSION = 8 diff --git a/scribe/db/migrators/migrate8to9.py b/hub/db/migrators/migrate8to9.py similarity index 100% rename from scribe/db/migrators/migrate8to9.py rename to hub/db/migrators/migrate8to9.py diff --git a/scribe/db/prefixes.py b/hub/db/prefixes.py similarity index 99% rename from scribe/db/prefixes.py rename to hub/db/prefixes.py index 4cb214b..3918652 100644 --- a/scribe/db/prefixes.py +++ b/hub/db/prefixes.py @@ -3,9 +3,9 @@ import struct import array import base64 from typing import Union, Tuple, NamedTuple, Optional -from scribe.db.common import DB_PREFIXES -from scribe.db.interface import BasePrefixDB, ROW_TYPES, PrefixRow -from scribe.schema.url import normalize_name +from hub.db.common import DB_PREFIXES +from hub.db.interface import BasePrefixDB, ROW_TYPES, PrefixRow +from hub.schema.url import normalize_name ACTIVATED_CLAIM_TXO_TYPE = 1 ACTIVATED_SUPPORT_TXO_TYPE = 2 diff --git a/scribe/db/revertable.py b/hub/db/revertable.py similarity index 99% rename from scribe/db/revertable.py rename to hub/db/revertable.py index a982a97..37fbc4e 100644 --- a/scribe/db/revertable.py +++ b/hub/db/revertable.py @@ -3,7 +3,7 @@ import logging from string import printable from collections import defaultdict from typing import Tuple, Iterable, Callable, Optional, List -from scribe.db.common import DB_PREFIXES +from hub.db.common import DB_PREFIXES _OP_STRUCT = struct.Struct('>BLL') log = logging.getLogger(__name__) @@ -58,7 +58,7 @@ class RevertableOp: return str(self) def __str__(self) -> str: - from scribe.db.prefixes import auto_decode_item + from hub.db.prefixes import auto_decode_item k, v = auto_decode_item(self.key, self.value) key = ''.join(c if c in printable else '.' for c in str(k)) val = ''.join(c if c in printable else '.' for c in str(v)) diff --git a/hub/elasticsearch/__init__.py b/hub/elasticsearch/__init__.py new file mode 100644 index 0000000..6138c4c --- /dev/null +++ b/hub/elasticsearch/__init__.py @@ -0,0 +1,2 @@ +from hub.elasticsearch.search import SearchIndex +from hub.elasticsearch.notifier_protocol import ElasticNotifierClientProtocol \ No newline at end of file diff --git a/scribe/elasticsearch/__main__.py b/hub/elasticsearch/__main__.py similarity index 81% rename from scribe/elasticsearch/__main__.py rename to hub/elasticsearch/__main__.py index f968a8c..71e9215 100644 --- a/scribe/elasticsearch/__main__.py +++ b/hub/elasticsearch/__main__.py @@ -2,9 +2,9 @@ import os import logging import traceback import argparse -from scribe.common import setup_logging -from scribe.elasticsearch.env import ElasticEnv -from scribe.elasticsearch.service import ElasticSyncService +from hub.common import setup_logging +from hub.elasticsearch.env import ElasticEnv +from hub.elasticsearch.service import ElasticSyncService def main(): diff --git a/scribe/elasticsearch/constants.py b/hub/elasticsearch/constants.py similarity index 100% rename from scribe/elasticsearch/constants.py rename to hub/elasticsearch/constants.py diff --git a/scribe/elasticsearch/env.py b/hub/elasticsearch/env.py similarity index 99% rename from scribe/elasticsearch/env.py rename to hub/elasticsearch/env.py index 58fcfad..b4a82cc 100644 --- a/scribe/elasticsearch/env.py +++ b/hub/elasticsearch/env.py @@ -1,4 +1,4 @@ -from scribe.env import Env +from hub.env import Env class ElasticEnv(Env): diff --git a/scribe/elasticsearch/fast_ar_trending.py b/hub/elasticsearch/fast_ar_trending.py similarity index 100% rename from scribe/elasticsearch/fast_ar_trending.py rename to hub/elasticsearch/fast_ar_trending.py diff --git a/scribe/elasticsearch/notifier_protocol.py b/hub/elasticsearch/notifier_protocol.py similarity index 100% rename from scribe/elasticsearch/notifier_protocol.py rename to hub/elasticsearch/notifier_protocol.py diff --git a/scribe/elasticsearch/search.py b/hub/elasticsearch/search.py similarity index 98% rename from scribe/elasticsearch/search.py rename to hub/elasticsearch/search.py index 7b77801..6bd68c8 100644 --- a/scribe/elasticsearch/search.py +++ b/hub/elasticsearch/search.py @@ -8,16 +8,16 @@ from operator import itemgetter from typing import Optional, List, Iterable, TYPE_CHECKING from elasticsearch import AsyncElasticsearch, NotFoundError, ConnectionError -from scribe.schema.result import Censor, Outputs -from scribe.schema.tags import clean_tags -from scribe.schema.url import normalize_name -from scribe.error import TooManyClaimSearchParametersError -from scribe.common import LRUCache -from scribe.db.common import CLAIM_TYPES, STREAM_TYPES -from scribe.elasticsearch.constants import INDEX_DEFAULT_SETTINGS, REPLACEMENTS, FIELDS, TEXT_FIELDS, RANGE_FIELDS -from scribe.db.common import ResolveResult +from hub.schema.result import Censor, Outputs +from hub.schema.tags import clean_tags +from hub.schema.url import normalize_name +from hub.error import TooManyClaimSearchParametersError +from hub.common import LRUCache +from hub.db.common import CLAIM_TYPES, STREAM_TYPES +from hub.elasticsearch.constants import INDEX_DEFAULT_SETTINGS, REPLACEMENTS, FIELDS, TEXT_FIELDS, RANGE_FIELDS +from hub.db.common import ResolveResult if TYPE_CHECKING: - from scribe.db import HubDB + from hub.db import HubDB class ChannelResolution(str): diff --git a/scribe/elasticsearch/service.py b/hub/elasticsearch/service.py similarity index 97% rename from scribe/elasticsearch/service.py rename to hub/elasticsearch/service.py index 9476d5b..8acaa2b 100644 --- a/scribe/elasticsearch/service.py +++ b/hub/elasticsearch/service.py @@ -5,16 +5,16 @@ import asyncio from collections import defaultdict from elasticsearch import AsyncElasticsearch, NotFoundError from elasticsearch.helpers import async_streaming_bulk -from scribe.schema.result import Censor -from scribe.service import BlockchainReaderService -from scribe.db.revertable import RevertableOp -from scribe.db.common import TrendingNotification, DB_PREFIXES -from scribe.elasticsearch.notifier_protocol import ElasticNotifierProtocol -from scribe.elasticsearch.search import IndexVersionMismatch, expand_query -from scribe.elasticsearch.constants import ALL_FIELDS, INDEX_DEFAULT_SETTINGS -from scribe.elasticsearch.fast_ar_trending import FAST_AR_TRENDING_SCRIPT +from hub.schema.result import Censor +from hub.service import BlockchainReaderService +from hub.db.revertable import RevertableOp +from hub.db.common import TrendingNotification, DB_PREFIXES +from hub.elasticsearch.notifier_protocol import ElasticNotifierProtocol +from hub.elasticsearch.search import IndexVersionMismatch, expand_query +from hub.elasticsearch.constants import ALL_FIELDS, INDEX_DEFAULT_SETTINGS +from hub.elasticsearch.fast_ar_trending import FAST_AR_TRENDING_SCRIPT if typing.TYPE_CHECKING: - from scribe.elasticsearch.env import ElasticEnv + from hub.elasticsearch.env import ElasticEnv class ElasticSyncService(BlockchainReaderService): diff --git a/scribe/env.py b/hub/env.py similarity index 99% rename from scribe/env.py rename to hub/env.py index 6ca1146..70ddb40 100644 --- a/scribe/env.py +++ b/hub/env.py @@ -3,7 +3,7 @@ import re import resource import logging from collections import namedtuple -from scribe.blockchain.network import LBCMainNet, LBCTestNet, LBCRegTest +from hub.blockchain.network import LBCMainNet, LBCTestNet, LBCRegTest NetIdentity = namedtuple('NetIdentity', 'host tcp_port ssl_port nick_suffix') diff --git a/scribe/error/Makefile b/hub/error/Makefile similarity index 100% rename from scribe/error/Makefile rename to hub/error/Makefile diff --git a/scribe/error/README.md b/hub/error/README.md similarity index 100% rename from scribe/error/README.md rename to hub/error/README.md diff --git a/scribe/error/__init__.py b/hub/error/__init__.py similarity index 100% rename from scribe/error/__init__.py rename to hub/error/__init__.py diff --git a/scribe/error/base.py b/hub/error/base.py similarity index 100% rename from scribe/error/base.py rename to hub/error/base.py diff --git a/scribe/error/generate.py b/hub/error/generate.py similarity index 100% rename from scribe/error/generate.py rename to hub/error/generate.py diff --git a/scribe/hub/__init__.py b/hub/hub/__init__.py similarity index 100% rename from scribe/hub/__init__.py rename to hub/hub/__init__.py diff --git a/scribe/hub/__main__.py b/hub/hub/__main__.py similarity index 82% rename from scribe/hub/__main__.py rename to hub/hub/__main__.py index 0862e87..f144431 100644 --- a/scribe/hub/__main__.py +++ b/hub/hub/__main__.py @@ -2,9 +2,9 @@ import os import logging import traceback import argparse -from scribe.common import setup_logging -from scribe.hub.env import ServerEnv -from scribe.hub.service import HubServerService +from hub.common import setup_logging +from hub.hub.env import ServerEnv +from hub.hub.service import HubServerService def main(): diff --git a/scribe/hub/common.py b/hub/hub/common.py similarity index 99% rename from scribe/hub/common.py rename to hub/hub/common.py index 7eb2c11..c1b00f9 100644 --- a/scribe/hub/common.py +++ b/hub/hub/common.py @@ -1,7 +1,7 @@ import inspect from collections import namedtuple from functools import lru_cache -from scribe.common import CodeMessageError +from hub.common import CodeMessageError SignatureInfo = namedtuple('SignatureInfo', 'min_args max_args ' diff --git a/scribe/hub/env.py b/hub/hub/env.py similarity index 99% rename from scribe/hub/env.py rename to hub/hub/env.py index 85ada99..da8f997 100644 --- a/scribe/hub/env.py +++ b/hub/hub/env.py @@ -1,5 +1,5 @@ import re -from scribe.env import Env +from hub.env import Env class ServerEnv(Env): diff --git a/scribe/hub/framer.py b/hub/hub/framer.py similarity index 100% rename from scribe/hub/framer.py rename to hub/hub/framer.py diff --git a/scribe/hub/jsonrpc.py b/hub/hub/jsonrpc.py similarity index 99% rename from scribe/hub/jsonrpc.py rename to hub/hub/jsonrpc.py index aae8c4e..d464e35 100644 --- a/scribe/hub/jsonrpc.py +++ b/hub/hub/jsonrpc.py @@ -6,8 +6,8 @@ import asyncio from asyncio import Event from functools import partial from numbers import Number -from scribe.common import RPCError, CodeMessageError -from scribe.hub.common import Notification, Request, Response, Batch, ProtocolError +from hub.common import RPCError, CodeMessageError +from hub.hub.common import Notification, Request, Response, Batch, ProtocolError class JSONRPC: diff --git a/scribe/hub/mempool.py b/hub/hub/mempool.py similarity index 97% rename from scribe/hub/mempool.py rename to hub/hub/mempool.py index a2db139..1359644 100644 --- a/scribe/hub/mempool.py +++ b/hub/hub/mempool.py @@ -6,14 +6,14 @@ import logging from collections import defaultdict from prometheus_client import Histogram, Gauge import rocksdb.errors -from scribe import PROMETHEUS_NAMESPACE -from scribe.common import HISTOGRAM_BUCKETS -from scribe.db.common import UTXO -from scribe.blockchain.transaction.deserializer import Deserializer +from hub import PROMETHEUS_NAMESPACE +from hub.common import HISTOGRAM_BUCKETS +from hub.db.common import UTXO +from hub.blockchain.transaction.deserializer import Deserializer if typing.TYPE_CHECKING: - from scribe.hub.session import SessionManager - from scribe.db import HubDB + from hub.hub.session import SessionManager + from hub.db import HubDB @attr.s(slots=True) diff --git a/scribe/hub/service.py b/hub/hub/service.py similarity index 93% rename from scribe/hub/service.py rename to hub/hub/service.py index ccc4d33..0dfad90 100644 --- a/scribe/hub/service.py +++ b/hub/hub/service.py @@ -1,14 +1,14 @@ import time import typing import asyncio -from scribe.blockchain.daemon import LBCDaemon -from scribe.hub.session import SessionManager -from scribe.hub.mempool import HubMemPool -from scribe.hub.udp import StatusServer -from scribe.service import BlockchainReaderService -from scribe.elasticsearch import ElasticNotifierClientProtocol +from hub.blockchain.daemon import LBCDaemon +from hub.hub.session import SessionManager +from hub.hub.mempool import HubMemPool +from hub.hub.udp import StatusServer +from hub.service import BlockchainReaderService +from hub.elasticsearch import ElasticNotifierClientProtocol if typing.TYPE_CHECKING: - from scribe.hub.env import ServerEnv + from hub.hub.env import ServerEnv class HubServerService(BlockchainReaderService): diff --git a/scribe/hub/session.py b/hub/hub/session.py similarity index 98% rename from scribe/hub/session.py rename to hub/hub/session.py index 3ea3cff..f2e54ce 100644 --- a/scribe/hub/session.py +++ b/hub/hub/session.py @@ -15,22 +15,22 @@ from contextlib import suppress from functools import partial from elasticsearch import ConnectionTimeout from prometheus_client import Counter, Info, Histogram, Gauge -from scribe.schema.result import Outputs -from scribe.error import ResolveCensoredError, TooManyClaimSearchParametersError -from scribe import __version__, PROMETHEUS_NAMESPACE -from scribe.hub import PROTOCOL_MIN, PROTOCOL_MAX, HUB_PROTOCOL_VERSION -from scribe.build_info import BUILD, COMMIT_HASH, DOCKER_TAG -from scribe.elasticsearch import SearchIndex -from scribe.common import sha256, hash_to_hex_str, hex_str_to_hash, HASHX_LEN, version_string, formatted_time -from scribe.common import protocol_version, RPCError, DaemonError, TaskGroup, HISTOGRAM_BUCKETS -from scribe.hub.jsonrpc import JSONRPCAutoDetect, JSONRPCConnection, JSONRPCv2, JSONRPC -from scribe.hub.common import BatchRequest, ProtocolError, Request, Batch, Notification -from scribe.hub.framer import NewlineFramer +from hub.schema.result import Outputs +from hub.error import ResolveCensoredError, TooManyClaimSearchParametersError +from hub import __version__, PROMETHEUS_NAMESPACE +from hub.hub import PROTOCOL_MIN, PROTOCOL_MAX, HUB_PROTOCOL_VERSION +from hub.build_info import BUILD, COMMIT_HASH, DOCKER_TAG +from hub.elasticsearch import SearchIndex +from hub.common import sha256, hash_to_hex_str, hex_str_to_hash, HASHX_LEN, version_string, formatted_time +from hub.common import protocol_version, RPCError, DaemonError, TaskGroup, HISTOGRAM_BUCKETS +from hub.hub.jsonrpc import JSONRPCAutoDetect, JSONRPCConnection, JSONRPCv2, JSONRPC +from hub.hub.common import BatchRequest, ProtocolError, Request, Batch, Notification +from hub.hub.framer import NewlineFramer if typing.TYPE_CHECKING: - from scribe.db import HubDB - from scribe.hub.env import ServerEnv - from scribe.blockchain.daemon import LBCDaemon - from scribe.hub.mempool import HubMemPool + from hub.db import HubDB + from hub.hub.env import ServerEnv + from hub.blockchain.daemon import LBCDaemon + from hub.hub.mempool import HubMemPool BAD_REQUEST = 1 DAEMON_ERROR = 2 diff --git a/scribe/hub/udp.py b/hub/hub/udp.py similarity index 98% rename from scribe/hub/udp.py rename to hub/hub/udp.py index f919324..af83d0c 100644 --- a/scribe/hub/udp.py +++ b/hub/hub/udp.py @@ -3,8 +3,8 @@ import struct from time import perf_counter import logging from typing import Optional, Tuple, NamedTuple -from scribe.schema.attrs import country_str_to_int, country_int_to_str -from scribe.common import LRUCache, is_valid_public_ipv4 +from hub.schema.attrs import country_str_to_int, country_int_to_str +from hub.common import LRUCache, is_valid_public_ipv4 log = logging.getLogger(__name__) diff --git a/scribe/metrics.py b/hub/metrics.py similarity index 100% rename from scribe/metrics.py rename to hub/metrics.py diff --git a/scribe/schema/Makefile b/hub/schema/Makefile similarity index 100% rename from scribe/schema/Makefile rename to hub/schema/Makefile diff --git a/scribe/schema/README.md b/hub/schema/README.md similarity index 100% rename from scribe/schema/README.md rename to hub/schema/README.md diff --git a/scribe/schema/__init__.py b/hub/schema/__init__.py similarity index 100% rename from scribe/schema/__init__.py rename to hub/schema/__init__.py diff --git a/scribe/schema/attrs.py b/hub/schema/attrs.py similarity index 97% rename from scribe/schema/attrs.py rename to hub/schema/attrs.py index a855f42..bd2751f 100644 --- a/scribe/schema/attrs.py +++ b/hub/schema/attrs.py @@ -7,13 +7,13 @@ from string import ascii_letters from decimal import Decimal, ROUND_UP from google.protobuf.json_format import MessageToDict -from scribe.schema.base58 import Base58, b58_encode -from scribe.error import MissingPublishedFileError, EmptyPublishedFileError +from hub.schema.base58 import Base58, b58_encode +from hub.error import MissingPublishedFileError, EmptyPublishedFileError -from scribe.schema.mime_types import guess_media_type -from scribe.schema.base import Metadata, BaseMessageList -from scribe.schema.tags import normalize_tag -from scribe.schema.types.v2.claim_pb2 import ( +from hub.schema.mime_types import guess_media_type +from hub.schema.base import Metadata, BaseMessageList +from hub.schema.tags import normalize_tag +from hub.schema.types.v2.claim_pb2 import ( Fee as FeeMessage, Location as LocationMessage, Language as LanguageMessage diff --git a/scribe/schema/base.py b/hub/schema/base.py similarity index 100% rename from scribe/schema/base.py rename to hub/schema/base.py diff --git a/scribe/schema/base58.py b/hub/schema/base58.py similarity index 100% rename from scribe/schema/base58.py rename to hub/schema/base58.py diff --git a/scribe/schema/bip32.py b/hub/schema/bip32.py similarity index 99% rename from scribe/schema/bip32.py rename to hub/schema/bip32.py index d12ff1b..f98824b 100644 --- a/scribe/schema/bip32.py +++ b/hub/schema/bip32.py @@ -8,7 +8,7 @@ from coincurve.utils import ( pem_to_der, lib as libsecp256k1, ffi as libsecp256k1_ffi ) from coincurve.ecdsa import CDATA_SIG_LENGTH -from scribe.schema.base58 import Base58 +from hub.schema.base58 import Base58 if (sys.version_info.major, sys.version_info.minor) > (3, 7): diff --git a/scribe/schema/claim.py b/hub/schema/claim.py similarity index 97% rename from scribe/schema/claim.py rename to hub/schema/claim.py index a4a46d8..f59a55f 100644 --- a/scribe/schema/claim.py +++ b/hub/schema/claim.py @@ -11,15 +11,15 @@ from hachoir.core.log import log as hachoir_log from hachoir.parser import createParser as binary_file_parser from hachoir.metadata import extractMetadata as binary_file_metadata -from scribe.schema import compat -from scribe.schema.base import Signable -from scribe.schema.mime_types import guess_media_type, guess_stream_type -from scribe.schema.attrs import ( +from hub.schema import compat +from hub.schema.base import Signable +from hub.schema.mime_types import guess_media_type, guess_stream_type +from hub.schema.attrs import ( Source, Playable, Dimmensional, Fee, Image, Video, Audio, LanguageList, LocationList, ClaimList, ClaimReference, TagList ) -from scribe.schema.types.v2.claim_pb2 import Claim as ClaimMessage -from scribe.error import InputValueIsNoneError +from hub.schema.types.v2.claim_pb2 import Claim as ClaimMessage +from hub.error import InputValueIsNoneError hachoir_log.use_print = False diff --git a/scribe/schema/compat.py b/hub/schema/compat.py similarity index 94% rename from scribe/schema/compat.py rename to hub/schema/compat.py index 2dc99b0..46dd72f 100644 --- a/scribe/schema/compat.py +++ b/hub/schema/compat.py @@ -3,9 +3,9 @@ from decimal import Decimal from google.protobuf.message import DecodeError -from scribe.schema.types.v1.legacy_claim_pb2 import Claim as OldClaimMessage -from scribe.schema.types.v1.certificate_pb2 import KeyType -from scribe.schema.types.v1.fee_pb2 import Fee as FeeMessage +from hub.schema.types.v1.legacy_claim_pb2 import Claim as OldClaimMessage +from hub.schema.types.v1.certificate_pb2 import KeyType +from hub.schema.types.v1.fee_pb2 import Fee as FeeMessage def from_old_json_schema(claim, payload: bytes): diff --git a/scribe/schema/mime_types.py b/hub/schema/mime_types.py similarity index 100% rename from scribe/schema/mime_types.py rename to hub/schema/mime_types.py diff --git a/scribe/schema/purchase.py b/hub/schema/purchase.py similarity index 94% rename from scribe/schema/purchase.py rename to hub/schema/purchase.py index 22148e6..cbb9329 100644 --- a/scribe/schema/purchase.py +++ b/hub/schema/purchase.py @@ -1,6 +1,6 @@ from google.protobuf.message import DecodeError from google.protobuf.json_format import MessageToDict -from scribe.schema.types.v2.purchase_pb2 import Purchase as PurchaseMessage +from hub.schema.types.v2.purchase_pb2 import Purchase as PurchaseMessage from .attrs import ClaimReference diff --git a/scribe/schema/result.py b/hub/schema/result.py similarity index 97% rename from scribe/schema/result.py rename to hub/schema/result.py index 2429e93..6acfd3e 100644 --- a/scribe/schema/result.py +++ b/hub/schema/result.py @@ -2,11 +2,11 @@ import base64 from typing import List, TYPE_CHECKING, Union, Optional, Dict, Set, Tuple from itertools import chain -from scribe.error import ResolveCensoredError -from scribe.schema.types.v2.result_pb2 import Outputs as OutputsMessage -from scribe.schema.types.v2.result_pb2 import Error as ErrorMessage +from hub.error import ResolveCensoredError +from hub.schema.types.v2.result_pb2 import Outputs as OutputsMessage +from hub.schema.types.v2.result_pb2 import Error as ErrorMessage if TYPE_CHECKING: - from scribe.db.common import ResolveResult + from hub.db.common import ResolveResult INVALID = ErrorMessage.Code.Name(ErrorMessage.INVALID) NOT_FOUND = ErrorMessage.Code.Name(ErrorMessage.NOT_FOUND) BLOCKED = ErrorMessage.Code.Name(ErrorMessage.BLOCKED) diff --git a/scribe/schema/support.py b/hub/schema/support.py similarity index 78% rename from scribe/schema/support.py rename to hub/schema/support.py index 35f60f6..ccd2cc7 100644 --- a/scribe/schema/support.py +++ b/hub/schema/support.py @@ -1,5 +1,5 @@ -from scribe.schema.base import Signable -from scribe.schema.types.v2.support_pb2 import Support as SupportMessage +from hub.schema.base import Signable +from hub.schema.types.v2.support_pb2 import Support as SupportMessage class Support(Signable): diff --git a/scribe/schema/tags.py b/hub/schema/tags.py similarity index 100% rename from scribe/schema/tags.py rename to hub/schema/tags.py diff --git a/scribe/schema/types/__init__.py b/hub/schema/types/__init__.py similarity index 100% rename from scribe/schema/types/__init__.py rename to hub/schema/types/__init__.py diff --git a/scribe/schema/types/v1/__init__.py b/hub/schema/types/v1/__init__.py similarity index 100% rename from scribe/schema/types/v1/__init__.py rename to hub/schema/types/v1/__init__.py diff --git a/scribe/schema/types/v1/certificate_pb2.py b/hub/schema/types/v1/certificate_pb2.py similarity index 100% rename from scribe/schema/types/v1/certificate_pb2.py rename to hub/schema/types/v1/certificate_pb2.py diff --git a/scribe/schema/types/v1/fee_pb2.py b/hub/schema/types/v1/fee_pb2.py similarity index 100% rename from scribe/schema/types/v1/fee_pb2.py rename to hub/schema/types/v1/fee_pb2.py diff --git a/scribe/schema/types/v1/legacy_claim_pb2.py b/hub/schema/types/v1/legacy_claim_pb2.py similarity index 100% rename from scribe/schema/types/v1/legacy_claim_pb2.py rename to hub/schema/types/v1/legacy_claim_pb2.py diff --git a/scribe/schema/types/v1/metadata_pb2.py b/hub/schema/types/v1/metadata_pb2.py similarity index 100% rename from scribe/schema/types/v1/metadata_pb2.py rename to hub/schema/types/v1/metadata_pb2.py diff --git a/scribe/schema/types/v1/signature_pb2.py b/hub/schema/types/v1/signature_pb2.py similarity index 100% rename from scribe/schema/types/v1/signature_pb2.py rename to hub/schema/types/v1/signature_pb2.py diff --git a/scribe/schema/types/v1/source_pb2.py b/hub/schema/types/v1/source_pb2.py similarity index 100% rename from scribe/schema/types/v1/source_pb2.py rename to hub/schema/types/v1/source_pb2.py diff --git a/scribe/schema/types/v1/stream_pb2.py b/hub/schema/types/v1/stream_pb2.py similarity index 100% rename from scribe/schema/types/v1/stream_pb2.py rename to hub/schema/types/v1/stream_pb2.py diff --git a/scribe/schema/types/v2/__init__.py b/hub/schema/types/v2/__init__.py similarity index 100% rename from scribe/schema/types/v2/__init__.py rename to hub/schema/types/v2/__init__.py diff --git a/scribe/schema/types/v2/claim_pb2.py b/hub/schema/types/v2/claim_pb2.py similarity index 100% rename from scribe/schema/types/v2/claim_pb2.py rename to hub/schema/types/v2/claim_pb2.py diff --git a/scribe/schema/types/v2/hub_pb2.py b/hub/schema/types/v2/hub_pb2.py similarity index 100% rename from scribe/schema/types/v2/hub_pb2.py rename to hub/schema/types/v2/hub_pb2.py diff --git a/scribe/schema/types/v2/hub_pb2_grpc.py b/hub/schema/types/v2/hub_pb2_grpc.py similarity index 100% rename from scribe/schema/types/v2/hub_pb2_grpc.py rename to hub/schema/types/v2/hub_pb2_grpc.py diff --git a/scribe/schema/types/v2/purchase_pb2.py b/hub/schema/types/v2/purchase_pb2.py similarity index 100% rename from scribe/schema/types/v2/purchase_pb2.py rename to hub/schema/types/v2/purchase_pb2.py diff --git a/scribe/schema/types/v2/result_pb2.py b/hub/schema/types/v2/result_pb2.py similarity index 100% rename from scribe/schema/types/v2/result_pb2.py rename to hub/schema/types/v2/result_pb2.py diff --git a/scribe/schema/types/v2/result_pb2_grpc.py b/hub/schema/types/v2/result_pb2_grpc.py similarity index 100% rename from scribe/schema/types/v2/result_pb2_grpc.py rename to hub/schema/types/v2/result_pb2_grpc.py diff --git a/scribe/schema/types/v2/support_pb2.py b/hub/schema/types/v2/support_pb2.py similarity index 100% rename from scribe/schema/types/v2/support_pb2.py rename to hub/schema/types/v2/support_pb2.py diff --git a/scribe/schema/url.py b/hub/schema/url.py similarity index 100% rename from scribe/schema/url.py rename to hub/schema/url.py diff --git a/scribe/service.py b/hub/service.py similarity index 97% rename from scribe/service.py rename to hub/service.py index d2227fd..52ddaf1 100644 --- a/scribe/service.py +++ b/hub/service.py @@ -5,12 +5,12 @@ import signal from concurrent.futures.thread import ThreadPoolExecutor from prometheus_client import Gauge, Histogram -from scribe import __version__, PROMETHEUS_NAMESPACE -from scribe.env import Env -from scribe.db import HubDB -from scribe.db.prefixes import DBState -from scribe.common import HISTOGRAM_BUCKETS -from scribe.metrics import PrometheusServer +from hub import __version__, PROMETHEUS_NAMESPACE +from hub.env import Env +from hub.db import HubDB +from hub.db.prefixes import DBState +from hub.common import HISTOGRAM_BUCKETS +from hub.metrics import PrometheusServer class BlockchainService: diff --git a/scribe/blockchain/__init__.py b/scribe/blockchain/__init__.py deleted file mode 100644 index 8e0dfca..0000000 --- a/scribe/blockchain/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from scribe.blockchain.network import LBCTestNet, LBCRegTest, LBCMainNet diff --git a/scribe/elasticsearch/__init__.py b/scribe/elasticsearch/__init__.py deleted file mode 100644 index 68591c9..0000000 --- a/scribe/elasticsearch/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from scribe.elasticsearch.search import SearchIndex -from scribe.elasticsearch.notifier_protocol import ElasticNotifierClientProtocol \ No newline at end of file diff --git a/setup.py b/setup.py index c546d40..f065699 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ import os -from scribe import __name__, __version__ +from hub import __name__, __version__ from setuptools import setup, find_packages BASE = os.path.dirname(__file__) diff --git a/tests/test_resolve_command.py b/tests/test_resolve_command.py index 6a266df..8d99fb9 100644 --- a/tests/test_resolve_command.py +++ b/tests/test_resolve_command.py @@ -8,7 +8,7 @@ from collections import defaultdict from typing import NamedTuple, List from lbry.testcase import CommandTestCase from lbry.wallet.transaction import Transaction, Output -from scribe.schema.compat import OldClaimMessage +from hub.schema.compat import OldClaimMessage from lbry.crypto.hash import sha256 from lbry.crypto.base58 import Base58 diff --git a/tests/test_revertable.py b/tests/test_revertable.py index 3d566f0..d6b0dce 100644 --- a/tests/test_revertable.py +++ b/tests/test_revertable.py @@ -1,8 +1,8 @@ import unittest import tempfile import shutil -from scribe.db.revertable import RevertableOpStack, RevertableDelete, RevertablePut, OpStackIntegrity -from scribe.db.prefixes import ClaimToTXOPrefixRow, PrefixDB +from hub.db.revertable import RevertableOpStack, RevertableDelete, RevertablePut, OpStackIntegrity +from hub.db.prefixes import ClaimToTXOPrefixRow, PrefixDB class TestRevertableOpStack(unittest.TestCase): diff --git a/tests/testcase.py b/tests/testcase.py index f9d3865..e31dceb 100644 --- a/tests/testcase.py +++ b/tests/testcase.py @@ -20,7 +20,7 @@ from lbry.wallet.util import satoshis_to_coins from lbry.wallet.dewies import lbc_to_dewies from lbry.wallet.orchstr8 import Conductor from lbry.wallet.orchstr8.node import LBCWalletNode, WalletNode, HubNode -from scribe.schema.claim import Claim +from hub.schema.claim import Claim from lbry.extras.daemon.daemon import Daemon, jsonrpc_dumps_pretty from lbry.extras.daemon.components import Component, WalletComponent -- 2.45.2 From 0da235bceb6ba6899c452bf8a5a5c5314f97cf06 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 18 May 2022 10:55:12 -0400 Subject: [PATCH 03/13] rename scribe.blockchain -> hub.scribe --- hub/blockchain/__init__.py | 1 - hub/db/db.py | 2 +- hub/env.py | 2 +- hub/hub/mempool.py | 2 +- hub/hub/service.py | 2 +- hub/hub/session.py | 2 +- hub/scribe/__init__.py | 1 + hub/{blockchain => scribe}/__main__.py | 4 ++-- hub/{blockchain => scribe}/daemon.py | 0 hub/{blockchain => scribe}/env.py | 0 hub/{blockchain => scribe}/mempool.py | 2 +- hub/{blockchain => scribe}/network.py | 6 +++--- hub/{blockchain => scribe}/prefetcher.py | 4 ++-- hub/{blockchain => scribe}/service.py | 10 +++++----- hub/{blockchain => scribe}/transaction/__init__.py | 0 hub/{blockchain => scribe}/transaction/deserializer.py | 4 ++-- hub/{blockchain => scribe}/transaction/script.py | 4 ++-- 17 files changed, 23 insertions(+), 23 deletions(-) delete mode 100644 hub/blockchain/__init__.py create mode 100644 hub/scribe/__init__.py rename hub/{blockchain => scribe}/__main__.py (86%) rename hub/{blockchain => scribe}/daemon.py (100%) rename hub/{blockchain => scribe}/env.py (100%) rename hub/{blockchain => scribe}/mempool.py (98%) rename hub/{blockchain => scribe}/network.py (97%) rename hub/{blockchain => scribe}/prefetcher.py (98%) rename hub/{blockchain => scribe}/service.py (99%) rename hub/{blockchain => scribe}/transaction/__init__.py (100%) rename hub/{blockchain => scribe}/transaction/deserializer.py (98%) rename hub/{blockchain => scribe}/transaction/script.py (98%) diff --git a/hub/blockchain/__init__.py b/hub/blockchain/__init__.py deleted file mode 100644 index 32c9bea..0000000 --- a/hub/blockchain/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from hub.blockchain.network import LBCTestNet, LBCRegTest, LBCMainNet diff --git a/hub/db/db.py b/hub/db/db.py index ded2e70..239e591 100644 --- a/hub/db/db.py +++ b/hub/db/db.py @@ -17,7 +17,7 @@ from hub.error import ResolveCensoredError from hub.schema.url import URL, normalize_name from hub.schema.claim import guess_stream_type from hub.schema.result import Censor -from hub.blockchain.transaction import TxInput +from hub.scribe.transaction import TxInput from hub.common import hash_to_hex_str, hash160, LRUCacheWithMetrics, sha256 from hub.db.merkle import Merkle, MerkleCache, FastMerkleCacheItem from hub.db.common import ResolveResult, STREAM_TYPES, CLAIM_TYPES, ExpandedResolveResult, DBError, UTXO diff --git a/hub/env.py b/hub/env.py index 70ddb40..ce18748 100644 --- a/hub/env.py +++ b/hub/env.py @@ -3,7 +3,7 @@ import re import resource import logging from collections import namedtuple -from hub.blockchain.network import LBCMainNet, LBCTestNet, LBCRegTest +from hub.scribe.network import LBCMainNet, LBCTestNet, LBCRegTest NetIdentity = namedtuple('NetIdentity', 'host tcp_port ssl_port nick_suffix') diff --git a/hub/hub/mempool.py b/hub/hub/mempool.py index 1359644..1e57a57 100644 --- a/hub/hub/mempool.py +++ b/hub/hub/mempool.py @@ -9,7 +9,7 @@ import rocksdb.errors from hub import PROMETHEUS_NAMESPACE from hub.common import HISTOGRAM_BUCKETS from hub.db.common import UTXO -from hub.blockchain.transaction.deserializer import Deserializer +from hub.scribe.transaction.deserializer import Deserializer if typing.TYPE_CHECKING: from hub.hub.session import SessionManager diff --git a/hub/hub/service.py b/hub/hub/service.py index 0dfad90..6981ca5 100644 --- a/hub/hub/service.py +++ b/hub/hub/service.py @@ -1,7 +1,7 @@ import time import typing import asyncio -from hub.blockchain.daemon import LBCDaemon +from hub.scribe.daemon import LBCDaemon from hub.hub.session import SessionManager from hub.hub.mempool import HubMemPool from hub.hub.udp import StatusServer diff --git a/hub/hub/session.py b/hub/hub/session.py index f2e54ce..f9c6341 100644 --- a/hub/hub/session.py +++ b/hub/hub/session.py @@ -29,7 +29,7 @@ from hub.hub.framer import NewlineFramer if typing.TYPE_CHECKING: from hub.db import HubDB from hub.hub.env import ServerEnv - from hub.blockchain.daemon import LBCDaemon + from hub.scribe.daemon import LBCDaemon from hub.hub.mempool import HubMemPool BAD_REQUEST = 1 diff --git a/hub/scribe/__init__.py b/hub/scribe/__init__.py new file mode 100644 index 0000000..2ee7962 --- /dev/null +++ b/hub/scribe/__init__.py @@ -0,0 +1 @@ +from hub.scribe.network import LBCTestNet, LBCRegTest, LBCMainNet diff --git a/hub/blockchain/__main__.py b/hub/scribe/__main__.py similarity index 86% rename from hub/blockchain/__main__.py rename to hub/scribe/__main__.py index 6f04bd6..1184b2b 100644 --- a/hub/blockchain/__main__.py +++ b/hub/scribe/__main__.py @@ -3,8 +3,8 @@ import logging import traceback import argparse from hub.common import setup_logging -from hub.blockchain.env import BlockchainEnv -from hub.blockchain.service import BlockchainProcessorService +from hub.scribe.env import BlockchainEnv +from hub.scribe.service import BlockchainProcessorService def main(): diff --git a/hub/blockchain/daemon.py b/hub/scribe/daemon.py similarity index 100% rename from hub/blockchain/daemon.py rename to hub/scribe/daemon.py diff --git a/hub/blockchain/env.py b/hub/scribe/env.py similarity index 100% rename from hub/blockchain/env.py rename to hub/scribe/env.py diff --git a/hub/blockchain/mempool.py b/hub/scribe/mempool.py similarity index 98% rename from hub/blockchain/mempool.py rename to hub/scribe/mempool.py index f516702..9bf6221 100644 --- a/hub/blockchain/mempool.py +++ b/hub/scribe/mempool.py @@ -2,7 +2,7 @@ import itertools import attr import typing from collections import defaultdict -from hub.blockchain.transaction.deserializer import Deserializer +from hub.scribe.transaction.deserializer import Deserializer if typing.TYPE_CHECKING: from hub.db import HubDB diff --git a/hub/blockchain/network.py b/hub/scribe/network.py similarity index 97% rename from hub/blockchain/network.py rename to hub/scribe/network.py index fc794db..851b02b 100644 --- a/hub/blockchain/network.py +++ b/hub/scribe/network.py @@ -7,9 +7,9 @@ from decimal import Decimal from hub.schema.base58 import Base58 from hub.schema.bip32 import PublicKey from hub.common import hash160, hash_to_hex_str, double_sha256 -from hub.blockchain.transaction import TxOutput, TxInput, Block -from hub.blockchain.transaction.deserializer import Deserializer -from hub.blockchain.transaction.script import OpCodes, P2PKH_script, P2SH_script, txo_script_parser +from hub.scribe.transaction import TxOutput, TxInput, Block +from hub.scribe.transaction.deserializer import Deserializer +from hub.scribe.transaction.script import OpCodes, P2PKH_script, P2SH_script, txo_script_parser HASHX_LEN = 11 diff --git a/hub/blockchain/prefetcher.py b/hub/scribe/prefetcher.py similarity index 98% rename from hub/blockchain/prefetcher.py rename to hub/scribe/prefetcher.py index ec569a0..612d1ab 100644 --- a/hub/blockchain/prefetcher.py +++ b/hub/scribe/prefetcher.py @@ -2,8 +2,8 @@ import asyncio import logging import typing if typing.TYPE_CHECKING: - from hub.blockchain.network import LBCMainNet - from hub.blockchain.daemon import LBCDaemon + from hub.scribe.network import LBCMainNet + from hub.scribe.daemon import LBCDaemon def chunks(items, size): diff --git a/hub/blockchain/service.py b/hub/scribe/service.py similarity index 99% rename from hub/blockchain/service.py rename to hub/scribe/service.py index 3a75442..b22451c 100644 --- a/hub/blockchain/service.py +++ b/hub/scribe/service.py @@ -12,14 +12,14 @@ from hub.db.prefixes import ACTIVATED_SUPPORT_TXO_TYPE, ACTIVATED_CLAIM_TXO_TYPE from hub.db.prefixes import PendingActivationKey, PendingActivationValue, ClaimToTXOValue from hub.error.base import ChainError from hub.common import hash_to_hex_str, hash160, RPCError, HISTOGRAM_BUCKETS, StagedClaimtrieItem, sha256, LRUCache -from hub.blockchain.daemon import LBCDaemon -from hub.blockchain.transaction import Tx, TxOutput, TxInput, Block -from hub.blockchain.prefetcher import Prefetcher -from hub.blockchain.mempool import MemPool +from hub.scribe.daemon import LBCDaemon +from hub.scribe.transaction import Tx, TxOutput, TxInput, Block +from hub.scribe.prefetcher import Prefetcher +from hub.scribe.mempool import MemPool from hub.schema.url import normalize_name from hub.service import BlockchainService if typing.TYPE_CHECKING: - from hub.blockchain.env import BlockchainEnv + from hub.scribe.env import BlockchainEnv from hub.db.revertable import RevertableOpStack diff --git a/hub/blockchain/transaction/__init__.py b/hub/scribe/transaction/__init__.py similarity index 100% rename from hub/blockchain/transaction/__init__.py rename to hub/scribe/transaction/__init__.py diff --git a/hub/blockchain/transaction/deserializer.py b/hub/scribe/transaction/deserializer.py similarity index 98% rename from hub/blockchain/transaction/deserializer.py rename to hub/scribe/transaction/deserializer.py index 073f743..7f540cb 100644 --- a/hub/blockchain/transaction/deserializer.py +++ b/hub/scribe/transaction/deserializer.py @@ -1,9 +1,9 @@ from hub.common import double_sha256 -from hub.blockchain.transaction import ( +from hub.scribe.transaction import ( unpack_le_int32_from, unpack_le_int64_from, unpack_le_uint16_from, unpack_le_uint32_from, unpack_le_uint64_from, Tx, TxInput, TxOutput ) -from hub.blockchain.transaction.script import txo_script_parser +from hub.scribe.transaction.script import txo_script_parser class Deserializer: diff --git a/hub/blockchain/transaction/script.py b/hub/scribe/transaction/script.py similarity index 98% rename from hub/blockchain/transaction/script.py rename to hub/scribe/transaction/script.py index 3edd72e..df4a2c4 100644 --- a/hub/blockchain/transaction/script.py +++ b/hub/scribe/transaction/script.py @@ -1,6 +1,6 @@ import typing -from hub.blockchain.transaction import NameClaim, ClaimUpdate, ClaimSupport -from hub.blockchain.transaction import unpack_le_uint16_from, unpack_le_uint32_from, pack_le_uint16, pack_le_uint32 +from hub.scribe.transaction import NameClaim, ClaimUpdate, ClaimSupport +from hub.scribe.transaction import unpack_le_uint16_from, unpack_le_uint32_from, pack_le_uint16, pack_le_uint32 class _OpCodes(typing.NamedTuple): -- 2.45.2 From 060e42370757f05107ae80c3127c9d34281fae4b Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 18 May 2022 10:56:09 -0400 Subject: [PATCH 04/13] rename scribe.hub -> hub.herald --- hub/{hub => herald}/__init__.py | 0 hub/{hub => herald}/__main__.py | 4 ++-- hub/{hub => herald}/common.py | 0 hub/{hub => herald}/env.py | 0 hub/{hub => herald}/framer.py | 0 hub/{hub => herald}/jsonrpc.py | 2 +- hub/{hub => herald}/mempool.py | 2 +- hub/{hub => herald}/service.py | 8 ++++---- hub/{hub => herald}/session.py | 12 ++++++------ hub/{hub => herald}/udp.py | 0 10 files changed, 14 insertions(+), 14 deletions(-) rename hub/{hub => herald}/__init__.py (100%) rename hub/{hub => herald}/__main__.py (88%) rename hub/{hub => herald}/common.py (100%) rename hub/{hub => herald}/env.py (100%) rename hub/{hub => herald}/framer.py (100%) rename hub/{hub => herald}/jsonrpc.py (99%) rename hub/{hub => herald}/mempool.py (99%) rename hub/{hub => herald}/service.py (96%) rename hub/{hub => herald}/session.py (99%) rename hub/{hub => herald}/udp.py (100%) diff --git a/hub/hub/__init__.py b/hub/herald/__init__.py similarity index 100% rename from hub/hub/__init__.py rename to hub/herald/__init__.py diff --git a/hub/hub/__main__.py b/hub/herald/__main__.py similarity index 88% rename from hub/hub/__main__.py rename to hub/herald/__main__.py index f144431..835820b 100644 --- a/hub/hub/__main__.py +++ b/hub/herald/__main__.py @@ -3,8 +3,8 @@ import logging import traceback import argparse from hub.common import setup_logging -from hub.hub.env import ServerEnv -from hub.hub.service import HubServerService +from hub.herald.env import ServerEnv +from hub.herald.service import HubServerService def main(): diff --git a/hub/hub/common.py b/hub/herald/common.py similarity index 100% rename from hub/hub/common.py rename to hub/herald/common.py diff --git a/hub/hub/env.py b/hub/herald/env.py similarity index 100% rename from hub/hub/env.py rename to hub/herald/env.py diff --git a/hub/hub/framer.py b/hub/herald/framer.py similarity index 100% rename from hub/hub/framer.py rename to hub/herald/framer.py diff --git a/hub/hub/jsonrpc.py b/hub/herald/jsonrpc.py similarity index 99% rename from hub/hub/jsonrpc.py rename to hub/herald/jsonrpc.py index d464e35..c940f40 100644 --- a/hub/hub/jsonrpc.py +++ b/hub/herald/jsonrpc.py @@ -7,7 +7,7 @@ from asyncio import Event from functools import partial from numbers import Number from hub.common import RPCError, CodeMessageError -from hub.hub.common import Notification, Request, Response, Batch, ProtocolError +from hub.herald.common import Notification, Request, Response, Batch, ProtocolError class JSONRPC: diff --git a/hub/hub/mempool.py b/hub/herald/mempool.py similarity index 99% rename from hub/hub/mempool.py rename to hub/herald/mempool.py index 1e57a57..0ec1e0d 100644 --- a/hub/hub/mempool.py +++ b/hub/herald/mempool.py @@ -12,7 +12,7 @@ from hub.db.common import UTXO from hub.scribe.transaction.deserializer import Deserializer if typing.TYPE_CHECKING: - from hub.hub.session import SessionManager + from hub.herald.session import SessionManager from hub.db import HubDB diff --git a/hub/hub/service.py b/hub/herald/service.py similarity index 96% rename from hub/hub/service.py rename to hub/herald/service.py index 6981ca5..c2fa69c 100644 --- a/hub/hub/service.py +++ b/hub/herald/service.py @@ -2,13 +2,13 @@ import time import typing import asyncio from hub.scribe.daemon import LBCDaemon -from hub.hub.session import SessionManager -from hub.hub.mempool import HubMemPool -from hub.hub.udp import StatusServer +from hub.herald.session import SessionManager +from hub.herald.mempool import HubMemPool +from hub.herald.udp import StatusServer from hub.service import BlockchainReaderService from hub.elasticsearch import ElasticNotifierClientProtocol if typing.TYPE_CHECKING: - from hub.hub.env import ServerEnv + from hub.herald.env import ServerEnv class HubServerService(BlockchainReaderService): diff --git a/hub/hub/session.py b/hub/herald/session.py similarity index 99% rename from hub/hub/session.py rename to hub/herald/session.py index f9c6341..06c4bab 100644 --- a/hub/hub/session.py +++ b/hub/herald/session.py @@ -18,19 +18,19 @@ from prometheus_client import Counter, Info, Histogram, Gauge from hub.schema.result import Outputs from hub.error import ResolveCensoredError, TooManyClaimSearchParametersError from hub import __version__, PROMETHEUS_NAMESPACE -from hub.hub import PROTOCOL_MIN, PROTOCOL_MAX, HUB_PROTOCOL_VERSION +from hub.herald import PROTOCOL_MIN, PROTOCOL_MAX, HUB_PROTOCOL_VERSION from hub.build_info import BUILD, COMMIT_HASH, DOCKER_TAG from hub.elasticsearch import SearchIndex from hub.common import sha256, hash_to_hex_str, hex_str_to_hash, HASHX_LEN, version_string, formatted_time from hub.common import protocol_version, RPCError, DaemonError, TaskGroup, HISTOGRAM_BUCKETS -from hub.hub.jsonrpc import JSONRPCAutoDetect, JSONRPCConnection, JSONRPCv2, JSONRPC -from hub.hub.common import BatchRequest, ProtocolError, Request, Batch, Notification -from hub.hub.framer import NewlineFramer +from hub.herald.jsonrpc import JSONRPCAutoDetect, JSONRPCConnection, JSONRPCv2, JSONRPC +from hub.herald.common import BatchRequest, ProtocolError, Request, Batch, Notification +from hub.herald.framer import NewlineFramer if typing.TYPE_CHECKING: from hub.db import HubDB - from hub.hub.env import ServerEnv + from hub.herald.env import ServerEnv from hub.scribe.daemon import LBCDaemon - from hub.hub.mempool import HubMemPool + from hub.herald.mempool import HubMemPool BAD_REQUEST = 1 DAEMON_ERROR = 2 diff --git a/hub/hub/udp.py b/hub/herald/udp.py similarity index 100% rename from hub/hub/udp.py rename to hub/herald/udp.py -- 2.45.2 From 301a0ca66d63732449f6586787c9318a2dfec822 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 18 May 2022 10:57:08 -0400 Subject: [PATCH 05/13] rename scribe.elasticsearch -> hub.elastic_sync --- hub/elastic_sync/__init__.py | 2 ++ hub/{elasticsearch => elastic_sync}/__main__.py | 4 ++-- hub/{elasticsearch => elastic_sync}/constants.py | 0 hub/{elasticsearch => elastic_sync}/env.py | 0 .../fast_ar_trending.py | 0 .../notifier_protocol.py | 0 hub/{elasticsearch => elastic_sync}/search.py | 2 +- hub/{elasticsearch => elastic_sync}/service.py | 10 +++++----- hub/elasticsearch/__init__.py | 2 -- hub/herald/service.py | 2 +- hub/herald/session.py | 2 +- 11 files changed, 12 insertions(+), 12 deletions(-) create mode 100644 hub/elastic_sync/__init__.py rename hub/{elasticsearch => elastic_sync}/__main__.py (87%) rename hub/{elasticsearch => elastic_sync}/constants.py (100%) rename hub/{elasticsearch => elastic_sync}/env.py (100%) rename hub/{elasticsearch => elastic_sync}/fast_ar_trending.py (100%) rename hub/{elasticsearch => elastic_sync}/notifier_protocol.py (100%) rename hub/{elasticsearch => elastic_sync}/search.py (99%) rename hub/{elasticsearch => elastic_sync}/service.py (98%) delete mode 100644 hub/elasticsearch/__init__.py diff --git a/hub/elastic_sync/__init__.py b/hub/elastic_sync/__init__.py new file mode 100644 index 0000000..ca4352e --- /dev/null +++ b/hub/elastic_sync/__init__.py @@ -0,0 +1,2 @@ +from hub.elastic_sync.search import SearchIndex +from hub.elastic_sync.notifier_protocol import ElasticNotifierClientProtocol \ No newline at end of file diff --git a/hub/elasticsearch/__main__.py b/hub/elastic_sync/__main__.py similarity index 87% rename from hub/elasticsearch/__main__.py rename to hub/elastic_sync/__main__.py index 71e9215..f8dfefc 100644 --- a/hub/elasticsearch/__main__.py +++ b/hub/elastic_sync/__main__.py @@ -3,8 +3,8 @@ import logging import traceback import argparse from hub.common import setup_logging -from hub.elasticsearch.env import ElasticEnv -from hub.elasticsearch.service import ElasticSyncService +from hub.elastic_sync.env import ElasticEnv +from hub.elastic_sync.service import ElasticSyncService def main(): diff --git a/hub/elasticsearch/constants.py b/hub/elastic_sync/constants.py similarity index 100% rename from hub/elasticsearch/constants.py rename to hub/elastic_sync/constants.py diff --git a/hub/elasticsearch/env.py b/hub/elastic_sync/env.py similarity index 100% rename from hub/elasticsearch/env.py rename to hub/elastic_sync/env.py diff --git a/hub/elasticsearch/fast_ar_trending.py b/hub/elastic_sync/fast_ar_trending.py similarity index 100% rename from hub/elasticsearch/fast_ar_trending.py rename to hub/elastic_sync/fast_ar_trending.py diff --git a/hub/elasticsearch/notifier_protocol.py b/hub/elastic_sync/notifier_protocol.py similarity index 100% rename from hub/elasticsearch/notifier_protocol.py rename to hub/elastic_sync/notifier_protocol.py diff --git a/hub/elasticsearch/search.py b/hub/elastic_sync/search.py similarity index 99% rename from hub/elasticsearch/search.py rename to hub/elastic_sync/search.py index 6bd68c8..93ace26 100644 --- a/hub/elasticsearch/search.py +++ b/hub/elastic_sync/search.py @@ -14,7 +14,7 @@ from hub.schema.url import normalize_name from hub.error import TooManyClaimSearchParametersError from hub.common import LRUCache from hub.db.common import CLAIM_TYPES, STREAM_TYPES -from hub.elasticsearch.constants import INDEX_DEFAULT_SETTINGS, REPLACEMENTS, FIELDS, TEXT_FIELDS, RANGE_FIELDS +from hub.elastic_sync.constants import INDEX_DEFAULT_SETTINGS, REPLACEMENTS, FIELDS, TEXT_FIELDS, RANGE_FIELDS from hub.db.common import ResolveResult if TYPE_CHECKING: from hub.db import HubDB diff --git a/hub/elasticsearch/service.py b/hub/elastic_sync/service.py similarity index 98% rename from hub/elasticsearch/service.py rename to hub/elastic_sync/service.py index 8acaa2b..ade8db3 100644 --- a/hub/elasticsearch/service.py +++ b/hub/elastic_sync/service.py @@ -9,12 +9,12 @@ from hub.schema.result import Censor from hub.service import BlockchainReaderService from hub.db.revertable import RevertableOp from hub.db.common import TrendingNotification, DB_PREFIXES -from hub.elasticsearch.notifier_protocol import ElasticNotifierProtocol -from hub.elasticsearch.search import IndexVersionMismatch, expand_query -from hub.elasticsearch.constants import ALL_FIELDS, INDEX_DEFAULT_SETTINGS -from hub.elasticsearch.fast_ar_trending import FAST_AR_TRENDING_SCRIPT +from hub.elastic_sync.notifier_protocol import ElasticNotifierProtocol +from hub.elastic_sync.search import IndexVersionMismatch, expand_query +from hub.elastic_sync.constants import ALL_FIELDS, INDEX_DEFAULT_SETTINGS +from hub.elastic_sync.fast_ar_trending import FAST_AR_TRENDING_SCRIPT if typing.TYPE_CHECKING: - from hub.elasticsearch.env import ElasticEnv + from hub.elastic_sync.env import ElasticEnv class ElasticSyncService(BlockchainReaderService): diff --git a/hub/elasticsearch/__init__.py b/hub/elasticsearch/__init__.py deleted file mode 100644 index 6138c4c..0000000 --- a/hub/elasticsearch/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from hub.elasticsearch.search import SearchIndex -from hub.elasticsearch.notifier_protocol import ElasticNotifierClientProtocol \ No newline at end of file diff --git a/hub/herald/service.py b/hub/herald/service.py index c2fa69c..f6015cf 100644 --- a/hub/herald/service.py +++ b/hub/herald/service.py @@ -6,7 +6,7 @@ from hub.herald.session import SessionManager from hub.herald.mempool import HubMemPool from hub.herald.udp import StatusServer from hub.service import BlockchainReaderService -from hub.elasticsearch import ElasticNotifierClientProtocol +from hub.elastic_sync import ElasticNotifierClientProtocol if typing.TYPE_CHECKING: from hub.herald.env import ServerEnv diff --git a/hub/herald/session.py b/hub/herald/session.py index 06c4bab..9e339b7 100644 --- a/hub/herald/session.py +++ b/hub/herald/session.py @@ -20,7 +20,7 @@ from hub.error import ResolveCensoredError, TooManyClaimSearchParametersError from hub import __version__, PROMETHEUS_NAMESPACE from hub.herald import PROTOCOL_MIN, PROTOCOL_MAX, HUB_PROTOCOL_VERSION from hub.build_info import BUILD, COMMIT_HASH, DOCKER_TAG -from hub.elasticsearch import SearchIndex +from hub.elastic_sync import SearchIndex from hub.common import sha256, hash_to_hex_str, hex_str_to_hash, HASHX_LEN, version_string, formatted_time from hub.common import protocol_version, RPCError, DaemonError, TaskGroup, HISTOGRAM_BUCKETS from hub.herald.jsonrpc import JSONRPCAutoDetect, JSONRPCConnection, JSONRPCv2, JSONRPC -- 2.45.2 From 30e295ec28de7ba9af4d4794e50a92b303b107df Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 18 May 2022 12:02:08 -0400 Subject: [PATCH 06/13] move notifier_protocol.py --- hub/common.py | 6 ++++++ hub/elastic_sync/__init__.py | 2 -- hub/elastic_sync/search.py | 8 +------- hub/elastic_sync/service.py | 5 +++-- hub/herald/service.py | 2 +- hub/herald/session.py | 2 +- hub/{elastic_sync => }/notifier_protocol.py | 0 7 files changed, 12 insertions(+), 13 deletions(-) rename hub/{elastic_sync => }/notifier_protocol.py (100%) diff --git a/hub/common.py b/hub/common.py index be4cb4c..32df6dd 100644 --- a/hub/common.py +++ b/hub/common.py @@ -398,3 +398,9 @@ class TaskGroup: task.cancel() self.done.set() self.started.clear() + + +class IndexVersionMismatch(Exception): + def __init__(self, got_version, expected_version): + self.got_version = got_version + self.expected_version = expected_version diff --git a/hub/elastic_sync/__init__.py b/hub/elastic_sync/__init__.py index ca4352e..e69de29 100644 --- a/hub/elastic_sync/__init__.py +++ b/hub/elastic_sync/__init__.py @@ -1,2 +0,0 @@ -from hub.elastic_sync.search import SearchIndex -from hub.elastic_sync.notifier_protocol import ElasticNotifierClientProtocol \ No newline at end of file diff --git a/hub/elastic_sync/search.py b/hub/elastic_sync/search.py index 93ace26..1cbe06d 100644 --- a/hub/elastic_sync/search.py +++ b/hub/elastic_sync/search.py @@ -12,7 +12,7 @@ from hub.schema.result import Censor, Outputs from hub.schema.tags import clean_tags from hub.schema.url import normalize_name from hub.error import TooManyClaimSearchParametersError -from hub.common import LRUCache +from hub.common import LRUCache, IndexVersionMismatch from hub.db.common import CLAIM_TYPES, STREAM_TYPES from hub.elastic_sync.constants import INDEX_DEFAULT_SETTINGS, REPLACEMENTS, FIELDS, TEXT_FIELDS, RANGE_FIELDS from hub.db.common import ResolveResult @@ -32,12 +32,6 @@ class StreamResolution(str): return LookupError(f'Could not find claim at "{url}".') -class IndexVersionMismatch(Exception): - def __init__(self, got_version, expected_version): - self.got_version = got_version - self.expected_version = expected_version - - class SearchIndex: VERSION = 1 diff --git a/hub/elastic_sync/service.py b/hub/elastic_sync/service.py index ade8db3..b4063a8 100644 --- a/hub/elastic_sync/service.py +++ b/hub/elastic_sync/service.py @@ -7,10 +7,11 @@ from elasticsearch import AsyncElasticsearch, NotFoundError from elasticsearch.helpers import async_streaming_bulk from hub.schema.result import Censor from hub.service import BlockchainReaderService +from hub.common import IndexVersionMismatch from hub.db.revertable import RevertableOp from hub.db.common import TrendingNotification, DB_PREFIXES -from hub.elastic_sync.notifier_protocol import ElasticNotifierProtocol -from hub.elastic_sync.search import IndexVersionMismatch, expand_query +from hub.notifier_protocol import ElasticNotifierProtocol +from hub.elastic_sync.search import expand_query from hub.elastic_sync.constants import ALL_FIELDS, INDEX_DEFAULT_SETTINGS from hub.elastic_sync.fast_ar_trending import FAST_AR_TRENDING_SCRIPT if typing.TYPE_CHECKING: diff --git a/hub/herald/service.py b/hub/herald/service.py index f6015cf..64f9415 100644 --- a/hub/herald/service.py +++ b/hub/herald/service.py @@ -6,7 +6,7 @@ from hub.herald.session import SessionManager from hub.herald.mempool import HubMemPool from hub.herald.udp import StatusServer from hub.service import BlockchainReaderService -from hub.elastic_sync import ElasticNotifierClientProtocol +from hub.notifier_protocol import ElasticNotifierClientProtocol if typing.TYPE_CHECKING: from hub.herald.env import ServerEnv diff --git a/hub/herald/session.py b/hub/herald/session.py index 9e339b7..3078a7d 100644 --- a/hub/herald/session.py +++ b/hub/herald/session.py @@ -20,7 +20,7 @@ from hub.error import ResolveCensoredError, TooManyClaimSearchParametersError from hub import __version__, PROMETHEUS_NAMESPACE from hub.herald import PROTOCOL_MIN, PROTOCOL_MAX, HUB_PROTOCOL_VERSION from hub.build_info import BUILD, COMMIT_HASH, DOCKER_TAG -from hub.elastic_sync import SearchIndex +from hub.elastic_sync.search import SearchIndex from hub.common import sha256, hash_to_hex_str, hex_str_to_hash, HASHX_LEN, version_string, formatted_time from hub.common import protocol_version, RPCError, DaemonError, TaskGroup, HISTOGRAM_BUCKETS from hub.herald.jsonrpc import JSONRPCAutoDetect, JSONRPCConnection, JSONRPCv2, JSONRPC diff --git a/hub/elastic_sync/notifier_protocol.py b/hub/notifier_protocol.py similarity index 100% rename from hub/elastic_sync/notifier_protocol.py rename to hub/notifier_protocol.py -- 2.45.2 From 37fc334c4601dea4e165c38d40a2335a49cd2306 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 18 May 2022 13:09:42 -0400 Subject: [PATCH 07/13] update docker and entrypoints --- docker/Dockerfile => Dockerfile | 6 +++--- docker/docker-compose.yml | 4 ++-- docker/hooks/build | 7 ------- docker/hub-compose.yml | 4 ++-- hub/herald/__main__.py | 4 ++-- scripts/deploy_scribe_dev.sh | 2 +- docker/scribe_entrypoint.sh => scripts/entrypoint.sh | 6 +++--- {docker => scripts}/set_build.py | 0 setup.py | 6 +++--- 9 files changed, 16 insertions(+), 23 deletions(-) rename docker/Dockerfile => Dockerfile (90%) delete mode 100644 docker/hooks/build rename docker/scribe_entrypoint.sh => scripts/entrypoint.sh (52%) rename {docker => scripts}/set_build.py (100%) diff --git a/docker/Dockerfile b/Dockerfile similarity index 90% rename from docker/Dockerfile rename to Dockerfile index e72a8bb..f62dbe6 100644 --- a/docker/Dockerfile +++ b/Dockerfile @@ -28,19 +28,19 @@ RUN groupadd -g 999 $user && useradd -m -u 999 -g $user $user RUN mkdir -p $db_dir RUN chown -R $user:$user $db_dir -COPY . $projects_dir +COPY docker $projects_dir RUN chown -R $user:$user $projects_dir USER $user WORKDIR $projects_dir RUN python3.9 -m pip install pip RUN python3.9 -m pip install -e . -RUN python3.9 docker/set_build.py +RUN python3.9 scripts/set_build.py RUN rm ~/.cache -rf # entry point VOLUME $db_dir ENV DB_DIRECTORY=$db_dir -COPY ./docker/scribe_entrypoint.sh /entrypoint.sh +COPY ./scripts/entrypoint.sh /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 3d4f617..6f6d321 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -55,8 +55,8 @@ services: volumes: - "lbry_rocksdb:/database" environment: - - HUB_COMMAND=scribe-hub - command: # for full options, see `scribe-hub --help` + - HUB_COMMAND=herald + command: # for full options, see `herald --help` - "--daemon_url=http://lbry:lbry@127.0.0.1:9245" - "--max_query_workers=4" - "--host=0.0.0.0" diff --git a/docker/hooks/build b/docker/hooks/build deleted file mode 100644 index 371640d..0000000 --- a/docker/hooks/build +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd "$DIR/../.." ## make sure we're in the right place. Docker Hub screws this up sometimes -echo "docker build dir: $(pwd)" - -docker build --build-arg DOCKER_TAG=$DOCKER_TAG --build-arg DOCKER_COMMIT=$SOURCE_COMMIT -f $DOCKERFILE_PATH -t $IMAGE_NAME . diff --git a/docker/hub-compose.yml b/docker/hub-compose.yml index cd61138..9038984 100644 --- a/docker/hub-compose.yml +++ b/docker/hub-compose.yml @@ -34,7 +34,7 @@ services: - "--max_query_workers=2" - "--filtering_channel_ids=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8" - "--blocking_channel_ids=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6" - scribe_hub: + herald: depends_on: - scribe_elastic_sync - scribe @@ -47,7 +47,7 @@ services: volumes: - "lbry_rocksdb:/database" environment: - - HUB_COMMAND=scribe-hub + - HUB_COMMAND=herald command: - "--daemon_url=http://lbry:lbry@127.0.0.1:9245" - "--elastic_host=127.0.0.1" diff --git a/hub/herald/__main__.py b/hub/herald/__main__.py index 835820b..3532469 100644 --- a/hub/herald/__main__.py +++ b/hub/herald/__main__.py @@ -9,13 +9,13 @@ from hub.herald.service import HubServerService def main(): parser = argparse.ArgumentParser( - prog='scribe-hub' + prog='herald' ) ServerEnv.contribute_to_arg_parser(parser) args = parser.parse_args() try: env = ServerEnv.from_arg_parser(args) - setup_logging(os.path.join(env.db_dir, 'scribe-hub.log')) + setup_logging(os.path.join(env.db_dir, 'herald.log')) server = HubServerService(env) server.run() except Exception: diff --git a/scripts/deploy_scribe_dev.sh b/scripts/deploy_scribe_dev.sh index e22c196..c84297e 100755 --- a/scripts/deploy_scribe_dev.sh +++ b/scripts/deploy_scribe_dev.sh @@ -7,7 +7,7 @@ SCRIPTS_DIR=`dirname $0` SCRIBE_DIR=`dirname $SCRIPTS_DIR` # build the image -docker build -f $SCRIBE_DIR/docker/Dockerfile -t lbry/scribe:development $SCRIBE_DIR +docker build -f $SCRIBE_DIR/Dockerfile -t lbry/scribe:development $SCRIBE_DIR IMAGE=`docker image inspect lbry/scribe:development | sed -n "s/^.*Id\":\s*\"sha256:\s*\(\S*\)\".*$/\1/p"` # push the image to the server diff --git a/docker/scribe_entrypoint.sh b/scripts/entrypoint.sh similarity index 52% rename from docker/scribe_entrypoint.sh rename to scripts/entrypoint.sh index 2f44c48..e78adec 100755 --- a/docker/scribe_entrypoint.sh +++ b/scripts/entrypoint.sh @@ -5,13 +5,13 @@ set -euo pipefail if [ -z "$HUB_COMMAND" ]; then - echo "HUB_COMMAND env variable must be scribe, scribe-hub, or scribe-elastic-sync" + echo "HUB_COMMAND env variable must be scribe, herald, or scribe-elastic-sync" exit 1 fi case "$HUB_COMMAND" in scribe ) exec /home/lbry/.local/bin/scribe "$@" ;; - scribe-hub ) exec /home/lbry/.local/bin/scribe-hub "$@" ;; + scribe-hub ) exec /home/lbry/.local/bin/herald "$@" ;; scribe-elastic-sync ) exec /home/lbry/.local/bin/scribe-elastic-sync "$@" ;; - * ) "HUB_COMMAND env variable must be scribe, scribe-hub, or scribe-elastic-sync" && exit 1 ;; + * ) "HUB_COMMAND env variable must be scribe, herald, or scribe-elastic-sync" && exit 1 ;; esac diff --git a/docker/set_build.py b/scripts/set_build.py similarity index 100% rename from docker/set_build.py rename to scripts/set_build.py diff --git a/setup.py b/setup.py index f065699..41ea264 100644 --- a/setup.py +++ b/setup.py @@ -23,9 +23,9 @@ setup( zip_safe=False, entry_points={ 'console_scripts': [ - 'scribe=scribe.blockchain.__main__:main', - 'scribe-hub=scribe.hub.__main__:main', - 'scribe-elastic-sync=scribe.elasticsearch.__main__:main', + 'scribe=hub.scribe.__main__:main', + 'herald=hub.herald.__main__:main', + 'scribe-elastic-sync=hub.elastic_sync.__main__:main', ], }, install_requires=[ -- 2.45.2 From a919a3a519eb47739cbac5cb956f0755cf6d662b Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 18 May 2022 13:34:36 -0400 Subject: [PATCH 08/13] move search.py to herald --- hub/common.py | 359 +++++++++++++++++++++++++ hub/{elastic_sync => }/constants.py | 0 hub/db/common.py | 16 -- hub/db/db.py | 4 +- hub/elastic_sync/service.py | 4 +- hub/{elastic_sync => herald}/search.py | 241 +---------------- hub/herald/session.py | 2 +- 7 files changed, 366 insertions(+), 260 deletions(-) rename hub/{elastic_sync => }/constants.py (100%) rename hub/{elastic_sync => herald}/search.py (55%) diff --git a/hub/common.py b/hub/common.py index 32df6dd..172a3b5 100644 --- a/hub/common.py +++ b/hub/common.py @@ -1,3 +1,4 @@ +import struct import hashlib import hmac import ipaddress @@ -5,8 +6,13 @@ import logging import logging.handlers import typing import collections +from decimal import Decimal +from typing import Iterable from asyncio import get_event_loop, Event from prometheus_client import Counter +from hub.schema.tags import clean_tags +from hub.schema.url import normalize_name +from hub.error import TooManyClaimSearchParametersError log = logging.getLogger(__name__) @@ -23,6 +29,22 @@ HISTOGRAM_BUCKETS = ( .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf') ) +CLAIM_TYPES = { + 'stream': 1, + 'channel': 2, + 'repost': 3, + 'collection': 4, +} + +STREAM_TYPES = { + 'video': 1, + 'audio': 2, + 'image': 3, + 'document': 4, + 'binary': 5, + 'model': 6, +} + def setup_logging(log_path: str): log = logging.getLogger('scribe') @@ -404,3 +426,340 @@ class IndexVersionMismatch(Exception): def __init__(self, got_version, expected_version): self.got_version = got_version self.expected_version = expected_version + + +# Elasticsearch constants + +INDEX_DEFAULT_SETTINGS = { + "settings": + {"analysis": + {"analyzer": { + "default": {"tokenizer": "whitespace", "filter": ["lowercase", "porter_stem"]}}}, + "index": + {"refresh_interval": -1, + "number_of_shards": 1, + "number_of_replicas": 0, + "sort": { + "field": ["trending_score", "release_time"], + "order": ["desc", "desc"] + }} + }, + "mappings": { + "properties": { + "claim_id": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text", + "index_prefixes": { + "min_chars": 1, + "max_chars": 10 + } + }, + "sd_hash": { + "fields": { + "keyword": { + "ignore_above": 96, + "type": "keyword" + } + }, + "type": "text", + "index_prefixes": { + "min_chars": 1, + "max_chars": 4 + } + }, + "height": {"type": "integer"}, + "claim_type": {"type": "byte"}, + "censor_type": {"type": "byte"}, + "trending_score": {"type": "double"}, + "release_time": {"type": "long"} + } + } +} + +FIELDS = { + '_id', + 'claim_id', 'claim_type', 'claim_name', 'normalized_name', + 'tx_id', 'tx_nout', 'tx_position', + 'short_url', 'canonical_url', + 'is_controlling', 'last_take_over_height', + 'public_key_bytes', 'public_key_id', 'claims_in_channel', + 'channel_id', 'signature', 'signature_digest', 'is_signature_valid', + 'amount', 'effective_amount', 'support_amount', + 'fee_amount', 'fee_currency', + 'height', 'creation_height', 'activation_height', 'expiration_height', + 'stream_type', 'media_type', 'censor_type', + 'title', 'author', 'description', + 'timestamp', 'creation_timestamp', + 'duration', 'release_time', + 'tags', 'languages', 'has_source', 'reposted_claim_type', + 'reposted_claim_id', 'repost_count', 'sd_hash', + 'trending_score', 'tx_num', + 'channel_tx_id', 'channel_tx_position', 'channel_height', 'reposted_tx_id', + 'reposted_tx_position', 'reposted_height', +} + +TEXT_FIELDS = { + 'author', 'canonical_url', 'channel_id', 'description', 'claim_id', 'censoring_channel_id', + 'media_type', 'normalized_name', 'public_key_bytes', 'public_key_id', 'short_url', 'signature', + 'claim_name', 'signature_digest', 'title', 'tx_id', 'fee_currency', 'reposted_claim_id', + 'tags', 'sd_hash', 'channel_tx_id', 'reposted_tx_id', +} + +RANGE_FIELDS = { + 'height', 'creation_height', 'activation_height', 'expiration_height', + 'timestamp', 'creation_timestamp', 'duration', 'release_time', 'fee_amount', + 'tx_position', 'repost_count', 'limit_claims_per_channel', + 'amount', 'effective_amount', 'support_amount', + 'trending_score', 'censor_type', 'tx_num', 'reposted_tx_position', 'reposted_height', + 'channel_tx_position', 'channel_height', +} + +ALL_FIELDS = RANGE_FIELDS | TEXT_FIELDS | FIELDS + +REPLACEMENTS = { + 'claim_name': 'normalized_name', + 'name': 'normalized_name', + 'txid': 'tx_id', + 'nout': 'tx_nout', + 'trending_group': 'trending_score', + 'trending_mixed': 'trending_score', + 'trending_global': 'trending_score', + 'trending_local': 'trending_score', + 'reposted': 'repost_count', + 'stream_types': 'stream_type', + 'media_types': 'media_type', + 'valid_channel_signature': 'is_signature_valid' +} + + +def expand_query(**kwargs): + if "amount_order" in kwargs: + kwargs["limit"] = 1 + kwargs["order_by"] = "effective_amount" + kwargs["offset"] = int(kwargs["amount_order"]) - 1 + if 'name' in kwargs: + kwargs['name'] = normalize_name(kwargs.pop('name')) + if kwargs.get('is_controlling') is False: + kwargs.pop('is_controlling') + query = {'must': [], 'must_not': []} + collapse = None + if 'fee_currency' in kwargs and kwargs['fee_currency'] is not None: + kwargs['fee_currency'] = kwargs['fee_currency'].upper() + for key, value in kwargs.items(): + key = key.replace('claim.', '') + many = key.endswith('__in') or isinstance(value, list) + if many and len(value) > 2048: + raise TooManyClaimSearchParametersError(key, 2048) + if many: + key = key.replace('__in', '') + value = list(filter(None, value)) + if value is None or isinstance(value, list) and len(value) == 0: + continue + key = REPLACEMENTS.get(key, key) + if key in FIELDS: + partial_id = False + if key == 'claim_type': + if isinstance(value, str): + value = CLAIM_TYPES[value] + else: + value = [CLAIM_TYPES[claim_type] for claim_type in value] + elif key == 'stream_type': + value = [STREAM_TYPES[value]] if isinstance(value, str) else list(map(STREAM_TYPES.get, value)) + if key == '_id': + if isinstance(value, Iterable): + value = [item[::-1].hex() for item in value] + else: + value = value[::-1].hex() + if not many and key in ('_id', 'claim_id', 'sd_hash') and len(value) < 20: + partial_id = True + if key in ('signature_valid', 'has_source'): + continue # handled later + if key in TEXT_FIELDS: + key += '.keyword' + ops = {'<=': 'lte', '>=': 'gte', '<': 'lt', '>': 'gt'} + if partial_id: + query['must'].append({"prefix": {key: value}}) + elif key in RANGE_FIELDS and isinstance(value, str) and value[0] in ops: + operator_length = 2 if value[:2] in ops else 1 + operator, value = value[:operator_length], value[operator_length:] + if key == 'fee_amount': + value = str(Decimal(value)*1000) + query['must'].append({"range": {key: {ops[operator]: value}}}) + elif key in RANGE_FIELDS and isinstance(value, list) and all(v[0] in ops for v in value): + range_constraints = [] + release_times = [] + for v in value: + operator_length = 2 if v[:2] in ops else 1 + operator, stripped_op_v = v[:operator_length], v[operator_length:] + if key == 'fee_amount': + stripped_op_v = str(Decimal(stripped_op_v)*1000) + if key == 'release_time': + release_times.append((operator, stripped_op_v)) + else: + range_constraints.append((operator, stripped_op_v)) + if key != 'release_time': + query['must'].append({"range": {key: {ops[operator]: v for operator, v in range_constraints}}}) + else: + query['must'].append( + {"bool": + {"should": [ + {"bool": { + "must_not": { + "exists": { + "field": "release_time" + } + } + }}, + {"bool": { + "must": [ + {"exists": {"field": "release_time"}}, + {'range': {key: {ops[operator]: v for operator, v in release_times}}}, + ]}}, + ]} + } + ) + elif many: + query['must'].append({"terms": {key: value}}) + else: + if key == 'fee_amount': + value = str(Decimal(value)*1000) + query['must'].append({"term": {key: {"value": value}}}) + elif key == 'not_channel_ids': + for channel_id in value: + query['must_not'].append({"term": {'channel_id.keyword': channel_id}}) + query['must_not'].append({"term": {'_id': channel_id}}) + elif key == 'channel_ids': + query['must'].append({"terms": {'channel_id.keyword': value}}) + elif key == 'claim_ids': + query['must'].append({"terms": {'claim_id.keyword': value}}) + elif key == 'media_types': + query['must'].append({"terms": {'media_type.keyword': value}}) + elif key == 'any_languages': + query['must'].append({"terms": {'languages': clean_tags(value)}}) + elif key == 'any_languages': + query['must'].append({"terms": {'languages': value}}) + elif key == 'all_languages': + query['must'].extend([{"term": {'languages': tag}} for tag in value]) + elif key == 'any_tags': + query['must'].append({"terms": {'tags.keyword': clean_tags(value)}}) + elif key == 'all_tags': + query['must'].extend([{"term": {'tags.keyword': tag}} for tag in clean_tags(value)]) + elif key == 'not_tags': + query['must_not'].extend([{"term": {'tags.keyword': tag}} for tag in clean_tags(value)]) + elif key == 'not_claim_id': + query['must_not'].extend([{"term": {'claim_id.keyword': cid}} for cid in value]) + elif key == 'limit_claims_per_channel': + collapse = ('channel_id.keyword', value) + if kwargs.get('has_channel_signature'): + query['must'].append({"exists": {"field": "signature"}}) + if 'signature_valid' in kwargs: + query['must'].append({"term": {"is_signature_valid": bool(kwargs["signature_valid"])}}) + elif 'signature_valid' in kwargs: + query['must'].append( + {"bool": + {"should": [ + {"bool": {"must_not": {"exists": {"field": "signature"}}}}, + {"bool" : {"must" : {"term": {"is_signature_valid": bool(kwargs["signature_valid"])}}}} + ]} + } + ) + if 'has_source' in kwargs: + is_stream_or_repost_terms = {"terms": {"claim_type": [CLAIM_TYPES['stream'], CLAIM_TYPES['repost']]}} + query['must'].append( + {"bool": + {"should": [ + {"bool": # when is_stream_or_repost AND has_source + {"must": [ + {"match": {"has_source": kwargs['has_source']}}, + is_stream_or_repost_terms, + ] + }, + }, + {"bool": # when not is_stream_or_repost + {"must_not": is_stream_or_repost_terms} + }, + {"bool": # when reposted_claim_type wouldn't have source + {"must_not": + [ + {"term": {"reposted_claim_type": CLAIM_TYPES['stream']}} + ], + "must": + [ + {"term": {"claim_type": CLAIM_TYPES['repost']}} + ] + } + } + ]} + } + ) + if kwargs.get('text'): + query['must'].append( + {"simple_query_string": + {"query": kwargs["text"], "fields": [ + "claim_name^4", "channel_name^8", "title^1", "description^.5", "author^1", "tags^.5" + ]}}) + query = { + "_source": {"excludes": ["description", "title"]}, + 'query': {'bool': query}, + "sort": [], + } + if "limit" in kwargs: + query["size"] = kwargs["limit"] + if 'offset' in kwargs: + query["from"] = kwargs["offset"] + if 'order_by' in kwargs: + if isinstance(kwargs["order_by"], str): + kwargs["order_by"] = [kwargs["order_by"]] + for value in kwargs['order_by']: + if 'trending_group' in value: + # fixme: trending_mixed is 0 for all records on variable decay, making sort slow. + continue + is_asc = value.startswith('^') + value = value[1:] if is_asc else value + value = REPLACEMENTS.get(value, value) + if value in TEXT_FIELDS: + value += '.keyword' + query['sort'].append({value: "asc" if is_asc else "desc"}) + if collapse: + query["collapse"] = { + "field": collapse[0], + "inner_hits": { + "name": collapse[0], + "size": collapse[1], + "sort": query["sort"] + } + } + return query + + +def expand_result(results): + inner_hits = [] + expanded = [] + for result in results: + if result.get("inner_hits"): + for _, inner_hit in result["inner_hits"].items(): + inner_hits.extend(inner_hit["hits"]["hits"]) + continue + result = result['_source'] + result['claim_hash'] = bytes.fromhex(result['claim_id'])[::-1] + if result['reposted_claim_id']: + result['reposted_claim_hash'] = bytes.fromhex(result['reposted_claim_id'])[::-1] + else: + result['reposted_claim_hash'] = None + result['channel_hash'] = bytes.fromhex(result['channel_id'])[::-1] if result['channel_id'] else None + result['txo_hash'] = bytes.fromhex(result['tx_id'])[::-1] + struct.pack(' 2048: - raise TooManyClaimSearchParametersError(key, 2048) - if many: - key = key.replace('__in', '') - value = list(filter(None, value)) - if value is None or isinstance(value, list) and len(value) == 0: - continue - key = REPLACEMENTS.get(key, key) - if key in FIELDS: - partial_id = False - if key == 'claim_type': - if isinstance(value, str): - value = CLAIM_TYPES[value] - else: - value = [CLAIM_TYPES[claim_type] for claim_type in value] - elif key == 'stream_type': - value = [STREAM_TYPES[value]] if isinstance(value, str) else list(map(STREAM_TYPES.get, value)) - if key == '_id': - if isinstance(value, Iterable): - value = [item[::-1].hex() for item in value] - else: - value = value[::-1].hex() - if not many and key in ('_id', 'claim_id', 'sd_hash') and len(value) < 20: - partial_id = True - if key in ('signature_valid', 'has_source'): - continue # handled later - if key in TEXT_FIELDS: - key += '.keyword' - ops = {'<=': 'lte', '>=': 'gte', '<': 'lt', '>': 'gt'} - if partial_id: - query['must'].append({"prefix": {key: value}}) - elif key in RANGE_FIELDS and isinstance(value, str) and value[0] in ops: - operator_length = 2 if value[:2] in ops else 1 - operator, value = value[:operator_length], value[operator_length:] - if key == 'fee_amount': - value = str(Decimal(value)*1000) - query['must'].append({"range": {key: {ops[operator]: value}}}) - elif key in RANGE_FIELDS and isinstance(value, list) and all(v[0] in ops for v in value): - range_constraints = [] - release_times = [] - for v in value: - operator_length = 2 if v[:2] in ops else 1 - operator, stripped_op_v = v[:operator_length], v[operator_length:] - if key == 'fee_amount': - stripped_op_v = str(Decimal(stripped_op_v)*1000) - if key == 'release_time': - release_times.append((operator, stripped_op_v)) - else: - range_constraints.append((operator, stripped_op_v)) - if key != 'release_time': - query['must'].append({"range": {key: {ops[operator]: v for operator, v in range_constraints}}}) - else: - query['must'].append( - {"bool": - {"should": [ - {"bool": { - "must_not": { - "exists": { - "field": "release_time" - } - } - }}, - {"bool": { - "must": [ - {"exists": {"field": "release_time"}}, - {'range': {key: {ops[operator]: v for operator, v in release_times}}}, - ]}}, - ]} - } - ) - elif many: - query['must'].append({"terms": {key: value}}) - else: - if key == 'fee_amount': - value = str(Decimal(value)*1000) - query['must'].append({"term": {key: {"value": value}}}) - elif key == 'not_channel_ids': - for channel_id in value: - query['must_not'].append({"term": {'channel_id.keyword': channel_id}}) - query['must_not'].append({"term": {'_id': channel_id}}) - elif key == 'channel_ids': - query['must'].append({"terms": {'channel_id.keyword': value}}) - elif key == 'claim_ids': - query['must'].append({"terms": {'claim_id.keyword': value}}) - elif key == 'media_types': - query['must'].append({"terms": {'media_type.keyword': value}}) - elif key == 'any_languages': - query['must'].append({"terms": {'languages': clean_tags(value)}}) - elif key == 'any_languages': - query['must'].append({"terms": {'languages': value}}) - elif key == 'all_languages': - query['must'].extend([{"term": {'languages': tag}} for tag in value]) - elif key == 'any_tags': - query['must'].append({"terms": {'tags.keyword': clean_tags(value)}}) - elif key == 'all_tags': - query['must'].extend([{"term": {'tags.keyword': tag}} for tag in clean_tags(value)]) - elif key == 'not_tags': - query['must_not'].extend([{"term": {'tags.keyword': tag}} for tag in clean_tags(value)]) - elif key == 'not_claim_id': - query['must_not'].extend([{"term": {'claim_id.keyword': cid}} for cid in value]) - elif key == 'limit_claims_per_channel': - collapse = ('channel_id.keyword', value) - if kwargs.get('has_channel_signature'): - query['must'].append({"exists": {"field": "signature"}}) - if 'signature_valid' in kwargs: - query['must'].append({"term": {"is_signature_valid": bool(kwargs["signature_valid"])}}) - elif 'signature_valid' in kwargs: - query['must'].append( - {"bool": - {"should": [ - {"bool": {"must_not": {"exists": {"field": "signature"}}}}, - {"bool" : {"must" : {"term": {"is_signature_valid": bool(kwargs["signature_valid"])}}}} - ]} - } - ) - if 'has_source' in kwargs: - is_stream_or_repost_terms = {"terms": {"claim_type": [CLAIM_TYPES['stream'], CLAIM_TYPES['repost']]}} - query['must'].append( - {"bool": - {"should": [ - {"bool": # when is_stream_or_repost AND has_source - {"must": [ - {"match": {"has_source": kwargs['has_source']}}, - is_stream_or_repost_terms, - ] - }, - }, - {"bool": # when not is_stream_or_repost - {"must_not": is_stream_or_repost_terms} - }, - {"bool": # when reposted_claim_type wouldn't have source - {"must_not": - [ - {"term": {"reposted_claim_type": CLAIM_TYPES['stream']}} - ], - "must": - [ - {"term": {"claim_type": CLAIM_TYPES['repost']}} - ] - } - } - ]} - } - ) - if kwargs.get('text'): - query['must'].append( - {"simple_query_string": - {"query": kwargs["text"], "fields": [ - "claim_name^4", "channel_name^8", "title^1", "description^.5", "author^1", "tags^.5" - ]}}) - query = { - "_source": {"excludes": ["description", "title"]}, - 'query': {'bool': query}, - "sort": [], - } - if "limit" in kwargs: - query["size"] = kwargs["limit"] - if 'offset' in kwargs: - query["from"] = kwargs["offset"] - if 'order_by' in kwargs: - if isinstance(kwargs["order_by"], str): - kwargs["order_by"] = [kwargs["order_by"]] - for value in kwargs['order_by']: - if 'trending_group' in value: - # fixme: trending_mixed is 0 for all records on variable decay, making sort slow. - continue - is_asc = value.startswith('^') - value = value[1:] if is_asc else value - value = REPLACEMENTS.get(value, value) - if value in TEXT_FIELDS: - value += '.keyword' - query['sort'].append({value: "asc" if is_asc else "desc"}) - if collapse: - query["collapse"] = { - "field": collapse[0], - "inner_hits": { - "name": collapse[0], - "size": collapse[1], - "sort": query["sort"] - } - } - return query - - -def expand_result(results): - inner_hits = [] - expanded = [] - for result in results: - if result.get("inner_hits"): - for _, inner_hit in result["inner_hits"].items(): - inner_hits.extend(inner_hit["hits"]["hits"]) - continue - result = result['_source'] - result['claim_hash'] = bytes.fromhex(result['claim_id'])[::-1] - if result['reposted_claim_id']: - result['reposted_claim_hash'] = bytes.fromhex(result['reposted_claim_id'])[::-1] - else: - result['reposted_claim_hash'] = None - result['channel_hash'] = bytes.fromhex(result['channel_id'])[::-1] if result['channel_id'] else None - result['txo_hash'] = bytes.fromhex(result['tx_id'])[::-1] + struct.pack(' Date: Wed, 18 May 2022 13:42:25 -0400 Subject: [PATCH 09/13] delete unused --- hub/constants.py | 105 ----------------------------------------------- 1 file changed, 105 deletions(-) delete mode 100644 hub/constants.py diff --git a/hub/constants.py b/hub/constants.py deleted file mode 100644 index 10193ef..0000000 --- a/hub/constants.py +++ /dev/null @@ -1,105 +0,0 @@ -INDEX_DEFAULT_SETTINGS = { - "settings": - {"analysis": - {"analyzer": { - "default": {"tokenizer": "whitespace", "filter": ["lowercase", "porter_stem"]}}}, - "index": - {"refresh_interval": -1, - "number_of_shards": 1, - "number_of_replicas": 0, - "sort": { - "field": ["trending_score", "release_time"], - "order": ["desc", "desc"] - }} - }, - "mappings": { - "properties": { - "claim_id": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text", - "index_prefixes": { - "min_chars": 1, - "max_chars": 10 - } - }, - "sd_hash": { - "fields": { - "keyword": { - "ignore_above": 96, - "type": "keyword" - } - }, - "type": "text", - "index_prefixes": { - "min_chars": 1, - "max_chars": 4 - } - }, - "height": {"type": "integer"}, - "claim_type": {"type": "byte"}, - "censor_type": {"type": "byte"}, - "trending_score": {"type": "double"}, - "release_time": {"type": "long"} - } - } -} - -FIELDS = { - '_id', - 'claim_id', 'claim_type', 'claim_name', 'normalized_name', - 'tx_id', 'tx_nout', 'tx_position', - 'short_url', 'canonical_url', - 'is_controlling', 'last_take_over_height', - 'public_key_bytes', 'public_key_id', 'claims_in_channel', - 'channel_id', 'signature', 'signature_digest', 'is_signature_valid', - 'amount', 'effective_amount', 'support_amount', - 'fee_amount', 'fee_currency', - 'height', 'creation_height', 'activation_height', 'expiration_height', - 'stream_type', 'media_type', 'censor_type', - 'title', 'author', 'description', - 'timestamp', 'creation_timestamp', - 'duration', 'release_time', - 'tags', 'languages', 'has_source', 'reposted_claim_type', - 'reposted_claim_id', 'repost_count', 'sd_hash', - 'trending_score', 'tx_num', - 'channel_tx_id', 'channel_tx_position', 'channel_height', 'reposted_tx_id', - 'reposted_tx_position', 'reposted_height', -} - -TEXT_FIELDS = { - 'author', 'canonical_url', 'channel_id', 'description', 'claim_id', 'censoring_channel_id', - 'media_type', 'normalized_name', 'public_key_bytes', 'public_key_id', 'short_url', 'signature', - 'claim_name', 'signature_digest', 'title', 'tx_id', 'fee_currency', 'reposted_claim_id', - 'tags', 'sd_hash', 'channel_tx_id', 'reposted_tx_id', -} - -RANGE_FIELDS = { - 'height', 'creation_height', 'activation_height', 'expiration_height', - 'timestamp', 'creation_timestamp', 'duration', 'release_time', 'fee_amount', - 'tx_position', 'repost_count', 'limit_claims_per_channel', - 'amount', 'effective_amount', 'support_amount', - 'trending_score', 'censor_type', 'tx_num', 'reposted_tx_position', 'reposted_height', - 'channel_tx_position', 'channel_height', -} - -ALL_FIELDS = RANGE_FIELDS | TEXT_FIELDS | FIELDS - -REPLACEMENTS = { - 'claim_name': 'normalized_name', - 'name': 'normalized_name', - 'txid': 'tx_id', - 'nout': 'tx_nout', - 'trending_group': 'trending_score', - 'trending_mixed': 'trending_score', - 'trending_global': 'trending_score', - 'trending_local': 'trending_score', - 'reposted': 'repost_count', - 'stream_types': 'stream_type', - 'media_types': 'media_type', - 'valid_channel_signature': 'is_signature_valid' -} -- 2.45.2 From f8022c9c9a32f0f0d54e3a8355125958243a6ee8 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 18 May 2022 13:42:36 -0400 Subject: [PATCH 10/13] move docker compose examples into docs --- {docker => docs/docker_examples}/docker-compose.yml | 0 {docker => docs/docker_examples}/elastic-compose.yml | 0 {docker => docs/docker_examples}/hub-compose.yml | 0 {docker => docs/docker_examples}/lbcd-compose.yml | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename {docker => docs/docker_examples}/docker-compose.yml (100%) rename {docker => docs/docker_examples}/elastic-compose.yml (100%) rename {docker => docs/docker_examples}/hub-compose.yml (100%) rename {docker => docs/docker_examples}/lbcd-compose.yml (100%) diff --git a/docker/docker-compose.yml b/docs/docker_examples/docker-compose.yml similarity index 100% rename from docker/docker-compose.yml rename to docs/docker_examples/docker-compose.yml diff --git a/docker/elastic-compose.yml b/docs/docker_examples/elastic-compose.yml similarity index 100% rename from docker/elastic-compose.yml rename to docs/docker_examples/elastic-compose.yml diff --git a/docker/hub-compose.yml b/docs/docker_examples/hub-compose.yml similarity index 100% rename from docker/hub-compose.yml rename to docs/docker_examples/hub-compose.yml diff --git a/docker/lbcd-compose.yml b/docs/docker_examples/lbcd-compose.yml similarity index 100% rename from docker/lbcd-compose.yml rename to docs/docker_examples/lbcd-compose.yml -- 2.45.2 From 706971edbeaa9cfe64d4c1cf8d38fab0daa778fc Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 18 May 2022 14:41:34 -0400 Subject: [PATCH 11/13] fix logging --- hub/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hub/common.py b/hub/common.py index 172a3b5..f077c01 100644 --- a/hub/common.py +++ b/hub/common.py @@ -47,7 +47,7 @@ STREAM_TYPES = { def setup_logging(log_path: str): - log = logging.getLogger('scribe') + log = logging.getLogger() fmt = logging.Formatter("%(asctime)s %(levelname)-4s %(name)s:%(lineno)d: %(message)s") handler = logging.handlers.RotatingFileHandler(log_path, maxBytes=1024*1024*5, backupCount=2) handler.setFormatter(fmt) -- 2.45.2 From b873f75ff6a2b81e11d66bc010ecaea6d06b34ee Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 18 May 2022 15:08:56 -0400 Subject: [PATCH 12/13] update entrypoint --- scripts/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index e78adec..6ff7cb1 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -11,7 +11,7 @@ fi case "$HUB_COMMAND" in scribe ) exec /home/lbry/.local/bin/scribe "$@" ;; - scribe-hub ) exec /home/lbry/.local/bin/herald "$@" ;; + herald ) exec /home/lbry/.local/bin/herald "$@" ;; scribe-elastic-sync ) exec /home/lbry/.local/bin/scribe-elastic-sync "$@" ;; * ) "HUB_COMMAND env variable must be scribe, herald, or scribe-elastic-sync" && exit 1 ;; esac -- 2.45.2 From 87c8457144dfc06a7e46783c6d73fafafda88b7f Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 18 May 2022 15:37:22 -0400 Subject: [PATCH 13/13] fix dockerfile --- Dockerfile | 2 +- scripts/deploy_scribe_dev.sh | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index f62dbe6..c8fd8ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,7 @@ RUN groupadd -g 999 $user && useradd -m -u 999 -g $user $user RUN mkdir -p $db_dir RUN chown -R $user:$user $db_dir -COPY docker $projects_dir +COPY . $projects_dir RUN chown -R $user:$user $projects_dir USER $user diff --git a/scripts/deploy_scribe_dev.sh b/scripts/deploy_scribe_dev.sh index c84297e..7f3c454 100755 --- a/scripts/deploy_scribe_dev.sh +++ b/scripts/deploy_scribe_dev.sh @@ -4,10 +4,9 @@ TARGET_HOST=$1 SCRIPTS_DIR=`dirname $0` -SCRIBE_DIR=`dirname $SCRIPTS_DIR` # build the image -docker build -f $SCRIBE_DIR/Dockerfile -t lbry/scribe:development $SCRIBE_DIR +docker build -t lbry/scribe:development . IMAGE=`docker image inspect lbry/scribe:development | sed -n "s/^.*Id\":\s*\"sha256:\s*\(\S*\)\".*$/\1/p"` # push the image to the server -- 2.45.2