docker compose update

Makefile

@@ -6,7 +6,6 @@ install:
 		--global-option=fetch \
 		--global-option=--version --global-option=3.30.1 --global-option=--all \
 		--global-option=build --global-option=--enable --global-option=fts5
-	python -m pip install elasticsearch[async]
 	pip install -e .
 
 tools:

docker/Dockerfile.wallet_server

@@ -13,6 +13,8 @@ RUN apt-get update && \
       wget \
       tar unzip \
       build-essential \
+      pkg-config \
+      libleveldb-dev \
       python3 \
       python3-dev \
       python3-pip \

docker/docker-compose-wallet-server.yml

@@ -3,6 +3,7 @@ version: "3"
 volumes:
   lbrycrd:
   wallet_server:
+  es01:
 
 services:
   lbrycrd:
@@ -34,3 +35,18 @@ services:
       # Curently not snapshot provided
       # - SNAPSHOT_URL=${WALLET_SERVER_SNAPSHOT_URL-https://lbry.com/snapshot/wallet}
       - DAEMON_URL=http://lbry:lbry@lbrycrd:9245
+  es01:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0
+    container_name: es01
+    environment:
+      - node.name=es01
+      - discovery.type=single-node
+      - bootstrap.memory_lock=true
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - es01:/usr/share/elasticsearch/data
+    ports:
+      - 127.0.0.1:9200:9200

lbry/wallet/server/db/elastic_search.py

@@ -5,7 +5,7 @@ from decimal import Decimal
 from operator import itemgetter
 from typing import Optional, List, Iterable
 
-from elasticsearch import AsyncElasticsearch, NotFoundError
+from elasticsearch import AsyncElasticsearch, NotFoundError, ConnectionError
 from elasticsearch.helpers import async_bulk
 
 from lbry.crypto.base58 import Base58
@@ -14,6 +14,7 @@ from lbry.schema.result import Outputs, Censor
 from lbry.schema.tags import clean_tags
 from lbry.schema.url import URL, normalize_name
 from lbry.wallet.server.db.common import CLAIM_TYPES, STREAM_TYPES
+from lbry.wallet.server.util import class_logger
 
 
 class SearchIndex:
@@ -21,51 +22,54 @@ class SearchIndex:
         self.client: Optional[AsyncElasticsearch] = None
         self.index = index_prefix + 'claims'
         self.sync_timeout = 600  # wont hit that 99% of the time, but can hit on a fresh import
+        self.logger = class_logger(__name__, self.__class__.__name__)
 
     async def start(self):
         if self.client:
             return
         self.client = AsyncElasticsearch(timeout=self.sync_timeout)
-        try:
+        while True:
-            if await self.client.indices.exists(self.index):
+            try:
-                return
+                await self.client.cluster.health(wait_for_status='yellow')
-            await self.client.indices.create(
+                break
-                self.index,
+            except ConnectionError:
-                {
+                self.logger.warning("Failed to connect to Elasticsearch. Waiting for it!")
-                    "settings":
+                await asyncio.sleep(1)
-                        {"analysis":
+        await self.client.indices.create(
-                            {"analyzer": {
+            self.index,
-                                "default": {"tokenizer": "whitespace", "filter": ["lowercase", "porter_stem"]}}},
+            {
-                            "index":
+                "settings":
-                                {"refresh_interval": -1,
+                    {"analysis":
-                                 "number_of_shards": 1,
+                        {"analyzer": {
-                                 "number_of_replicas": 0}
+                            "default": {"tokenizer": "whitespace", "filter": ["lowercase", "porter_stem"]}}},
-                        },
+                        "index":
-                    "mappings": {
+                            {"refresh_interval": -1,
-                        "properties": {
+                             "number_of_shards": 1,
-                            "claim_id": {
+                             "number_of_replicas": 0}
-                                "fields": {
+                    },
-                                    "keyword": {
+                "mappings": {
-                                        "ignore_above": 256,
+                    "properties": {
-                                        "type": "keyword"
+                        "claim_id": {
-                                    }
+                            "fields": {
-                                },
+                                "keyword": {
-                                "type": "text",
+                                    "ignore_above": 256,
-                                "index_prefixes": {
+                                    "type": "keyword"
-                                    "min_chars": 1,
-                                    "max_chars": 10
                                 }
                             },
-                            "height": {"type": "integer"},
+                            "type": "text",
-                            "claim_type": {"type": "byte"},
+                            "index_prefixes": {
-                            "censor_type": {"type": "byte"},
+                                "min_chars": 1,
-                            "trending_mixed": {"type": "float"},
+                                "max_chars": 10
-                        }
+                            }
+                        },
+                        "height": {"type": "integer"},
+                        "claim_type": {"type": "byte"},
+                        "censor_type": {"type": "byte"},
+                        "trending_mixed": {"type": "float"},
                     }
                 }
-            )
+            }, ignore=400
-        except Exception as e:
+        )
-            raise
 
     def stop(self):
         client = self.client