From 53206a08619a6e8a73b15c4cc183c3acd552a120 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 4 May 2022 15:24:47 -0400 Subject: [PATCH] update readme and docker files --- README.md | 194 +++++++++++++++++++++++++++++++++++++- docker/Dockerfile | 7 -- docker/docker-compose.yml | 59 ++++++------ docker/hub-compose.yml | 54 +++++------ 4 files changed, 248 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index ffd1c1e..9f8c4f1 100644 --- a/README.md +++ b/README.md @@ -64,15 +64,203 @@ With options for high performance, if you have 64gb of memory and 12 cores, ever As of block 1147423 (4/21/22) the size of the scribe rocksdb database is 120GB and the size of the elasticsearch volume is 63GB. ### docker-compose - The recommended way to run a scribe hub is with docker. +If you have the resources to run all of the services on one machine (at least 300gb of fast storage, preferably nvme, 64gb of RAM, 12 fast cores), see [this](https://github.com/lbryio/scribe/blob/master/docker/docker-compose.yml) docker-compose example. + #### Cluster environment +For best performance the recommended setup uses three server instances, these can be rented VPSs, self hosted VMs (ideally not on one physical host unless the host is sufficiently powerful), or physical computers. One is a dedicated lbcd node, one an elasticsearch server, and the third runs the scribe services. With this configuration the lbcd and elasticsearch servers can be shared between multiple scribe hub servers - more on that later. +Server Requirements (space requirements are at least double what's needed so it's possible to copy snapshots into place or make snapshots): + - lbcd: 2 cores, 8gb ram (slightly more may be required syncing from scratch, from a snapshot 8 is plenty), 150gb of NVMe storage + - elasticsearch: 8 cores, 9gb of ram (8gb minimum given to ES), 150gb of SSD speed storage + - scribe: 8 cores, 32gb of ram, 200gb of NVMe storage -[Example docker-compose.yml](https://github.com/lbryio/scribe/blob/master/docker/elastic-compose.yml) for the elasticsearch instance +All servers are assumed to be running ubuntu 20.04 with user named `lbry` with passwordless sudo and docker group permissions, ssh configured, ulimits set high, and docker + docker-compose installed. The server running elasticsearch should have swap disabled. The three servers need to be able to communicate with each other, they can be on a local network together or communicate over the internet. This guide will assume the three servers are on the internet. -[Example docker-compose.yml](https://github.com/lbryio/scribe/blob/master/docker/hub-compose.yml) for the hub instance +##### Setting up the lbcd instance +Log in to the lbcd instance and perform the following steps: + - Build the lbcd docker image by running +``` +git clone https://github.com/lbryio/lbcd.git +cd lbcd +docker build . -t lbry/lbcd:latest +``` + - Copy the following to `~/docker-compose.yml` +``` +version: "3" +volumes: + lbcd: + +services: + lbcd: + image: lbry/lbcd:latest + restart: always + network_mode: host + command: + - "--rpcuser=lbry" + - "--rpcpass=lbry" + - "--rpclisten=127.0.0.1" + volumes: + - "lbcd:/root/.lbcd" + ports: + - "127.0.0.1:9245:9245" + - "9246:9246" # p2p port +``` + - Start lbcd by running `docker-compose up -d` + - Check the progress with `docker-compose logs -f --tail 100` + +##### Setting up the elasticsearch instance +Log in to the elasticsearch instance and perform the following steps: + - Copy the following to `~/docker-compose.yml` +``` +version: "3" + +volumes: + es01: + +services: + es01: + image: docker.elastic.co/elasticsearch/elasticsearch:7.16.0 + container_name: es01 + environment: + - node.name=es01 + - discovery.type=single-node + - indices.query.bool.max_clause_count=8192 + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Dlog4j2.formatMsgNoLookups=true -Xms8g -Xmx8g" # no more than 32, remember to disable swap + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - "es01:/usr/share/elasticsearch/data" + ports: + - "127.0.0.1:9200:9200" +``` + - Start elasticsearch by running `docker-compose up -d` + - Check the status with `docker-compose logs -f --tail 100` + +##### Setting up the scribe hub instance + - Log in (ssh) to the scribe instance and generate and print out a ssh key, this is needed to set up port forwards to the other two instances. Copy the output of the following: +``` +ssh-keygen -q -t ed25519 -N '' -f ~/.ssh/id_ed25519 <</dev/null 2>&1 +``` + - After copying the above key, log out of the scribe hub instance. + + - Log in to the elasticsearch instance add the copied key to `~/.ssh/authorized_keys` (see [this](https://stackoverflow.com/questions/6377009/adding-a-public-key-to-ssh-authorized-keys-does-not-log-me-in-automatically) if confused). Log out of the elasticsearch instance once done. + - Log in to the lbcd instance and add the copied key to `~/.ssh/authorized_keys`, log out when done. + - Log in to the scribe instance and copy the following to `/etc/systemd/system/es-tunnel.service`, replacing `lbry` with your user and `your-elastic-ip` with your elasticsearch instance ip. +``` +[Unit] +Description=Persistent SSH Tunnel for ES +After=network.target + +[Service] +Restart=on-failure +RestartSec=5 +ExecStart=/usr/bin/ssh -NTC -o ServerAliveInterval=60 -o ExitOnForwardFailure=yes -L 127.0.0.1:9200:127.0.0.1:9200 lbry@your-elastic-ip +User=lbry +Group=lbry + +[Install] +WantedBy=multi-user.target +``` + - Next, copy the following to `/etc/systemd/system/lbcd-tunnel.service` on the scribe instance, replacing `lbry` with your user and `your-lbcd-ip` with your lbcd instance ip. +``` +[Unit] +Description=Persistent SSH Tunnel for lbcd +After=network.target + +[Service] +Restart=on-failure +RestartSec=5 +ExecStart=/usr/bin/ssh -NTC -o ServerAliveInterval=60 -o ExitOnForwardFailure=yes -L 127.0.0.1:9245:127.0.0.1:9245 lbry@your-lbcd-ip +User=lbry +Group=lbry + +[Install] +WantedBy=multi-user.target +``` + - Verify you can ssh in to the elasticsearch and lbcd instances from the scribe instance + - Enable and start the ssh port forward services on the scribe instance +``` +sudo systemctl enable es-tunnel.service +sudo systemctl enable lbcd-tunnel.service +sudo systemctl start es-tunnel.service +sudo systemctl start lbcd-tunnel.service +``` + - Build the scribe docker image on the scribe hub instance by running the following: +``` +git clone https://github.com/lbryio/scribe.git +cd scribe +docker build -f ./docker/Dockerfile.scribe -t lbry/scribe:development . +``` + - Copy the following to `~/docker-compose.yml` on the scribe instance +``` +version: "3" + +volumes: + lbry_rocksdb: + +services: + scribe: + depends_on: + - scribe_elastic_sync + image: lbry/scribe:${SCRIBE_TAG:-development} + restart: always + network_mode: host + volumes: + - "lbry_rocksdb:/database" + environment: + - HUB_COMMAND=scribe + command: + - "--daemon_url=http://lbry:lbry@127.0.0.1:9245" + - "--max_query_workers=2" + - "--cache_all_tx_hashes" + scribe_elastic_sync: + image: lbry/scribe:${SCRIBE_TAG:-development} + restart: always + network_mode: host + ports: + - "127.0.0.1:19080:19080" # elastic notifier port + volumes: + - "lbry_rocksdb:/database" + environment: + - HUB_COMMAND=scribe-elastic-sync + command: + - "--elastic_host=127.0.0.1" + - "--elastic_port=9200" + - "--max_query_workers=2" + - "--filtering_channel_ids=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8" + - "--blocking_channel_ids=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6" + scribe_hub: + depends_on: + - scribe_elastic_sync + - scribe + image: lbry/scribe:${SCRIBE_TAG:-development} + restart: always + network_mode: host + ports: + - "50001:50001" # electrum rpc port and udp ping port + - "2112:2112" # comment out to disable prometheus metrics + volumes: + - "lbry_rocksdb:/database" + environment: + - HUB_COMMAND=scribe-hub + command: + - "--daemon_url=http://lbry:lbry@127.0.0.1:9245" + - "--elastic_host=127.0.0.1" + - "--elastic_port=9200" + - "--max_query_workers=4" + - "--host=0.0.0.0" + - "--max_sessions=100000" + - "--filtering_channel_ids=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8" + - "--blocking_channel_ids=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6" + - "--prometheus_port=2112" # comment out to disable prometheus metrics +``` + - Start the scribe hub services by running `docker-compose up -d` + - Check the status with `docker-compose logs -f --tail 100` ### From source diff --git a/docker/Dockerfile b/docker/Dockerfile index 256ee1d..a83e9ee 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -39,15 +39,8 @@ RUN python3.9 docker/set_build.py RUN rm ~/.cache -rf # entry point -ARG host=localhost -ARG tcp_port=50001 -ARG daemon_url=http://lbry:lbry@localhost:9245/ VOLUME $db_dir -ENV TCP_PORT=$tcp_port -ENV HOST=$host -ENV DAEMON_URL=$daemon_url ENV DB_DIRECTORY=$db_dir -ENV MAX_SESSIONS=100000 ENV MAX_SEND=1000000000000000000 ENV MAX_RECEIVE=1000000000000000000 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index fcf950b..3d4f617 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -10,41 +10,43 @@ services: depends_on: - lbcd - scribe_elastic_sync - image: lbry/scribe:${SCRIBE_TAG:-latest-release} + image: lbry/scribe:${SCRIBE_TAG:-latest} restart: always network_mode: host volumes: - "lbry_rocksdb:/database" - environment: # for full options, see `scribe --help` + environment: - HUB_COMMAND=scribe - - DAEMON_URL=http://lbry:lbry@127.0.0.1:9245 - - MAX_QUERY_WORKERS=2 - #- CACHE_ALL_TX_HASHES=Yes # uncomment to keep an index of all tx hashes in memory. This uses lots (10+g) of memory but substantially improves performance. + command: # for full options, see `scribe --help` + - "--daemon_url=http://lbry:lbry@127.0.0.1:9245" + - "--max_query_workers=2" + # - "--cache_all_tx_hashes" # uncomment to keep an index of all tx hashes in memory. This uses lots (10+g) of memory but substantially improves performance. scribe_elastic_sync: depends_on: - es01 - image: lbry/scribe:${SCRIBE_TAG:-latest-release} + image: lbry/scribe:${SCRIBE_TAG:-latest} restart: always network_mode: host ports: - "127.0.0.1:19080:19080" # elastic notifier port volumes: - "lbry_rocksdb:/database" - environment: # for full options, see `scribe-elastic-sync --help` + environment: - HUB_COMMAND=scribe-elastic-sync - - MAX_QUERY_WORKERS=2 - - FILTERING_CHANNEL_IDS=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8 - - BLOCKING_CHANNEL_IDS=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6 - #- ELASTIC_HOST=127.0.0.1 # elasticsearch host - #- ELASTIC_PORT=9200 # elasticsearch port - #- ELASTIC_NOTIFIER_HOST=127.0.0.1 # address for the elastic sync notifier to connect to - #- ELASTIC_NOTIFIER_PORT=19080 + command: # for full options, see `scribe-elastic-sync --help` + - "--max_query_workers=2" + - "--filtering_channel_ids=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8" + - "--blocking_channel_ids=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6" + - "--elastic_host=127.0.0.1" # elasticsearch host + - "--elastic_port=9200" # elasticsearch port + - "--elastic_notifier_host=127.0.0.1" # address for the elastic sync notifier to connect to + - "--elastic_notifier_port=19080" scribe_hub: depends_on: - lbcd - scribe_elastic_sync - scribe - image: lbry/scribe:${SCRIBE_TAG:-latest-release} + image: lbry/scribe:${SCRIBE_TAG:-latest} restart: always network_mode: host ports: @@ -52,20 +54,21 @@ services: - "2112:2112" # comment out to disable prometheus metrics volumes: - "lbry_rocksdb:/database" - environment: # for full options, see `scribe-hub --help` + environment: - HUB_COMMAND=scribe-hub - - DAEMON_URL=http://lbry:lbry@127.0.0.1:9245 # used for broadcasting transactions - - HOST=0.0.0.0 - - MAX_QUERY_WORKERS=4 # reader threads - - PROMETHEUS_PORT=2112 # comment out to disable prometheus metrics - - FILTERING_CHANNEL_IDS=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8 - - BLOCKING_CHANNEL_IDS=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6 - #- MAX_SESSIONS=100000 # uncomment to increase the maximum number of electrum connections, defaults to 1000 - #- ALLOW_LAN_UDP=Yes # uncomment to reply to clients on the local network - #- ELASTIC_HOST=127.0.0.1 # elasticsearch host - #- ELASTIC_PORT=9200 # elasticsearch port - #- ELASTIC_NOTIFIER_HOST=127.0.0.1 # address for the elastic sync notifier to connect to - #- ELASTIC_NOTIFIER_PORT=19080 + command: # for full options, see `scribe-hub --help` + - "--daemon_url=http://lbry:lbry@127.0.0.1:9245" + - "--max_query_workers=4" + - "--host=0.0.0.0" + - "--filtering_channel_ids=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8" + - "--blocking_channel_ids=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6" + - "--prometheus_port=2112" # comment out to disable prometheus metrics + # - "--elastic_host=127.0.0.1" # elasticsearch host + # - "--elastic_port=9200" # elasticsearch port + # - "--elastic_notifier_host=127.0.0.1" # address for the elastic sync notifier to connect to + # - "--elastic_notifier_port=19080" + # - "--max_sessions=100000 # uncomment to increase the maximum number of electrum connections, defaults to 1000 + # - "--allow_lan_udp" # uncomment to reply to clients on the local network es01: image: docker.elastic.co/elasticsearch/elasticsearch:7.16.0 container_name: es01 diff --git a/docker/hub-compose.yml b/docker/hub-compose.yml index 733933a..7364ce2 100644 --- a/docker/hub-compose.yml +++ b/docker/hub-compose.yml @@ -7,38 +7,38 @@ services: scribe: depends_on: - scribe_elastic_sync - image: lbry/scribe:${SCRIBE_TAG:-latest-release} + image: lbry/scribe:${SCRIBE_TAG:-development} restart: always network_mode: host volumes: - "lbry_rocksdb:/database" - environment: # for full options, see `scribe --help` + environment: - HUB_COMMAND=scribe - - DAEMON_URL=http://lbry:lbry@127.0.0.1:9245 - - MAX_QUERY_WORKERS=2 - #- CACHE_ALL_TX_HASHES=Yes # uncomment to keep an index of all tx hashes in memory. This uses lots (10+g) of memory but substantially improves performance. + command: + - "--daemon_url=http://lbry:lbry@127.0.0.1:9245" + - "--max_query_workers=2" + - "--cache_all_tx_hashes" scribe_elastic_sync: - image: lbry/scribe:${SCRIBE_TAG:-latest-release} + image: lbry/scribe:${SCRIBE_TAG:-development} restart: always network_mode: host ports: - "127.0.0.1:19080:19080" # elastic notifier port volumes: - "lbry_rocksdb:/database" - environment: # for full options, see `scribe-elastic-sync --help` + environment: - HUB_COMMAND=scribe-elastic-sync - - MAX_QUERY_WORKERS=2 - - FILTERING_CHANNEL_IDS=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8 - - BLOCKING_CHANNEL_IDS=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6 - #- ELASTIC_HOST=127.0.0.1 # elasticsearch host - #- ELASTIC_PORT=9200 # elasticsearch port - #- ELASTIC_NOTIFIER_HOST=127.0.0.1 # address for the elastic sync notifier to connect to - #- ELASTIC_NOTIFIER_PORT=19080 + command: + - "--elastic_host=127.0.0.1" + - "--elastic_port=9200" + - "--max_query_workers=2" + - "--filtering_channel_ids=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8" + - "--blocking_channel_ids=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6" scribe_hub: depends_on: - scribe_elastic_sync - scribe - image: lbry/scribe:${SCRIBE_TAG:-latest-release} + image: lbry/scribe:${SCRIBE_TAG:-development} restart: always network_mode: host ports: @@ -46,17 +46,15 @@ services: - "2112:2112" # comment out to disable prometheus metrics volumes: - "lbry_rocksdb:/database" - environment: # for full options, see `scribe-hub --help` + environment: - HUB_COMMAND=scribe-hub - - DAEMON_URL=http://lbry:lbry@127.0.0.1:9245 # used for broadcasting transactions - - HOST=0.0.0.0 - - MAX_QUERY_WORKERS=4 # reader threads - - PROMETHEUS_PORT=2112 # comment out to disable prometheus metrics - - FILTERING_CHANNEL_IDS=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8 - - BLOCKING_CHANNEL_IDS=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6 - #- MAX_SESSIONS=100000 # uncomment to increase the maximum number of electrum connections, defaults to 1000 - #- ALLOW_LAN_UDP=Yes # uncomment to reply to clients on the local network - #- ELASTIC_HOST=127.0.0.1 # elasticsearch host - #- ELASTIC_PORT=9200 # elasticsearch port - #- ELASTIC_NOTIFIER_HOST=127.0.0.1 # address for the elastic sync notifier to connect to - #- ELASTIC_NOTIFIER_PORT=19080 + command: + - "--daemon_url=http://lbry:lbry@127.0.0.1:9245" + - "--elastic_host=127.0.0.1" + - "--elastic_port=9200" + - "--max_query_workers=4" + - "--host=0.0.0.0" + - "--max_sessions=100000" + - "--filtering_channel_ids=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8" + - "--blocking_channel_ids=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6" + - "--prometheus_port=2112" # comment out to disable prometheus metrics \ No newline at end of file