From fbe68d516c9bf9ccf880d6be54883899a8b471d2 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 14 Jun 2022 13:55:53 -0400 Subject: [PATCH 1/2] optionally initialize rocksdb from a snapshot fixes https://github.com/lbryio/hub/issues/10 --- docs/docker_examples/docker-compose.yml | 3 ++- docs/docker_examples/hub-compose.yml | 1 + scripts/entrypoint.sh | 16 ++++++++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/docker_examples/docker-compose.yml b/docs/docker_examples/docker-compose.yml index cc979af..5d33924 100644 --- a/docs/docker_examples/docker-compose.yml +++ b/docs/docker_examples/docker-compose.yml @@ -17,6 +17,7 @@ services: - "lbry_rocksdb:/database" environment: - HUB_COMMAND=scribe + - SNAPSHOT_URL=https://snapshots.lbry.com/hub/lbry-rocksdb.zip command: # for full options, see `scribe --help` - "--daemon_url=http://lbry:lbry@127.0.0.1:9245" - "--max_query_workers=2" @@ -41,7 +42,7 @@ services: - "--elastic_port=9200" # elasticsearch port - "--elastic_notifier_host=127.0.0.1" # address for the elastic sync notifier to connect to - "--elastic_notifier_port=19080" - scribe_hub: + herald: depends_on: - lbcd - scribe_elastic_sync diff --git a/docs/docker_examples/hub-compose.yml b/docs/docker_examples/hub-compose.yml index 2517e63..96e70f5 100644 --- a/docs/docker_examples/hub-compose.yml +++ b/docs/docker_examples/hub-compose.yml @@ -14,6 +14,7 @@ services: - "lbry_rocksdb:/database" environment: - HUB_COMMAND=scribe + - SNAPSHOT_URL=https://snapshots.lbry.com/hub/lbry-rocksdb.zip command: - "--daemon_url=http://lbry:lbry@127.0.0.1:9245" - "--max_query_workers=2" diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index 6ff7cb1..bef8856 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -4,6 +4,22 @@ set -euo pipefail +SNAPSHOT_URL="${SNAPSHOT_URL:-}" #off by default. latest snapshot at https://lbry.com/snapshot/hub + +if [[ "$HUB_COMMAND" == "scribe" ]] && [[ -n "$SNAPSHOT_URL" ]] && [[ ! -d /database/lbry-rocksdb ]]; then + files="$(ls)" + echo "Downloading hub snapshot from $SNAPSHOT_URL" + wget --no-verbose --trust-server-names --content-disposition "$SNAPSHOT_URL" + echo "Extracting snapshot..." + filename="$(grep -vf <(echo "$files") <(ls))" # finds the file that was not there before + case "$filename" in + *.tgz|*.tar.gz|*.tar.bz2 ) tar xvf "$filename" --directory /database ;; + *.zip ) unzip "$filename" -d /database/ ;; + * ) echo "Don't know how to extract ${filename}. SNAPSHOT COULD NOT BE LOADED" && exit 1 ;; + esac + rm "$filename" +fi + if [ -z "$HUB_COMMAND" ]; then echo "HUB_COMMAND env variable must be scribe, herald, or scribe-elastic-sync" exit 1 -- 2.45.2 From 09ea58c062f44343beec00d0bb1240b58fa05fd1 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 14 Jun 2022 14:20:31 -0400 Subject: [PATCH 2/2] update examples --- docs/cluster_guide.md | 3 +++ docs/docker_examples/docker-compose.yml | 2 ++ 2 files changed, 5 insertions(+) diff --git a/docs/cluster_guide.md b/docs/cluster_guide.md index 2f993b2..c5c41fa 100644 --- a/docs/cluster_guide.md +++ b/docs/cluster_guide.md @@ -146,10 +146,12 @@ services: - "lbry_rocksdb:/database" environment: - HUB_COMMAND=scribe + - SNAPSHOT_URL=https://snapshots.lbry.com/hub/lbry-rocksdb.zip command: - "--daemon_url=http://lbry:lbry@127.0.0.1:9245" - "--max_query_workers=2" - "--cache_all_tx_hashes" + - "--index_address_statuses" scribe_elastic_sync: image: lbry/hub:${SCRIBE_TAG:-development} restart: always @@ -183,6 +185,7 @@ services: - FILTERING_CHANNEL_IDS=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8 - BLOCKING_CHANNEL_IDS=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6 command: + - "--index_address_statuses" - "--daemon_url=http://lbry:lbry@127.0.0.1:9245" - "--elastic_host=127.0.0.1" - "--elastic_port=9200" diff --git a/docs/docker_examples/docker-compose.yml b/docs/docker_examples/docker-compose.yml index 5d33924..406d321 100644 --- a/docs/docker_examples/docker-compose.yml +++ b/docs/docker_examples/docker-compose.yml @@ -22,6 +22,7 @@ services: - "--daemon_url=http://lbry:lbry@127.0.0.1:9245" - "--max_query_workers=2" # - "--cache_all_tx_hashes" # uncomment to keep an index of all tx hashes in memory. This uses lots (10+g) of memory but substantially improves performance. + - "--index_address_statuses" scribe_elastic_sync: depends_on: - es01 @@ -60,6 +61,7 @@ services: - FILTERING_CHANNEL_IDS=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8 - BLOCKING_CHANNEL_IDS=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6 command: # for full options, see `herald --help` + - "--index_address_statuses" - "--daemon_url=http://lbry:lbry@127.0.0.1:9245" - "--max_query_workers=4" - "--host=0.0.0.0" -- 2.45.2