Merge pull request #48 from lbryio/initialize-from-snapshot
Optionally initialize rocksdb from a snapshot
This commit is contained in:
commit
6fd718f353
4 changed files with 24 additions and 1 deletions
|
@ -146,10 +146,12 @@ services:
|
|||
- "lbry_rocksdb:/database"
|
||||
environment:
|
||||
- HUB_COMMAND=scribe
|
||||
- SNAPSHOT_URL=https://snapshots.lbry.com/hub/lbry-rocksdb.zip
|
||||
command:
|
||||
- "--daemon_url=http://lbry:lbry@127.0.0.1:9245"
|
||||
- "--max_query_workers=2"
|
||||
- "--cache_all_tx_hashes"
|
||||
- "--index_address_statuses"
|
||||
scribe_elastic_sync:
|
||||
image: lbry/hub:${SCRIBE_TAG:-development}
|
||||
restart: always
|
||||
|
@ -183,6 +185,7 @@ services:
|
|||
- FILTERING_CHANNEL_IDS=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8
|
||||
- BLOCKING_CHANNEL_IDS=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6
|
||||
command:
|
||||
- "--index_address_statuses"
|
||||
- "--daemon_url=http://lbry:lbry@127.0.0.1:9245"
|
||||
- "--elastic_host=127.0.0.1"
|
||||
- "--elastic_port=9200"
|
||||
|
|
|
@ -17,10 +17,12 @@ services:
|
|||
- "lbry_rocksdb:/database"
|
||||
environment:
|
||||
- HUB_COMMAND=scribe
|
||||
- SNAPSHOT_URL=https://snapshots.lbry.com/hub/lbry-rocksdb.zip
|
||||
command: # for full options, see `scribe --help`
|
||||
- "--daemon_url=http://lbry:lbry@127.0.0.1:9245"
|
||||
- "--max_query_workers=2"
|
||||
# - "--cache_all_tx_hashes" # uncomment to keep an index of all tx hashes in memory. This uses lots (10+g) of memory but substantially improves performance.
|
||||
- "--index_address_statuses"
|
||||
scribe_elastic_sync:
|
||||
depends_on:
|
||||
- es01
|
||||
|
@ -41,7 +43,7 @@ services:
|
|||
- "--elastic_port=9200" # elasticsearch port
|
||||
- "--elastic_notifier_host=127.0.0.1" # address for the elastic sync notifier to connect to
|
||||
- "--elastic_notifier_port=19080"
|
||||
scribe_hub:
|
||||
herald:
|
||||
depends_on:
|
||||
- lbcd
|
||||
- scribe_elastic_sync
|
||||
|
@ -59,6 +61,7 @@ services:
|
|||
- FILTERING_CHANNEL_IDS=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8
|
||||
- BLOCKING_CHANNEL_IDS=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6
|
||||
command: # for full options, see `herald --help`
|
||||
- "--index_address_statuses"
|
||||
- "--daemon_url=http://lbry:lbry@127.0.0.1:9245"
|
||||
- "--max_query_workers=4"
|
||||
- "--host=0.0.0.0"
|
||||
|
|
|
@ -14,6 +14,7 @@ services:
|
|||
- "lbry_rocksdb:/database"
|
||||
environment:
|
||||
- HUB_COMMAND=scribe
|
||||
- SNAPSHOT_URL=https://snapshots.lbry.com/hub/lbry-rocksdb.zip
|
||||
command:
|
||||
- "--daemon_url=http://lbry:lbry@127.0.0.1:9245"
|
||||
- "--max_query_workers=2"
|
||||
|
|
|
@ -4,6 +4,22 @@
|
|||
|
||||
set -euo pipefail
|
||||
|
||||
SNAPSHOT_URL="${SNAPSHOT_URL:-}" #off by default. latest snapshot at https://lbry.com/snapshot/hub
|
||||
|
||||
if [[ "$HUB_COMMAND" == "scribe" ]] && [[ -n "$SNAPSHOT_URL" ]] && [[ ! -d /database/lbry-rocksdb ]]; then
|
||||
files="$(ls)"
|
||||
echo "Downloading hub snapshot from $SNAPSHOT_URL"
|
||||
wget --no-verbose --trust-server-names --content-disposition "$SNAPSHOT_URL"
|
||||
echo "Extracting snapshot..."
|
||||
filename="$(grep -vf <(echo "$files") <(ls))" # finds the file that was not there before
|
||||
case "$filename" in
|
||||
*.tgz|*.tar.gz|*.tar.bz2 ) tar xvf "$filename" --directory /database ;;
|
||||
*.zip ) unzip "$filename" -d /database/ ;;
|
||||
* ) echo "Don't know how to extract ${filename}. SNAPSHOT COULD NOT BE LOADED" && exit 1 ;;
|
||||
esac
|
||||
rm "$filename"
|
||||
fi
|
||||
|
||||
if [ -z "$HUB_COMMAND" ]; then
|
||||
echo "HUB_COMMAND env variable must be scribe, herald, or scribe-elastic-sync"
|
||||
exit 1
|
||||
|
|
Loading…
Reference in a new issue