Optionally initialize rocksdb from a snapshot #48

Merged
jackrobison merged 2 commits from initialize-from-snapshot into master 2022-06-14 20:35:22 +02:00
3 changed files with 19 additions and 1 deletions
Showing only changes of commit fbe68d516c - Show all commits

View file

@ -17,6 +17,7 @@ services:
- "lbry_rocksdb:/database" - "lbry_rocksdb:/database"
environment: environment:
- HUB_COMMAND=scribe - HUB_COMMAND=scribe
- SNAPSHOT_URL=https://snapshots.lbry.com/hub/lbry-rocksdb.zip
command: # for full options, see `scribe --help` command: # for full options, see `scribe --help`
- "--daemon_url=http://lbry:lbry@127.0.0.1:9245" - "--daemon_url=http://lbry:lbry@127.0.0.1:9245"
- "--max_query_workers=2" - "--max_query_workers=2"
@ -41,7 +42,7 @@ services:
- "--elastic_port=9200" # elasticsearch port - "--elastic_port=9200" # elasticsearch port
- "--elastic_notifier_host=127.0.0.1" # address for the elastic sync notifier to connect to - "--elastic_notifier_host=127.0.0.1" # address for the elastic sync notifier to connect to
- "--elastic_notifier_port=19080" - "--elastic_notifier_port=19080"
scribe_hub: herald:
depends_on: depends_on:
- lbcd - lbcd
- scribe_elastic_sync - scribe_elastic_sync

View file

@ -14,6 +14,7 @@ services:
- "lbry_rocksdb:/database" - "lbry_rocksdb:/database"
environment: environment:
- HUB_COMMAND=scribe - HUB_COMMAND=scribe
- SNAPSHOT_URL=https://snapshots.lbry.com/hub/lbry-rocksdb.zip
command: command:
- "--daemon_url=http://lbry:lbry@127.0.0.1:9245" - "--daemon_url=http://lbry:lbry@127.0.0.1:9245"
- "--max_query_workers=2" - "--max_query_workers=2"

View file

@ -4,6 +4,22 @@
set -euo pipefail set -euo pipefail
SNAPSHOT_URL="${SNAPSHOT_URL:-}" #off by default. latest snapshot at https://lbry.com/snapshot/hub
if [[ "$HUB_COMMAND" == "scribe" ]] && [[ -n "$SNAPSHOT_URL" ]] && [[ ! -d /database/lbry-rocksdb ]]; then
files="$(ls)"
echo "Downloading hub snapshot from $SNAPSHOT_URL"
wget --no-verbose --trust-server-names --content-disposition "$SNAPSHOT_URL"
echo "Extracting snapshot..."
filename="$(grep -vf <(echo "$files") <(ls))" # finds the file that was not there before
case "$filename" in
*.tgz|*.tar.gz|*.tar.bz2 ) tar xvf "$filename" --directory /database ;;
*.zip ) unzip "$filename" -d /database/ ;;
* ) echo "Don't know how to extract ${filename}. SNAPSHOT COULD NOT BE LOADED" && exit 1 ;;
esac
rm "$filename"
fi
if [ -z "$HUB_COMMAND" ]; then if [ -z "$HUB_COMMAND" ]; then
echo "HUB_COMMAND env variable must be scribe, herald, or scribe-elastic-sync" echo "HUB_COMMAND env variable must be scribe, herald, or scribe-elastic-sync"
exit 1 exit 1