add wallet snapshotting
This commit is contained in:
Alex Grintsvayg
parent
94c15da1b6
commit
9f4327bb52
4 changed files with 61 additions and 78 deletions
|
|
@ -1,33 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
echo "Running $0"
|
||||
|
||||
echo "Installing a few tools"
|
||||
sudo apt update && \
|
||||
sudo apt install -y unzip awscli
|
||||
|
||||
echo "Installing lbrycrd"
|
||||
wget -O $HOME/lbrycrd-linux.zip $(curl -s https://api.github.com/repos/lbryio/lbrycrd/releases | grep -F 'lbrycrd-linux' | grep download | head -n 1 | cut -d'"' -f4) && \
|
||||
unzip $HOME/lbrycrd-linux.zip -d $HOME && \
|
||||
rm $HOME/lbrycrd-linux.zip
|
||||
|
||||
mkdir -p "$HOME/.lbrycrd"
|
||||
|
||||
echo "Downloading snapshot"
|
||||
wget -O $HOME/blockchain_snapshot.tar.bz2 https://lbry.com/snapshot/blockchain && \
|
||||
tar xvjf $HOME/blockchain_snapshot.tar.bz2 --directory $HOME/.lbrycrd/ && \
|
||||
rm $HOME/blockchain_snapshot.tar.bz2
|
||||
|
||||
echo "Creating lbrycrd config"
|
||||
cat << EOF | tee "$HOME/.lbrycrd/lbrycrd.conf"
|
||||
port=9246
|
||||
rpcallowip=127.0.0.1
|
||||
rpcbind=127.0.0.1
|
||||
rpcport=9245
|
||||
rpcuser=lbry
|
||||
rpcpassword=$(cat /dev/urandom | tr -dc 'A-Za-z0-9' | head -c 24)
|
||||
server=1
|
||||
txindex=1
|
||||
maxtxfee=0.5
|
||||
dustrelayfee=0.00000001
|
||||
EOF
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
echo "Running $0"
|
||||
|
||||
echo "Starting lbrycrdd"
|
||||
$HOME/lbrycrdd -daemon -datadir="$(echo $HOME)/.lbrycrd"
|
||||
|
||||
echo "Waiting until lbrycrdd has caught up to the blockchain tip"
|
||||
HEIGHT=$(curl -s https://explorer.lbry.com/api/v1/status | egrep -o 'height":([0-9])+' | egrep -o '[0-9]+')
|
||||
while true; do
|
||||
#set +eo pipefail
|
||||
info=$($HOME/lbrycrd-cli getblockchaininfo 2>/dev/null)
|
||||
ret=$?
|
||||
if [ "$ret" == 28 ]; then echo "Waiting for lbrycrd to start"; sleep 5; continue; fi
|
||||
#set -eo pipefail
|
||||
|
||||
HEADERS=$(echo "$info" | grep headers | egrep -o '[0-9]+')
|
||||
BLOCKS=$(echo "$info" | grep blocks | egrep -o '[0-9]+')
|
||||
echo "$HEIGHT $HEADERS $BLOCKS"
|
||||
if [[ "$HEADERS" -ge "$HEIGHT" && "$BLOCKS" -ge "$HEADERS" ]]; then break; else sleep 1; fi
|
||||
done
|
||||
echo "final: $HEIGHT $HEADERS $BLOCKS"
|
||||
|
||||
echo "Stopping lbrycrdd"
|
||||
$HOME/lbrycrd-cli stop
|
||||
sleep 5 # make sure it has shut down
|
||||
|
||||
SNAPSHOT="$HOME/lbrycrd_snapshot_${BLOCKS}_$(date +%F).tar.bz2"
|
||||
echo "Making snapshot $SNAPSHOT"
|
||||
(
|
||||
cd $HOME/.lbrycrd
|
||||
tar -cjvf "$SNAPSHOT" blocks/ chainstate/ claimtrie/ indexes/
|
||||
)
|
||||
|
||||
echo "Uploading snapshot to s3"
|
||||
aws s3 cp $SNAPSHOT s3://snapshots.lbry.com/blockchain/ --region us-east-2
|
||||
|
||||
# shutdown instance (which will terminate it if shutdown behavior is set to terminate)
|
||||
# sudo poweroff
|
||||
18
ec2-userdata
18
ec2-userdata
|
|
@ -5,13 +5,19 @@
|
|||
# exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
|
||||
# see https://aws.amazon.com/premiumsupport/knowledge-center/ec2-linux-log-user-data/
|
||||
|
||||
echo "creating setup script"
|
||||
sudo apt update && \
|
||||
sudo apt install -y apt-transport-https ca-certificates curl software-properties-common unzip awscli && \
|
||||
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - && \
|
||||
sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \
|
||||
sudo apt install -y docker-ce docker-ce-cli containerd.io && \
|
||||
sudo systemctl enable docker && sudo systemctl start docker && \
|
||||
sudo curl -L "https://github.com/docker/compose/releases/download/1.24.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose && \
|
||||
sudo chmod +x /usr/local/bin/docker-compose
|
||||
|
||||
wget -O /home/ubuntu/setup.sh https://raw.githubusercontent.com/lbryio/snapshots/master/blockchain_setup.sh
|
||||
wget -O /home/ubuntu/snapshot.sh https://raw.githubusercontent.com/lbryio/snapshots/master/blockchain_snapshot.sh
|
||||
wget -O /home/ubuntu/snapshot.sh "https://raw.githubusercontent.com/lbryio/snapshots/master/snapshot.sh"
|
||||
wget -O /home/ubuntu/docker-compose.yml "https://gist.githubusercontent.com/lyoshenka/2557c08344bfe1020f0c0a13b9c5b0ce/raw/docker-compose.yml"
|
||||
|
||||
chmod +x /home/ubuntu/setup.sh /home/ubuntu/snapshot.sh
|
||||
chown ubuntu:ubuntu /home/ubuntu/setup.sh /home/ubuntu/snapshot.sh
|
||||
chown ubuntu:ubuntu /home/ubuntu/snapshot.sh /home/ubuntu/docker-compose.yml
|
||||
chmod +x /home/ubuntu/snapshot.sh
|
||||
|
||||
sudo --set-home --non-interactive --user=ubuntu /bin/bash -c 'cd /home/ubuntu; ./setup.sh'
|
||||
sudo --set-home --non-interactive --user=ubuntu /bin/bash -c 'cd /home/ubuntu; ./snapshot.sh'
|
||||
|
|
|
|||
49
snapshot.sh
Normal file
49
snapshot.sh
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
#!/bin/bash
|
||||
|
||||
# set -x
|
||||
|
||||
echo "Running $0"
|
||||
|
||||
echo "Starting servers"
|
||||
sudo docker-compose up --detach
|
||||
|
||||
echo "Waiting until lbrycrdd has caught up to the blockchain tip"
|
||||
HEIGHT=$(curl -s https://explorer.lbry.com/api/v1/status | egrep -o 'height":([0-9])+' | egrep -o '[0-9]+')
|
||||
while true; do
|
||||
#set +eo pipefail
|
||||
info=$(sudo docker exec ubuntu_lbrycrd_1 lbrycrd-cli -conf=/etc/lbry/lbrycrd.conf getblockchaininfo 2>/dev/null)
|
||||
ret=$?
|
||||
if [ "$ret" != 0 ]; then echo "Waiting for lbrycrd to start"; sleep 5; continue; fi
|
||||
WALLET_BLOCKS=$(echo '{"id":1,"method":"blockchain.block.get_server_height"}' | nc localhost 50001 | egrep -m 1 -o 'result": [0-9]+' | egrep -o '[0-9]+')
|
||||
if [ -z "$WALLET_BLOCKS" ]; then echo "Waiting for wallet server to bind port"; sleep 5; continue; fi
|
||||
#set -eo pipefail
|
||||
|
||||
HEADERS=$(echo "$info" | grep headers | egrep -o '[0-9]+')
|
||||
BLOCKS=$(echo "$info" | grep blocks | egrep -o '[0-9]+')
|
||||
echo "$HEIGHT $HEADERS $BLOCKS $WALLET_BLOCKS"
|
||||
if [[ "$HEADERS" -ge "$HEIGHT" && "$BLOCKS" -ge "$HEADERS" && "$WALLET_BLOCKS" -ge "$BLOCKS" ]]; then break; else sleep 1; fi
|
||||
done
|
||||
echo "final: $HEIGHT $HEADERS $BLOCKS $WALLET_BLOCKS"
|
||||
|
||||
echo "Stopping servers"
|
||||
sudo docker-compose down
|
||||
|
||||
# TODO: get volume data locations from `docker volume inspect` instead of hardcoding them below
|
||||
|
||||
BLOCKCHAIN_SNAPSHOT="$HOME/blockchain_snapshot_${WALLET_BLOCKS}_$(date +%F).tar.bz2"
|
||||
echo "Making blockchain snapshot $BLOCKCHAIN_SNAPSHOT"
|
||||
sudo tar -cjvf "$BLOCKCHAIN_SNAPSHOT" -C /var/lib/docker/volumes/ubuntu_lbrycrd/_data --group=$(whoami) --owner=$(whoami) blocks/ chainstate/ claimtrie/ indexes/
|
||||
echo "Uploading blockchain snapshot to s3"
|
||||
aws s3 cp "$BLOCKCHAIN_SNAPSHOT" s3://snapshots.lbry.com/blockchain/ --region us-east-2
|
||||
|
||||
WALLET_SNAPSHOT="$HOME/wallet_snapshot_${WALLET_BLOCKS}_$(date +%F).tar.bz2"
|
||||
echo "Making wallet snapshot $SNAPSHOT"
|
||||
sudo tar -cjvf "$WALLET_SNAPSHOT" -C /var/lib/docker/volumes/ubuntu_wallet_server/_data --group=$(whoami) --owner=$(whoami) claims.db hist/ meta/ utxo/
|
||||
echo "Uploading wallet snapshot to s3"
|
||||
aws s3 cp "$WALLET_SNAPSHOT" s3://snapshots.lbry.com/wallet/ --region us-east-2
|
||||
|
||||
rm -f "$BLOCKCHAIN_SNAPSHOT" "$WALLET_SNAPSHOT"
|
||||
|
||||
|
||||
# shutdown instance (which will terminate it if shutdown behavior is set to terminate)
|
||||
# sudo poweroff
|
||||
Loading…
Reference in a new issue