create an instance that does blockchain snapshots

2019-11-01 12:44:30 -04:00 · 2019-11-01 12:44:30 -04:00 · df6781f309
parent d9c851b15a
commit df6781f309
6 changed files with 199 additions and 0 deletions
--- a/blockchain_setup.sh
+++ b/blockchain_setup.sh
@ -0,0 +1,33 @@
+#!/bin/bash
+
+echo "Running $0"
+
+echo "Installing a few tools"
+sudo apt update && \
+sudo apt install -y unzip awscli
+
+echo "Installing lbrycrd"
+wget -O $HOME/lbrycrd-linux.zip $(curl -s https://api.github.com/repos/lbryio/lbrycrd/releases | grep -F 'lbrycrd-linux' | grep download | head -n 1 | cut -d'"' -f4) && \
+unzip $HOME/lbrycrd-linux.zip -d $HOME && \
+rm $HOME/lbrycrd-linux.zip
+
+mkdir -p "$HOME/.lbrycrd"
+
+echo "Downloading snapshot"
+wget -O $HOME/blockchain_snapshot.tar.bz2 https://lbry.com/snapshot/blockchain && \
+tar xvjf $HOME/blockchain_snapshot.tar.bz2 --directory $HOME/.lbrycrd/ && \
+rm $HOME/blockchain_snapshot.tar.bz2
+
+echo "Creating lbrycrd config"
+cat << EOF | tee "$HOME/.lbrycrd/lbrycrd.conf"
+port=9246
+rpcallowip=127.0.0.1
+rpcbind=127.0.0.1
+rpcport=9245
+rpcuser=lbry
+rpcpassword=$(cat /dev/urandom | tr -dc 'A-Za-z0-9' | head -c 24)
+server=1
+txindex=1
+maxtxfee=0.5
+dustrelayfee=0.00000001
+EOF
--- a/blockchain_snapshot.sh
+++ b/blockchain_snapshot.sh
@ -0,0 +1,39 @@
+#!/bin/bash
+
+echo "Running $0"
+
+echo "Starting lbrycrdd"
+$HOME/lbrycrdd -daemon -datadir="$(echo $HOME)/.lbrycrd"
+
+echo "Waiting until lbrycrdd has caught up to the blockchain tip"
+HEIGHT=$(curl -s https://explorer.lbry.com/api/v1/status | egrep -o 'height":([0-9])+' | egrep -o '[0-9]+')
+while true; do
+  #set +eo pipefail
+  info=$($HOME/lbrycrd-cli getblockchaininfo 2>/dev/null)
+  ret=$?
+  if [ "$ret" == 28 ]; then echo "Waiting for lbrycrd to start"; sleep 5; continue; fi
+  #set -eo pipefail
+
+  HEADERS=$(echo "$info" | grep headers | egrep -o '[0-9]+')
+  BLOCKS=$(echo "$info" | grep blocks | egrep -o '[0-9]+')
+  echo "$HEIGHT $HEADERS $BLOCKS"
+  if [[ "$HEADERS" -ge "$HEIGHT" && "$BLOCKS" -ge "$HEADERS" ]]; then break; else sleep 1; fi
+done
+echo "final: $HEIGHT $HEADERS $BLOCKS"
+
+echo "Stopping lbrycrdd"
+$HOME/lbrycrd-cli stop
+sleep 5 # make sure it has shut down
+
+SNAPSHOT="$HOME/lbrycrd_snapshot_${BLOCKS}_$(date +%F).tar.bz2"
+echo "Making snapshot $SNAPSHOT"
+(
+  cd $HOME/.lbrycrd
+  tar -cjvf "$SNAPSHOT" blocks/ chainstate/ claimtrie/ indexes/
+)
+
+echo "Uploading snapshot to s3"
+aws s3 cp $SNAPSHOT s3://snapshots.lbry.com/blockchain/ --region us-east-2
+
+# shutdown instance (which will terminate it if shutdown behavior is set to terminate)
+# sudo poweroff
--- a/ec2-instance-props.json
+++ b/ec2-instance-props.json
@ -0,0 +1,12 @@
+{
+    "BlockDeviceMappings": [{"DeviceName": "/dev/sda1", "Ebs": {"DeleteOnTermination": true, "VolumeSize": 50, "VolumeType": "gp2"}}],
+    "ImageId": "ami-0d5d9d301c853a04a",
+    "InstanceType": "t3.large",
+    "KeyName": "master-key-20181218",
+    "SecurityGroupIds": ["sg-d64cafbe"],
+    "SubnetId": "subnet-eef8ee96",
+    "UserData": "",
+    "InstanceInitiatedShutdownBehavior": "terminate",
+    "IamInstanceProfile": {"Arn": "arn:aws:iam::275835546274:instance-profile/ec2-snapshotter"},
+    "TagSpecifications": [{"ResourceType": "instance", "Tags": [{"Key": "Name", "Value": "snapshotter"}]}]
+}
--- a/17
+++ b/17
@ -0,0 +1,17 @@
+#!/bin/bash
+
+# reminder: logs from userdata script go to /var/log/cloud-init-output.log
+# you can also log script output by adding this to the top of the script:
+#   exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
+# see https://aws.amazon.com/premiumsupport/knowledge-center/ec2-linux-log-user-data/
+
+echo "creating setup script"
+
+wget -O /home/ubuntu/setup.sh https://raw.githubusercontent.com/lbryio/snapshots/master/blockchain_setup.sh
+wget -O /home/ubuntu/snapshot.sh https://raw.githubusercontent.com/lbryio/snapshots/master/blockchain_snapshot.sh
+
+chmod +x /home/ubuntu/setup.sh /home/ubuntu/snapshot.sh
+chown ubuntu:ubuntu /home/ubuntu/setup.sh /home/ubuntu/snapshot.sh
+
+sudo --set-home --non-interactive --user=ubuntu /bin/bash -c 'cd /home/ubuntu; ./setup.sh'
+sudo --set-home --non-interactive --user=ubuntu /bin/bash -c 'cd /home/ubuntu; ./snapshot.sh'
--- a/ec2go.sh
+++ b/ec2go.sh
@ -0,0 +1,6 @@
+#!/bin/bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+pushd $DIR
+aws --profile mfa ec2 run-instances --region us-east-2 --cli-input-json file://ec2-instance-props.json --user-data file://ec2-userdata
+popd
--- a/92
+++ b/92
@ -0,0 +1,92 @@
+#!/bin/bash
+
+echo "creating setup script"
+
+cat << 'SETUPSCRIPT' > /home/ubuntu/setup.sh
+#!/bin/bash
+
+#exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
+echo "Running setup.sh"
+
+echo "Installing a few tools"
+sudo apt update && \
+sudo apt install -y unzip awscli
+
+echo "Installing lbrycrd"
+wget -O $HOME/lbrycrd-linux.zip $(curl -s https://api.github.com/repos/lbryio/lbrycrd/releases | grep -F 'lbrycrd-linux' | grep download | head -n 1 | cut -d'"' -f4) && \
+unzip $HOME/lbrycrd-linux.zip -d $HOME && \
+rm $HOME/lbrycrd-linux.zip
+
+mkdir -p "$HOME/.lbrycrd"
+
+echo "Downloading snapshot"
+wget -O $HOME/blockchain_snapshot.tar.bz2 https://lbry.com/snapshot/blockchain && \
+tar xvjf $HOME/blockchain_snapshot.tar.bz2 --directory $HOME/.lbrycrd/ && \
+rm $HOME/blockchain_snapshot.tar.bz2
+
+echo "Creating lbrycrd config"
+cat << EOF | tee "$HOME/.lbrycrd/lbrycrd.conf"
+port=9246
+rpcallowip=127.0.0.1
+rpcbind=127.0.0.1
+rpcport=9245
+rpcuser=lbry
+rpcpassword=$(cat /dev/urandom | tr -dc 'A-Za-z0-9' | head -c 24)
+server=1
+txindex=1
+maxtxfee=0.5
+dustrelayfee=0.00000001
+EOF
+SETUPSCRIPT
+
+
+echo "creating snapshot script"
+
+cat << 'SNAPSHOTSCRIPT' > /home/ubuntu/snapshot.sh
+#!/bin/bash
+
+#exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
+echo "Running snapshot.sh"
+
+echo "Starting lbrycrdd"
+$HOME/lbrycrdd -daemon -datadir="$(echo $HOME)/.lbrycrd"
+
+echo "Waiting until lbrycrdd has caught up to the blockchain tip"
+HEIGHT=$(curl -s https://explorer.lbry.com/api/v1/status | egrep -o 'height":([0-9])+' | egrep -o '[0-9]+')
+while true; do
+  #set +eo pipefail
+  info=$($HOME/lbrycrd-cli getblockchaininfo 2>/dev/null)
+  ret=$?
+  if [ "$ret" == 28 ]; then echo "Waiting for lbrycrd to start"; sleep 5; continue; fi
+  #set -eo pipefail
+
+  HEADERS=$(echo "$info" | grep headers | egrep -o '[0-9]+')
+  BLOCKS=$(echo "$info" | grep blocks | egrep -o '[0-9]+')
+  echo "$HEIGHT $HEADERS $BLOCKS"
+  if [[ "$HEADERS" -ge "$HEIGHT" && "$BLOCKS" -ge "$HEADERS" ]]; then break; else sleep 1; fi
+done
+echo "final: $HEIGHT $HEADERS $BLOCKS"
+
+echo "Stopping lbrycrdd"
+$HOME/lbrycrd-cli stop
+sleep 5 # make sure it has shut down
+
+echo "Making snapshot $HOME/$SNAPSHOT"
+SNAPSHOT="$HOME/lbrycrd_snapshot_${BLOCKS}_$(date +%F).tar.bz2"
+(
+  cd $HOME/.lbrycrd
+  tar -cjvf "$SNAPSHOT" blocks/ chainstate/ claimtrie/ indexes/
+)
+
+echo "Uploading snapshot"
+aws s3 cp $SNAPSHOT s3://snapshots.lbry.com/blockchain/ --region us-east-2
+
+# shutdown instance (which will terminate it if shutdown behavior is set to terminate)
+# sudo poweroff
+SNAPSHOTSCRIPT
+
+
+chmod +x /home/ubuntu/setup.sh /home/ubuntu/snapshot.sh
+chown ubuntu:ubuntu /home/ubuntu/setup.sh /home/ubuntu/snapshot.sh
+sudo --set-home --non-interactive --user=ubuntu /bin/bash -c 'cd /home/ubuntu; ./setup.sh'
+#sudo -H -u ubuntu /home/ubuntu/snapshot.sh