create an instance that does blockchain snapshots
This commit is contained in:
Alex Grintsvayg
parent
d9c851b15a
commit
df6781f309
6 changed files with 199 additions and 0 deletions
33
blockchain_setup.sh
Normal file
33
blockchain_setup.sh
Normal file
|
|
@ -0,0 +1,33 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
echo "Running $0"
|
||||||
|
|
||||||
|
echo "Installing a few tools"
|
||||||
|
sudo apt update && \
|
||||||
|
sudo apt install -y unzip awscli
|
||||||
|
|
||||||
|
echo "Installing lbrycrd"
|
||||||
|
wget -O $HOME/lbrycrd-linux.zip $(curl -s https://api.github.com/repos/lbryio/lbrycrd/releases | grep -F 'lbrycrd-linux' | grep download | head -n 1 | cut -d'"' -f4) && \
|
||||||
|
unzip $HOME/lbrycrd-linux.zip -d $HOME && \
|
||||||
|
rm $HOME/lbrycrd-linux.zip
|
||||||
|
|
||||||
|
mkdir -p "$HOME/.lbrycrd"
|
||||||
|
|
||||||
|
echo "Downloading snapshot"
|
||||||
|
wget -O $HOME/blockchain_snapshot.tar.bz2 https://lbry.com/snapshot/blockchain && \
|
||||||
|
tar xvjf $HOME/blockchain_snapshot.tar.bz2 --directory $HOME/.lbrycrd/ && \
|
||||||
|
rm $HOME/blockchain_snapshot.tar.bz2
|
||||||
|
|
||||||
|
echo "Creating lbrycrd config"
|
||||||
|
cat << EOF | tee "$HOME/.lbrycrd/lbrycrd.conf"
|
||||||
|
port=9246
|
||||||
|
rpcallowip=127.0.0.1
|
||||||
|
rpcbind=127.0.0.1
|
||||||
|
rpcport=9245
|
||||||
|
rpcuser=lbry
|
||||||
|
rpcpassword=$(cat /dev/urandom | tr -dc 'A-Za-z0-9' | head -c 24)
|
||||||
|
server=1
|
||||||
|
txindex=1
|
||||||
|
maxtxfee=0.5
|
||||||
|
dustrelayfee=0.00000001
|
||||||
|
EOF
|
||||||
39
blockchain_snapshot.sh
Normal file
39
blockchain_snapshot.sh
Normal file
|
|
@ -0,0 +1,39 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
echo "Running $0"
|
||||||
|
|
||||||
|
echo "Starting lbrycrdd"
|
||||||
|
$HOME/lbrycrdd -daemon -datadir="$(echo $HOME)/.lbrycrd"
|
||||||
|
|
||||||
|
echo "Waiting until lbrycrdd has caught up to the blockchain tip"
|
||||||
|
HEIGHT=$(curl -s https://explorer.lbry.com/api/v1/status | egrep -o 'height":([0-9])+' | egrep -o '[0-9]+')
|
||||||
|
while true; do
|
||||||
|
#set +eo pipefail
|
||||||
|
info=$($HOME/lbrycrd-cli getblockchaininfo 2>/dev/null)
|
||||||
|
ret=$?
|
||||||
|
if [ "$ret" == 28 ]; then echo "Waiting for lbrycrd to start"; sleep 5; continue; fi
|
||||||
|
#set -eo pipefail
|
||||||
|
|
||||||
|
HEADERS=$(echo "$info" | grep headers | egrep -o '[0-9]+')
|
||||||
|
BLOCKS=$(echo "$info" | grep blocks | egrep -o '[0-9]+')
|
||||||
|
echo "$HEIGHT $HEADERS $BLOCKS"
|
||||||
|
if [[ "$HEADERS" -ge "$HEIGHT" && "$BLOCKS" -ge "$HEADERS" ]]; then break; else sleep 1; fi
|
||||||
|
done
|
||||||
|
echo "final: $HEIGHT $HEADERS $BLOCKS"
|
||||||
|
|
||||||
|
echo "Stopping lbrycrdd"
|
||||||
|
$HOME/lbrycrd-cli stop
|
||||||
|
sleep 5 # make sure it has shut down
|
||||||
|
|
||||||
|
SNAPSHOT="$HOME/lbrycrd_snapshot_${BLOCKS}_$(date +%F).tar.bz2"
|
||||||
|
echo "Making snapshot $SNAPSHOT"
|
||||||
|
(
|
||||||
|
cd $HOME/.lbrycrd
|
||||||
|
tar -cjvf "$SNAPSHOT" blocks/ chainstate/ claimtrie/ indexes/
|
||||||
|
)
|
||||||
|
|
||||||
|
echo "Uploading snapshot to s3"
|
||||||
|
aws s3 cp $SNAPSHOT s3://snapshots.lbry.com/blockchain/ --region us-east-2
|
||||||
|
|
||||||
|
# shutdown instance (which will terminate it if shutdown behavior is set to terminate)
|
||||||
|
# sudo poweroff
|
||||||
12
ec2-instance-props.json
Normal file
12
ec2-instance-props.json
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
{
|
||||||
|
"BlockDeviceMappings": [{"DeviceName": "/dev/sda1", "Ebs": {"DeleteOnTermination": true, "VolumeSize": 50, "VolumeType": "gp2"}}],
|
||||||
|
"ImageId": "ami-0d5d9d301c853a04a",
|
||||||
|
"InstanceType": "t3.large",
|
||||||
|
"KeyName": "master-key-20181218",
|
||||||
|
"SecurityGroupIds": ["sg-d64cafbe"],
|
||||||
|
"SubnetId": "subnet-eef8ee96",
|
||||||
|
"UserData": "",
|
||||||
|
"InstanceInitiatedShutdownBehavior": "terminate",
|
||||||
|
"IamInstanceProfile": {"Arn": "arn:aws:iam::275835546274:instance-profile/ec2-snapshotter"},
|
||||||
|
"TagSpecifications": [{"ResourceType": "instance", "Tags": [{"Key": "Name", "Value": "snapshotter"}]}]
|
||||||
|
}
|
||||||
17
ec2-userdata
Normal file
17
ec2-userdata
Normal file
|
|
@ -0,0 +1,17 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# reminder: logs from userdata script go to /var/log/cloud-init-output.log
|
||||||
|
# you can also log script output by adding this to the top of the script:
|
||||||
|
# exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
|
||||||
|
# see https://aws.amazon.com/premiumsupport/knowledge-center/ec2-linux-log-user-data/
|
||||||
|
|
||||||
|
echo "creating setup script"
|
||||||
|
|
||||||
|
wget -O /home/ubuntu/setup.sh https://raw.githubusercontent.com/lbryio/snapshots/master/blockchain_setup.sh
|
||||||
|
wget -O /home/ubuntu/snapshot.sh https://raw.githubusercontent.com/lbryio/snapshots/master/blockchain_snapshot.sh
|
||||||
|
|
||||||
|
chmod +x /home/ubuntu/setup.sh /home/ubuntu/snapshot.sh
|
||||||
|
chown ubuntu:ubuntu /home/ubuntu/setup.sh /home/ubuntu/snapshot.sh
|
||||||
|
|
||||||
|
sudo --set-home --non-interactive --user=ubuntu /bin/bash -c 'cd /home/ubuntu; ./setup.sh'
|
||||||
|
sudo --set-home --non-interactive --user=ubuntu /bin/bash -c 'cd /home/ubuntu; ./snapshot.sh'
|
||||||
6
ec2go.sh
Executable file
6
ec2go.sh
Executable file
|
|
@ -0,0 +1,6 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||||
|
pushd $DIR
|
||||||
|
aws --profile mfa ec2 run-instances --region us-east-2 --cli-input-json file://ec2-instance-props.json --user-data file://ec2-userdata
|
||||||
|
popd
|
||||||
92
old-userdata
Normal file
92
old-userdata
Normal file
|
|
@ -0,0 +1,92 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
echo "creating setup script"
|
||||||
|
|
||||||
|
cat << 'SETUPSCRIPT' > /home/ubuntu/setup.sh
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
#exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
|
||||||
|
echo "Running setup.sh"
|
||||||
|
|
||||||
|
echo "Installing a few tools"
|
||||||
|
sudo apt update && \
|
||||||
|
sudo apt install -y unzip awscli
|
||||||
|
|
||||||
|
echo "Installing lbrycrd"
|
||||||
|
wget -O $HOME/lbrycrd-linux.zip $(curl -s https://api.github.com/repos/lbryio/lbrycrd/releases | grep -F 'lbrycrd-linux' | grep download | head -n 1 | cut -d'"' -f4) && \
|
||||||
|
unzip $HOME/lbrycrd-linux.zip -d $HOME && \
|
||||||
|
rm $HOME/lbrycrd-linux.zip
|
||||||
|
|
||||||
|
mkdir -p "$HOME/.lbrycrd"
|
||||||
|
|
||||||
|
echo "Downloading snapshot"
|
||||||
|
wget -O $HOME/blockchain_snapshot.tar.bz2 https://lbry.com/snapshot/blockchain && \
|
||||||
|
tar xvjf $HOME/blockchain_snapshot.tar.bz2 --directory $HOME/.lbrycrd/ && \
|
||||||
|
rm $HOME/blockchain_snapshot.tar.bz2
|
||||||
|
|
||||||
|
echo "Creating lbrycrd config"
|
||||||
|
cat << EOF | tee "$HOME/.lbrycrd/lbrycrd.conf"
|
||||||
|
port=9246
|
||||||
|
rpcallowip=127.0.0.1
|
||||||
|
rpcbind=127.0.0.1
|
||||||
|
rpcport=9245
|
||||||
|
rpcuser=lbry
|
||||||
|
rpcpassword=$(cat /dev/urandom | tr -dc 'A-Za-z0-9' | head -c 24)
|
||||||
|
server=1
|
||||||
|
txindex=1
|
||||||
|
maxtxfee=0.5
|
||||||
|
dustrelayfee=0.00000001
|
||||||
|
EOF
|
||||||
|
SETUPSCRIPT
|
||||||
|
|
||||||
|
|
||||||
|
echo "creating snapshot script"
|
||||||
|
|
||||||
|
cat << 'SNAPSHOTSCRIPT' > /home/ubuntu/snapshot.sh
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
#exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
|
||||||
|
echo "Running snapshot.sh"
|
||||||
|
|
||||||
|
echo "Starting lbrycrdd"
|
||||||
|
$HOME/lbrycrdd -daemon -datadir="$(echo $HOME)/.lbrycrd"
|
||||||
|
|
||||||
|
echo "Waiting until lbrycrdd has caught up to the blockchain tip"
|
||||||
|
HEIGHT=$(curl -s https://explorer.lbry.com/api/v1/status | egrep -o 'height":([0-9])+' | egrep -o '[0-9]+')
|
||||||
|
while true; do
|
||||||
|
#set +eo pipefail
|
||||||
|
info=$($HOME/lbrycrd-cli getblockchaininfo 2>/dev/null)
|
||||||
|
ret=$?
|
||||||
|
if [ "$ret" == 28 ]; then echo "Waiting for lbrycrd to start"; sleep 5; continue; fi
|
||||||
|
#set -eo pipefail
|
||||||
|
|
||||||
|
HEADERS=$(echo "$info" | grep headers | egrep -o '[0-9]+')
|
||||||
|
BLOCKS=$(echo "$info" | grep blocks | egrep -o '[0-9]+')
|
||||||
|
echo "$HEIGHT $HEADERS $BLOCKS"
|
||||||
|
if [[ "$HEADERS" -ge "$HEIGHT" && "$BLOCKS" -ge "$HEADERS" ]]; then break; else sleep 1; fi
|
||||||
|
done
|
||||||
|
echo "final: $HEIGHT $HEADERS $BLOCKS"
|
||||||
|
|
||||||
|
echo "Stopping lbrycrdd"
|
||||||
|
$HOME/lbrycrd-cli stop
|
||||||
|
sleep 5 # make sure it has shut down
|
||||||
|
|
||||||
|
echo "Making snapshot $HOME/$SNAPSHOT"
|
||||||
|
SNAPSHOT="$HOME/lbrycrd_snapshot_${BLOCKS}_$(date +%F).tar.bz2"
|
||||||
|
(
|
||||||
|
cd $HOME/.lbrycrd
|
||||||
|
tar -cjvf "$SNAPSHOT" blocks/ chainstate/ claimtrie/ indexes/
|
||||||
|
)
|
||||||
|
|
||||||
|
echo "Uploading snapshot"
|
||||||
|
aws s3 cp $SNAPSHOT s3://snapshots.lbry.com/blockchain/ --region us-east-2
|
||||||
|
|
||||||
|
# shutdown instance (which will terminate it if shutdown behavior is set to terminate)
|
||||||
|
# sudo poweroff
|
||||||
|
SNAPSHOTSCRIPT
|
||||||
|
|
||||||
|
|
||||||
|
chmod +x /home/ubuntu/setup.sh /home/ubuntu/snapshot.sh
|
||||||
|
chown ubuntu:ubuntu /home/ubuntu/setup.sh /home/ubuntu/snapshot.sh
|
||||||
|
sudo --set-home --non-interactive --user=ubuntu /bin/bash -c 'cd /home/ubuntu; ./setup.sh'
|
||||||
|
#sudo -H -u ubuntu /home/ubuntu/snapshot.sh
|
||||||
Loading…
Reference in a new issue