create an instance that does blockchain snapshots
This commit is contained in:
Alex Grintsvayg
parent
d9c851b15a
commit
df6781f309
6 changed files with 199 additions and 0 deletions
33
blockchain_setup.sh
Normal file
33
blockchain_setup.sh
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
#!/bin/bash
|
||||
|
||||
echo "Running $0"
|
||||
|
||||
echo "Installing a few tools"
|
||||
sudo apt update && \
|
||||
sudo apt install -y unzip awscli
|
||||
|
||||
echo "Installing lbrycrd"
|
||||
wget -O $HOME/lbrycrd-linux.zip $(curl -s https://api.github.com/repos/lbryio/lbrycrd/releases | grep -F 'lbrycrd-linux' | grep download | head -n 1 | cut -d'"' -f4) && \
|
||||
unzip $HOME/lbrycrd-linux.zip -d $HOME && \
|
||||
rm $HOME/lbrycrd-linux.zip
|
||||
|
||||
mkdir -p "$HOME/.lbrycrd"
|
||||
|
||||
echo "Downloading snapshot"
|
||||
wget -O $HOME/blockchain_snapshot.tar.bz2 https://lbry.com/snapshot/blockchain && \
|
||||
tar xvjf $HOME/blockchain_snapshot.tar.bz2 --directory $HOME/.lbrycrd/ && \
|
||||
rm $HOME/blockchain_snapshot.tar.bz2
|
||||
|
||||
echo "Creating lbrycrd config"
|
||||
cat << EOF | tee "$HOME/.lbrycrd/lbrycrd.conf"
|
||||
port=9246
|
||||
rpcallowip=127.0.0.1
|
||||
rpcbind=127.0.0.1
|
||||
rpcport=9245
|
||||
rpcuser=lbry
|
||||
rpcpassword=$(cat /dev/urandom | tr -dc 'A-Za-z0-9' | head -c 24)
|
||||
server=1
|
||||
txindex=1
|
||||
maxtxfee=0.5
|
||||
dustrelayfee=0.00000001
|
||||
EOF
|
||||
39
blockchain_snapshot.sh
Normal file
39
blockchain_snapshot.sh
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
#!/bin/bash
|
||||
|
||||
echo "Running $0"
|
||||
|
||||
echo "Starting lbrycrdd"
|
||||
$HOME/lbrycrdd -daemon -datadir="$(echo $HOME)/.lbrycrd"
|
||||
|
||||
echo "Waiting until lbrycrdd has caught up to the blockchain tip"
|
||||
HEIGHT=$(curl -s https://explorer.lbry.com/api/v1/status | egrep -o 'height":([0-9])+' | egrep -o '[0-9]+')
|
||||
while true; do
|
||||
#set +eo pipefail
|
||||
info=$($HOME/lbrycrd-cli getblockchaininfo 2>/dev/null)
|
||||
ret=$?
|
||||
if [ "$ret" == 28 ]; then echo "Waiting for lbrycrd to start"; sleep 5; continue; fi
|
||||
#set -eo pipefail
|
||||
|
||||
HEADERS=$(echo "$info" | grep headers | egrep -o '[0-9]+')
|
||||
BLOCKS=$(echo "$info" | grep blocks | egrep -o '[0-9]+')
|
||||
echo "$HEIGHT $HEADERS $BLOCKS"
|
||||
if [[ "$HEADERS" -ge "$HEIGHT" && "$BLOCKS" -ge "$HEADERS" ]]; then break; else sleep 1; fi
|
||||
done
|
||||
echo "final: $HEIGHT $HEADERS $BLOCKS"
|
||||
|
||||
echo "Stopping lbrycrdd"
|
||||
$HOME/lbrycrd-cli stop
|
||||
sleep 5 # make sure it has shut down
|
||||
|
||||
SNAPSHOT="$HOME/lbrycrd_snapshot_${BLOCKS}_$(date +%F).tar.bz2"
|
||||
echo "Making snapshot $SNAPSHOT"
|
||||
(
|
||||
cd $HOME/.lbrycrd
|
||||
tar -cjvf "$SNAPSHOT" blocks/ chainstate/ claimtrie/ indexes/
|
||||
)
|
||||
|
||||
echo "Uploading snapshot to s3"
|
||||
aws s3 cp $SNAPSHOT s3://snapshots.lbry.com/blockchain/ --region us-east-2
|
||||
|
||||
# shutdown instance (which will terminate it if shutdown behavior is set to terminate)
|
||||
# sudo poweroff
|
||||
12
ec2-instance-props.json
Normal file
12
ec2-instance-props.json
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
{
|
||||
"BlockDeviceMappings": [{"DeviceName": "/dev/sda1", "Ebs": {"DeleteOnTermination": true, "VolumeSize": 50, "VolumeType": "gp2"}}],
|
||||
"ImageId": "ami-0d5d9d301c853a04a",
|
||||
"InstanceType": "t3.large",
|
||||
"KeyName": "master-key-20181218",
|
||||
"SecurityGroupIds": ["sg-d64cafbe"],
|
||||
"SubnetId": "subnet-eef8ee96",
|
||||
"UserData": "",
|
||||
"InstanceInitiatedShutdownBehavior": "terminate",
|
||||
"IamInstanceProfile": {"Arn": "arn:aws:iam::275835546274:instance-profile/ec2-snapshotter"},
|
||||
"TagSpecifications": [{"ResourceType": "instance", "Tags": [{"Key": "Name", "Value": "snapshotter"}]}]
|
||||
}
|
||||
17
ec2-userdata
Normal file
17
ec2-userdata
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
#!/bin/bash
|
||||
|
||||
# reminder: logs from userdata script go to /var/log/cloud-init-output.log
|
||||
# you can also log script output by adding this to the top of the script:
|
||||
# exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
|
||||
# see https://aws.amazon.com/premiumsupport/knowledge-center/ec2-linux-log-user-data/
|
||||
|
||||
echo "creating setup script"
|
||||
|
||||
wget -O /home/ubuntu/setup.sh https://raw.githubusercontent.com/lbryio/snapshots/master/blockchain_setup.sh
|
||||
wget -O /home/ubuntu/snapshot.sh https://raw.githubusercontent.com/lbryio/snapshots/master/blockchain_snapshot.sh
|
||||
|
||||
chmod +x /home/ubuntu/setup.sh /home/ubuntu/snapshot.sh
|
||||
chown ubuntu:ubuntu /home/ubuntu/setup.sh /home/ubuntu/snapshot.sh
|
||||
|
||||
sudo --set-home --non-interactive --user=ubuntu /bin/bash -c 'cd /home/ubuntu; ./setup.sh'
|
||||
sudo --set-home --non-interactive --user=ubuntu /bin/bash -c 'cd /home/ubuntu; ./snapshot.sh'
|
||||
6
ec2go.sh
Executable file
6
ec2go.sh
Executable file
|
|
@ -0,0 +1,6 @@
|
|||
#!/bin/bash
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
pushd $DIR
|
||||
aws --profile mfa ec2 run-instances --region us-east-2 --cli-input-json file://ec2-instance-props.json --user-data file://ec2-userdata
|
||||
popd
|
||||
92
old-userdata
Normal file
92
old-userdata
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
#!/bin/bash
|
||||
|
||||
echo "creating setup script"
|
||||
|
||||
cat << 'SETUPSCRIPT' > /home/ubuntu/setup.sh
|
||||
#!/bin/bash
|
||||
|
||||
#exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
|
||||
echo "Running setup.sh"
|
||||
|
||||
echo "Installing a few tools"
|
||||
sudo apt update && \
|
||||
sudo apt install -y unzip awscli
|
||||
|
||||
echo "Installing lbrycrd"
|
||||
wget -O $HOME/lbrycrd-linux.zip $(curl -s https://api.github.com/repos/lbryio/lbrycrd/releases | grep -F 'lbrycrd-linux' | grep download | head -n 1 | cut -d'"' -f4) && \
|
||||
unzip $HOME/lbrycrd-linux.zip -d $HOME && \
|
||||
rm $HOME/lbrycrd-linux.zip
|
||||
|
||||
mkdir -p "$HOME/.lbrycrd"
|
||||
|
||||
echo "Downloading snapshot"
|
||||
wget -O $HOME/blockchain_snapshot.tar.bz2 https://lbry.com/snapshot/blockchain && \
|
||||
tar xvjf $HOME/blockchain_snapshot.tar.bz2 --directory $HOME/.lbrycrd/ && \
|
||||
rm $HOME/blockchain_snapshot.tar.bz2
|
||||
|
||||
echo "Creating lbrycrd config"
|
||||
cat << EOF | tee "$HOME/.lbrycrd/lbrycrd.conf"
|
||||
port=9246
|
||||
rpcallowip=127.0.0.1
|
||||
rpcbind=127.0.0.1
|
||||
rpcport=9245
|
||||
rpcuser=lbry
|
||||
rpcpassword=$(cat /dev/urandom | tr -dc 'A-Za-z0-9' | head -c 24)
|
||||
server=1
|
||||
txindex=1
|
||||
maxtxfee=0.5
|
||||
dustrelayfee=0.00000001
|
||||
EOF
|
||||
SETUPSCRIPT
|
||||
|
||||
|
||||
echo "creating snapshot script"
|
||||
|
||||
cat << 'SNAPSHOTSCRIPT' > /home/ubuntu/snapshot.sh
|
||||
#!/bin/bash
|
||||
|
||||
#exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
|
||||
echo "Running snapshot.sh"
|
||||
|
||||
echo "Starting lbrycrdd"
|
||||
$HOME/lbrycrdd -daemon -datadir="$(echo $HOME)/.lbrycrd"
|
||||
|
||||
echo "Waiting until lbrycrdd has caught up to the blockchain tip"
|
||||
HEIGHT=$(curl -s https://explorer.lbry.com/api/v1/status | egrep -o 'height":([0-9])+' | egrep -o '[0-9]+')
|
||||
while true; do
|
||||
#set +eo pipefail
|
||||
info=$($HOME/lbrycrd-cli getblockchaininfo 2>/dev/null)
|
||||
ret=$?
|
||||
if [ "$ret" == 28 ]; then echo "Waiting for lbrycrd to start"; sleep 5; continue; fi
|
||||
#set -eo pipefail
|
||||
|
||||
HEADERS=$(echo "$info" | grep headers | egrep -o '[0-9]+')
|
||||
BLOCKS=$(echo "$info" | grep blocks | egrep -o '[0-9]+')
|
||||
echo "$HEIGHT $HEADERS $BLOCKS"
|
||||
if [[ "$HEADERS" -ge "$HEIGHT" && "$BLOCKS" -ge "$HEADERS" ]]; then break; else sleep 1; fi
|
||||
done
|
||||
echo "final: $HEIGHT $HEADERS $BLOCKS"
|
||||
|
||||
echo "Stopping lbrycrdd"
|
||||
$HOME/lbrycrd-cli stop
|
||||
sleep 5 # make sure it has shut down
|
||||
|
||||
echo "Making snapshot $HOME/$SNAPSHOT"
|
||||
SNAPSHOT="$HOME/lbrycrd_snapshot_${BLOCKS}_$(date +%F).tar.bz2"
|
||||
(
|
||||
cd $HOME/.lbrycrd
|
||||
tar -cjvf "$SNAPSHOT" blocks/ chainstate/ claimtrie/ indexes/
|
||||
)
|
||||
|
||||
echo "Uploading snapshot"
|
||||
aws s3 cp $SNAPSHOT s3://snapshots.lbry.com/blockchain/ --region us-east-2
|
||||
|
||||
# shutdown instance (which will terminate it if shutdown behavior is set to terminate)
|
||||
# sudo poweroff
|
||||
SNAPSHOTSCRIPT
|
||||
|
||||
|
||||
chmod +x /home/ubuntu/setup.sh /home/ubuntu/snapshot.sh
|
||||
chown ubuntu:ubuntu /home/ubuntu/setup.sh /home/ubuntu/snapshot.sh
|
||||
sudo --set-home --non-interactive --user=ubuntu /bin/bash -c 'cd /home/ubuntu; ./setup.sh'
|
||||
#sudo -H -u ubuntu /home/ubuntu/snapshot.sh
|
||||
Loading…
Reference in a new issue