Serialize effective amount #231

Closed
bvbfan wants to merge 309 commits from serialize_effective_amount into master
3 changed files with 312 additions and 146 deletions
Showing only changes of commit 783f5fd7cd - Show all commits

View file

@ -1,10 +1,31 @@
sudo: required matrix:
dist: trusty include:
- os: linux
sudo: required
dist: trusty
language: c
- os: osx
language: c
osx_image: xcode7.3
cache:
ccache: true
directories:
- build
before_install:
- date +%s > "${TRAVIS_BUILD_DIR}/start_time"
- ls -lh build
- du -h -d 2 build
install: true install: true
script: script:
- ./reproducible_build.sh -t - ./reproducible_build.sh -t -o -c
- zip -j lbrycrd-linux.zip src/lbrycrdd src/lbrycrd-cli src/lbrycrd-tx - if [[ "${TRAVIS_OS_NAME}" = "linux" ]]; then zip -j lbrycrd-linux.zip src/lbrycrdd src/lbrycrd-cli src/lbrycrd-tx; fi
before_cache:
- ls -lh build
- du -h -d 2 build
deploy: deploy:
- provider: releases - provider: releases
@ -12,5 +33,7 @@ deploy:
skip_cleanup: true skip_cleanup: true
on: on:
tags: true tags: true
condition: "$TRAVIS_OS_NAME = linux"
api_key: api_key:
secure: "ibN+PQg84f0tgJLV3KaHjkHmgFNOPqBLIrSU42moen22xxT2VIn0j7DkFaLLts1fs/+/gxmrvhOQ0vUFJqmflZ3kCeyedP/CVgZz7D7HRVK0cYUMJ2F9VWMZAFIX6A67OUlypqZqzKAKrZHx3HXky6Z8NBU23JgWqDm8PcrbbgMLvB2lvWVASjJVKreaL9BOxFOOOrAj6xAIm9ogTOnFVI/AhufNOKOTZaiprcbqMngwHeaILbwEJVqv5P2YUurC6Sq+QaH26tX00DtsYOW+n8AA3Fe48+L6rqa6Q8ru+dhPuR/Apr2+DTZ+npzY8leV3A7mYUeLo4JLaEH6n1TLVt65cX7nuesS7KsSKBSxs7q7bsqeapDcMskZz5JR4EK4S96CuHZEnn6+efpH8IrxWpbeO3EibJfyT8tMJKw7Zym5mPr+sz6xSlVMoBVcCm+7z9e7Zcqukdvdy8jM7sGP9qusqEhpojh0o+gPgr9p13SfR+6OCpj9gwxPCnZO8RBfh1cbz9vFSVlDyUYcpI2CCCpPazA1glTsEDk+VtznnUMmWksXWfxBFV81jQB2SnoZKlnsiEuNMwhiMCMEkRFo74+AsUr2UjmKJ5KC8ACwsn475PoLZBh2J6fyXBHMdbrGfK+Zt5K2IHRRtDaZVT4eHqYzxh8KhalsuBNxCo3FygI=" secure: "ibN+PQg84f0tgJLV3KaHjkHmgFNOPqBLIrSU42moen22xxT2VIn0j7DkFaLLts1fs/+/gxmrvhOQ0vUFJqmflZ3kCeyedP/CVgZz7D7HRVK0cYUMJ2F9VWMZAFIX6A67OUlypqZqzKAKrZHx3HXky6Z8NBU23JgWqDm8PcrbbgMLvB2lvWVASjJVKreaL9BOxFOOOrAj6xAIm9ogTOnFVI/AhufNOKOTZaiprcbqMngwHeaILbwEJVqv5P2YUurC6Sq+QaH26tX00DtsYOW+n8AA3Fe48+L6rqa6Q8ru+dhPuR/Apr2+DTZ+npzY8leV3A7mYUeLo4JLaEH6n1TLVt65cX7nuesS7KsSKBSxs7q7bsqeapDcMskZz5JR4EK4S96CuHZEnn6+efpH8IrxWpbeO3EibJfyT8tMJKw7Zym5mPr+sz6xSlVMoBVcCm+7z9e7Zcqukdvdy8jM7sGP9qusqEhpojh0o+gPgr9p13SfR+6OCpj9gwxPCnZO8RBfh1cbz9vFSVlDyUYcpI2CCCpPazA1glTsEDk+VtznnUMmWksXWfxBFV81jQB2SnoZKlnsiEuNMwhiMCMEkRFo74+AsUr2UjmKJ5KC8ACwsn475PoLZBh2J6fyXBHMdbrGfK+Zt5K2IHRRtDaZVT4eHqYzxh8KhalsuBNxCo3FygI="

View file

@ -3,26 +3,50 @@
set -euo pipefail set -euo pipefail
function HELP { function HELP {
echo "Build a lbrycrd" echo "Build lbrycrd"
echo "-----" echo "-----"
echo "When run without any arguments, this script expects the current directory" echo "When run without any arguments, this script expects the current directory"
echo "to be the lbrycrd repo and it builds what is in that directory" echo "to be the lbrycrd repo and it builds what is in that directory"
echo echo
echo "This is a long build process so it can be split into two parts"
echo "Specify the -d flag to build only the dependencies"
echo "and the -l flag to build only lbrycrd. This will fail"
echo "if the dependencies weren't built earlier"
echo
echo "Optional arguments:" echo "Optional arguments:"
echo echo
echo "-c: clone a fresh copy of the repo" echo "-c: don't clone a fresh copy of the repo"
echo "-h: show help" echo "-r: remove intermediate files."
echo "-l: build only lbrycrd"
echo "-d: build only the dependencies"
echo "-o: timeout build after 45 minutes"
echo "-t: turn trace on" echo "-t: turn trace on"
echo "-h: show help"
exit 1 exit 1
} }
CLONE=false CLONE=true
CLEAN=false
BUILD_DEPENDENCIES=true
BUILD_LBRYCRD=true
TIMEOUT=false
while getopts :crldoth:w:d: FLAG; do
while getopts :hctb:w:d: FLAG; do
case $FLAG in case $FLAG in
c) c)
CLONE=true CLONE=false
;;
r)
CLEAN=true
;;
l)
BUILD_DEPENDENCIES=false
;;
d)
BUILD_LBRYCRD=false
;;
o)
TIMEOUT=true
;; ;;
t) t)
set -o xtrace set -o xtrace
@ -44,86 +68,277 @@ done
shift $((OPTIND-1)) shift $((OPTIND-1))
SUDO='' SUDO=''
if (( $EUID != 0 )); then if (( EUID != 0 )); then
SUDO='sudo' SUDO='sudo'
fi fi
if [ "$CLONE" = false ]; then if [ "${CLONE}" = false ]; then
if [ `basename $PWD` != "lbrycrd" ]; then if [ "$(basename "$PWD")" != "lbrycrd" ]; then
echo "Not currently in the lbrycrd directory. Cowardly refusing to go forward" echo "Not currently in the lbrycrd directory. Cowardly refusing to go forward"
exit 1 exit 1
fi fi
SOURCE_DIR=$PWD SOURCE_DIR=$PWD
fi fi
if [ -z ${TRAVIS+x} ]; then if [ -z "${TRAVIS_OS_NAME+x}" ]; then
# if not on travis, its nice to see progress if [ "$(uname -s)" = "Darwin" ]; then
QUIET="" OS_NAME="osx"
else
OS_NAME="linux"
fi
else else
QUIET="-qq" OS_NAME="${TRAVIS_OS_NAME}"
fi fi
# get the required OS packages if [ -z "${TRAVIS_BUILD_DIR+x}" ]; then
$SUDO apt-get ${QUIET} update START_TIME_FILE="$PWD/start_time"
$SUDO apt-get ${QUIET} install -y --no-install-recommends \ else
build-essential python-dev libbz2-dev libtool \ # if we are on travis (the primary use case for setting a timeout)
autotools-dev autoconf git pkg-config wget \ # this file is created when the build starts
ca-certificates automake bsdmainutils START_TIME_FILE="$TRAVIS_BUILD_DIR/start_time"
fi
mkdir dependencies if [ ! -f "${START_TIME_FILE}" ]; then
cd dependencies date +%s > "${START_TIME_FILE}"
fi
wget http://download.oracle.com/berkeley-db/db-4.8.30.NC.tar.gz
tar xf db-4.8.30.NC.tar.gz
export BDB_PREFIX="`pwd`/bdb"
cd db-4.8.30.NC/build_unix
../dist/configure --prefix=$BDB_PREFIX --enable-cxx --disable-shared --with-pic
make
make install
cd ../../
wget https://www.openssl.org/source/openssl-1.0.1p.tar.gz
tar xf openssl-1.0.1p.tar.gz
export OPENSSL_PREFIX="`pwd`/openssl_build"
mkdir $OPENSSL_PREFIX
cd openssl-1.0.1p
./Configure --prefix=$OPENSSL_PREFIX --openssldir=$OPENSSL_PREFIX/ssl linux-x86_64 -fPIC -static no-shared no-dso
make
make install
cd ..
wget http://sourceforge.net/projects/boost/files/boost/1.59.0/boost_1_59_0.tar.bz2/download -O boost_1_59_0.tar.bz2 NEXT_TIME=60
tar xf boost_1_59_0.tar.bz2 function exit_at_45() {
export BOOST_ROOT="`pwd`/boost_1_59_0" if [ -f "${START_TIME_FILE}" ]; then
cd boost_1_59_0 NOW=$(date +%s)
./bootstrap.sh START=$(cat "${START_TIME_FILE}")
./b2 link=static cxxflags=-fPIC stage TIMEOUT_SECS=2700 # 45 * 60
cd ../../ TIME=$((NOW - START))
if (( TIME > NEXT_TIME )); then
echo "Build has taken $((TIME / 60)) minutes: $1"
NEXT_TIME=$((TIME + 60))
fi
if [ "$TIMEOUT" = true ] && (( TIME > TIMEOUT_SECS )); then
echo 'Exiting at 45 minutes to allow the cache to populate'
exit 1
fi
fi
}
# two arguments
# - pid (probably from $!)
# - echo message
function wait_and_echo() {
PID=$1
(set -o | grep xtrace | grep -q on)
TRACE_STATUS=$?
# disable xtrace or else this will get verbose, which is what
# I'm trying to avoid by going through all of this nonsense anyway
set +o xtrace
TIME=0
SLEEP=5
# loop until the process is no longer running
# check every $SLEEP seconds, echoing a message every minute
while (ps -p "${PID}" > /dev/null); do
exit_at_45 "$2"
sleep "${SLEEP}"
done
# restore the xtrace setting
if [ "${TRACE_STATUS}" -eq 0 ]; then
set -o xtrace
fi
wait "$PID"
return $?
}
mkdir libevent_build # run a command ($1) in the background
git clone https://github.com/libevent/libevent.git # logging its stdout and stderr to $2
export LIBEVENT_PREFIX="`pwd`/libevent_build" # and wait until it completed
cd libevent function background() {
./autogen.sh $1 >> "$2" 2>&1 &
./configure --prefix=$LIBEVENT_PREFIX --enable-static --disable-shared --with-pic LDFLAGS="-L${OPENSSL_PREFIX}/lib/" CPPFLAGS="-I${OPENSSL_PREFIX}/include" wait_and_echo $! "$3"
make }
make install
cd .. function cleanup() {
rv=$?
# cat the log file if it exists
if [ -f "$2" ]; then
cat "$2"
fi
# delete the build directory
rm -rf "$1"
echo "Build failed. Removing $1"
exit $rv
}
function install_brew_packages() {
brew update > /dev/null
brew install autoconf
brew install automake
brew install libtool
brew install pkg-config
brew install protobuf
if ! brew ls | grep gmp --quiet; then
brew install gmp
fi
}
function install_apt_packages() {
if [ -z "${TRAVIS+x}" ]; then
# if not on travis, its nice to see progress
QUIET=""
else
QUIET="-qq"
fi
# get the required OS packages
$SUDO apt-get ${QUIET} update
$SUDO apt-get ${QUIET} install -y --no-install-recommends \
build-essential python-dev libbz2-dev libtool \
autotools-dev autoconf git pkg-config wget \
ca-certificates automake bsdmainutils
}
function build_dependencies() {
if [ "${OS_NAME}" = "osx" ]; then
install_brew_packages
else
install_apt_packages
fi
if [ "$CLEAN" = true ]; then
rm -rf "${LBRYCRD_DEPENDENCIES}"
rm -rf "${OUTPUT_DIR}"
fi
if [ ! -d "${LBRYCRD_DEPENDENCIES}" ]; then
git clone https://github.com/lbryio/lbrycrd-dependencies.git "${LBRYCRD_DEPENDENCIES}"
fi
mkdir -p "${LOG_DIR}"
build_dependency "${BDB_PREFIX}" "${LOG_DIR}/bdb_build.log" build_bdb
build_dependency "${OPENSSL_PREFIX}" "${LOG_DIR}/openssl_build.log" build_openssl
set +u
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/"
set -u
build_dependency "${BOOST_PREFIX}" "${LOG_DIR}/boost_build.log" build_boost
build_dependency "${LIBEVENT_PREFIX}" "${LOG_DIR}/libevent_build.log" build_libevent
}
function build_bdb() {
BDB_LOG="$1"
if [ "${OS_NAME}" = "osx" ]; then
patch db-4.8.30.NC/dbinc/atomic.h < atomic.patch
fi
cd db-4.8.30.NC/build_unix
echo "Building bdb. tail -f $BDB_LOG to see the details and monitor progress"
../dist/configure --prefix="${BDB_PREFIX}" --enable-cxx --disable-shared --with-pic > "${BDB_LOG}"
background make "${BDB_LOG}" "Waiting for bdb to finish building"
make install >> "${BDB_LOG}" 2>&1
}
function build_openssl() {
OPENSSL_LOG="$1"
mkdir -p "${OPENSSL_PREFIX}/ssl"
cd openssl-1.0.1p
echo "Building openssl. tail -f $OPENSSL_LOG to see the details and monitor progress"
if [ "${OS_NAME}" = "osx" ]; then
./Configure --prefix="${OPENSSL_PREFIX}" --openssldir="${OPENSSL_PREFIX}/ssl" \
-fPIC darwin64-x86_64-cc \
no-shared no-dso no-engines > "${OPENSSL_LOG}"
else
./Configure --prefix="${OPENSSL_PREFIX}" --openssldir="${OPENSSL_PREFIX}/ssl" \
linux-x86_64 -fPIC -static no-shared no-dso > "${OPENSSL_LOG}"
fi
background make "${OPENSSL_LOG}" "Waiting for openssl to finish building"
make install >> "${OPENSSL_LOG}" 2>&1
}
function build_boost() {
BOOST_LOG="$1"
cd boost_1_59_0
echo "Building Boost. tail -f ${BOOST_LOG} to see the details and monitor progress"
./bootstrap.sh --prefix="${BOOST_PREFIX}" > "${BOOST_LOG}" 2>&1
background "./b2 link=static cxxflags=-fPIC install" \
"${BOOST_LOG}" \
"Waiting for boost to finish building"
}
function build_libevent() {
LIBEVENT_LOG="$1"
git clone https://github.com/libevent/libevent.git
cd libevent
echo "Building libevent. tail -f ${LIBEVENT_LOG} to see the details and monitor progress"
./autogen.sh > "${LIBEVENT_LOG}" 2>&1
./configure --prefix="${LIBEVENT_PREFIX}" --enable-static --disable-shared --with-pic \
LDFLAGS="-L${OPENSSL_PREFIX}/lib/" \
CPPFLAGS="-I${OPENSSL_PREFIX}/include" >> "${LIBEVENT_LOG}" 2>&1
background make "${LIBEVENT_LOG}" "Waiting for libevent to finish building"
make install >> "${LIBEVENT_LOG}"
}
function build_dependency() {
PREFIX=$1
LOG=$2
BUILD=$3
if [ ! -d "${PREFIX}" ]; then
trap 'cleanup "${PREFIX}" "${LOG}"' INT TERM EXIT
cd "${LBRYCRD_DEPENDENCIES}"
mkdir -p "${PREFIX}"
"${BUILD}" "${LOG}"
trap - INT TERM EXIT
fi
}
function build_lbrycrd() {
if [ "$CLONE" == true ]; then
cd "${LBRYCRD_DEPENDENCIES}"
git clone https://github.com/lbryio/lbrycrd
cd lbrycrd
else
cd "${SOURCE_DIR}"
fi
./autogen.sh
LDFLAGS="-L${OPENSSL_PREFIX}/lib/ -L${BDB_PREFIX}/lib/ -L${LIBEVENT_PREFIX}/lib/ -static-libstdc++"
CPPFLAGS="-I${OPENSSL_PREFIX}/include -I${BDB_PREFIX}/include -I${LIBEVENT_PREFIX}/include/"
if [ "${OS_NAME}" = "osx" ]; then
./configure --without-gui --enable-cxx --enable-static --disable-shared --with-pic \
--with-boost="${BOOST_PREFIX}" \
LDFLAGS="${LDFLAGS}" \
CPPFLAGS="${CPPFLAGS}"
else
./configure --without-gui --with-boost="${BOOST_PREFIX}" \
LDFLAGS="${LDFLAGS}" \
CPPFLAGS="${CPPFLAGS}"
fi
make
# tests don't work on OSX. Should definitely figure out why
# that is but, for now, not letting that stop the rest
# of the build
if [ "${OS_NAME}" = "linux" ]; then
src/test/test_lbrycrd
fi
strip src/lbrycrdd
strip src/lbrycrd-cli
strip src/lbrycrd-tx
strip src/test/test_lbrycrd
}
# these variables are needed in both functions
LBRYCRD_DEPENDENCIES="$(pwd)/lbrycrd-dependencies"
OUTPUT_DIR="$(pwd)/build"
LOG_DIR="$(pwd)/logs"
BDB_PREFIX="${OUTPUT_DIR}/bdb"
OPENSSL_PREFIX="${OUTPUT_DIR}/openssl"
BOOST_PREFIX="${OUTPUT_DIR}/boost"
LIBEVENT_PREFIX="${OUTPUT_DIR}/libevent"
if [ "${BUILD_DEPENDENCIES}" = true ]; then
build_dependencies
fi
set +u set +u
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/:${LIBEVENT_PREFIX}/lib/pkgconfig" export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/:${LIBEVENT_PREFIX}/lib/pkgconfig/"
set -u set -u
if [ "$CLONE" == true ]; then if [ "${BUILD_LBRYCRD}" = true ]; then
git clone https://github.com/lbryio/lbrycrd LBRYCRD_LOG="${LOG_DIR}/lbrycrd_build.log"
cd lbrycrd echo "Building lbrycrd. tail -f ${LBRYCRD_LOG} to see the details and monitor progress"
background build_lbrycrd "${LBRYCRD_LOG}" "Waiting for lbrycrd to finish building"
fi fi
./autogen.sh
./configure --without-gui LDFLAGS="-L${OPENSSL_PREFIX}/lib/ -L${BDB_PREFIX}/lib/ -L${LIBEVENT_PREFIX}/lib/ -static-libstdc++" CPPFLAGS="-I${OPENSSL_PREFIX}/include -I${BDB_PREFIX}/include -I${LIBEVENT_PREFIX}/include/"
make
strip src/lbrycrdd
strip src/lbrycrd-cli
strip src/lbrycrd-tx
strip src/test/test_lbrycrd

View file

@ -1,72 +0,0 @@
brew update
brew install autoconf
brew install automake
brew install libtool
brew install pkg-config
brew install protobuf
brew install gmp
mkdir dependencies
cd dependencies
export LBRYCRD_DEPENDENCIES="`pwd`"
#download, patch, and build bdb
wget http://download.oracle.com/berkeley-db/db-4.8.30.NC.tar.gz
tar xf db-4.8.30.NC.tar.gz
export BDB_PREFIX="`pwd`/bdb"
curl -OL https://raw.github.com/narkoleptik/os-x-berkeleydb-patch/master/atomic.patch
patch db-4.8.30.NC/dbinc/atomic.h < atomic.patch
cd db-4.8.30.NC/build_unix
../dist/configure --prefix=$BDB_PREFIX --enable-cxx --disable-shared --with-pic
make
make install
#download and build openssl
cd $LBRYCRD_DEPENDENCIES
wget https://www.openssl.org/source/openssl-1.0.1p.tar.gz
tar xf openssl-1.0.1p.tar.gz
export OPENSSL_PREFIX="`pwd`/openssl_build"
mkdir $OPENSSL_PREFIX
mkdir $OPENSSL_PREFIX/ssl
cd openssl-1.0.1p
./Configure --prefix=$OPENSSL_PREFIX --openssldir=$OPENSSL_PREFIX/ssl -fPIC darwin64-x86_64-cc no-shared no-dso no-engines
make
make install
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/"
#download and build boost
cd $LBRYCRD_DEPENDENCIES
wget http://sourceforge.net/projects/boost/files/boost/1.59.0/boost_1_59_0.tar.bz2/download -O boost_1_59_0.tar.bz2
tar xf boost_1_59_0.tar.bz2
export BOOST_ROOT="`pwd`/boost_1_59_0"
cd boost_1_59_0
./bootstrap.sh
./b2 link=static cxxflags=-fPIC stage
#download and build libevent
cd $LBRYCRD_DEPENDENCIES
mkdir libevent_build
git clone https://github.com/libevent/libevent.git
export LIBEVENT_PREFIX="`pwd`/libevent_build"
cd libevent
./autogen.sh
./configure --prefix=$LIBEVENT_PREFIX --enable-static --disable-shared --with-pic LDFLAGS="-L${OPENSSL_PREFIX}/lib/" CPPFLAGS="-I${OPENSSL_PREFIX}/include"
make
make install
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${LIBEVENT_PREFIX}/lib/pkgconfig/"
#download and build lbrycrd
cd $LBRYCRD_DEPENDENCIES
git clone https://github.com/lbryio/lbrycrd
cd lbrycrd
git checkout real2
./autogen.sh
./configure --without-gui --enable-cxx --enable-static --disable-shared --with-pic \
LDFLAGS="-L${OPENSSL_PREFIX}/lib/ -L${BDB_PREFIX}/lib/ -L${LIBEVENT_PREFIX}/lib/ -static-libstdc++" \
CPPFLAGS="-I${OPENSSL_PREFIX}/include -I${BDB_PREFIX}/include -I${LIBEVENT_PREFIX}/include/"
make
strip src/lbrycrdd
strip src/lbrycrd-cli
strip src/lbrycrd-tx
strip src/test/test-lbrycrd