Serialize effective amount #231

Closed
bvbfan wants to merge 309 commits from serialize_effective_amount into master
3 changed files with 312 additions and 146 deletions
Showing only changes of commit 783f5fd7cd - Show all commits

View file

@ -1,10 +1,31 @@
matrix:
include:
- os: linux
sudo: required sudo: required
dist: trusty dist: trusty
language: c
- os: osx
language: c
osx_image: xcode7.3
cache:
ccache: true
directories:
- build
before_install:
- date +%s > "${TRAVIS_BUILD_DIR}/start_time"
- ls -lh build
- du -h -d 2 build
install: true install: true
script: script:
- ./reproducible_build.sh -t - ./reproducible_build.sh -t -o -c
- zip -j lbrycrd-linux.zip src/lbrycrdd src/lbrycrd-cli src/lbrycrd-tx - if [[ "${TRAVIS_OS_NAME}" = "linux" ]]; then zip -j lbrycrd-linux.zip src/lbrycrdd src/lbrycrd-cli src/lbrycrd-tx; fi
before_cache:
- ls -lh build
- du -h -d 2 build
deploy: deploy:
- provider: releases - provider: releases
@ -12,5 +33,7 @@ deploy:
skip_cleanup: true skip_cleanup: true
on: on:
tags: true tags: true
condition: "$TRAVIS_OS_NAME = linux"
api_key: api_key:
secure: "ibN+PQg84f0tgJLV3KaHjkHmgFNOPqBLIrSU42moen22xxT2VIn0j7DkFaLLts1fs/+/gxmrvhOQ0vUFJqmflZ3kCeyedP/CVgZz7D7HRVK0cYUMJ2F9VWMZAFIX6A67OUlypqZqzKAKrZHx3HXky6Z8NBU23JgWqDm8PcrbbgMLvB2lvWVASjJVKreaL9BOxFOOOrAj6xAIm9ogTOnFVI/AhufNOKOTZaiprcbqMngwHeaILbwEJVqv5P2YUurC6Sq+QaH26tX00DtsYOW+n8AA3Fe48+L6rqa6Q8ru+dhPuR/Apr2+DTZ+npzY8leV3A7mYUeLo4JLaEH6n1TLVt65cX7nuesS7KsSKBSxs7q7bsqeapDcMskZz5JR4EK4S96CuHZEnn6+efpH8IrxWpbeO3EibJfyT8tMJKw7Zym5mPr+sz6xSlVMoBVcCm+7z9e7Zcqukdvdy8jM7sGP9qusqEhpojh0o+gPgr9p13SfR+6OCpj9gwxPCnZO8RBfh1cbz9vFSVlDyUYcpI2CCCpPazA1glTsEDk+VtznnUMmWksXWfxBFV81jQB2SnoZKlnsiEuNMwhiMCMEkRFo74+AsUr2UjmKJ5KC8ACwsn475PoLZBh2J6fyXBHMdbrGfK+Zt5K2IHRRtDaZVT4eHqYzxh8KhalsuBNxCo3FygI=" secure: "ibN+PQg84f0tgJLV3KaHjkHmgFNOPqBLIrSU42moen22xxT2VIn0j7DkFaLLts1fs/+/gxmrvhOQ0vUFJqmflZ3kCeyedP/CVgZz7D7HRVK0cYUMJ2F9VWMZAFIX6A67OUlypqZqzKAKrZHx3HXky6Z8NBU23JgWqDm8PcrbbgMLvB2lvWVASjJVKreaL9BOxFOOOrAj6xAIm9ogTOnFVI/AhufNOKOTZaiprcbqMngwHeaILbwEJVqv5P2YUurC6Sq+QaH26tX00DtsYOW+n8AA3Fe48+L6rqa6Q8ru+dhPuR/Apr2+DTZ+npzY8leV3A7mYUeLo4JLaEH6n1TLVt65cX7nuesS7KsSKBSxs7q7bsqeapDcMskZz5JR4EK4S96CuHZEnn6+efpH8IrxWpbeO3EibJfyT8tMJKw7Zym5mPr+sz6xSlVMoBVcCm+7z9e7Zcqukdvdy8jM7sGP9qusqEhpojh0o+gPgr9p13SfR+6OCpj9gwxPCnZO8RBfh1cbz9vFSVlDyUYcpI2CCCpPazA1glTsEDk+VtznnUMmWksXWfxBFV81jQB2SnoZKlnsiEuNMwhiMCMEkRFo74+AsUr2UjmKJ5KC8ACwsn475PoLZBh2J6fyXBHMdbrGfK+Zt5K2IHRRtDaZVT4eHqYzxh8KhalsuBNxCo3FygI="

View file

@ -3,26 +3,50 @@
set -euo pipefail set -euo pipefail
function HELP { function HELP {
echo "Build a lbrycrd" echo "Build lbrycrd"
echo "-----" echo "-----"
echo "When run without any arguments, this script expects the current directory" echo "When run without any arguments, this script expects the current directory"
echo "to be the lbrycrd repo and it builds what is in that directory" echo "to be the lbrycrd repo and it builds what is in that directory"
echo echo
echo "This is a long build process so it can be split into two parts"
echo "Specify the -d flag to build only the dependencies"
echo "and the -l flag to build only lbrycrd. This will fail"
echo "if the dependencies weren't built earlier"
echo
echo "Optional arguments:" echo "Optional arguments:"
echo echo
echo "-c: clone a fresh copy of the repo" echo "-c: don't clone a fresh copy of the repo"
echo "-h: show help" echo "-r: remove intermediate files."
echo "-l: build only lbrycrd"
echo "-d: build only the dependencies"
echo "-o: timeout build after 45 minutes"
echo "-t: turn trace on" echo "-t: turn trace on"
echo "-h: show help"
exit 1 exit 1
} }
CLONE=false CLONE=true
CLEAN=false
BUILD_DEPENDENCIES=true
BUILD_LBRYCRD=true
TIMEOUT=false
while getopts :crldoth:w:d: FLAG; do
while getopts :hctb:w:d: FLAG; do
case $FLAG in case $FLAG in
c) c)
CLONE=true CLONE=false
;;
r)
CLEAN=true
;;
l)
BUILD_DEPENDENCIES=false
;;
d)
BUILD_LBRYCRD=false
;;
o)
TIMEOUT=true
;; ;;
t) t)
set -o xtrace set -o xtrace
@ -44,86 +68,277 @@ done
shift $((OPTIND-1)) shift $((OPTIND-1))
SUDO='' SUDO=''
if (( $EUID != 0 )); then if (( EUID != 0 )); then
SUDO='sudo' SUDO='sudo'
fi fi
if [ "$CLONE" = false ]; then if [ "${CLONE}" = false ]; then
if [ `basename $PWD` != "lbrycrd" ]; then if [ "$(basename "$PWD")" != "lbrycrd" ]; then
echo "Not currently in the lbrycrd directory. Cowardly refusing to go forward" echo "Not currently in the lbrycrd directory. Cowardly refusing to go forward"
exit 1 exit 1
fi fi
SOURCE_DIR=$PWD SOURCE_DIR=$PWD
fi fi
if [ -z ${TRAVIS+x} ]; then if [ -z "${TRAVIS_OS_NAME+x}" ]; then
if [ "$(uname -s)" = "Darwin" ]; then
OS_NAME="osx"
else
OS_NAME="linux"
fi
else
OS_NAME="${TRAVIS_OS_NAME}"
fi
if [ -z "${TRAVIS_BUILD_DIR+x}" ]; then
START_TIME_FILE="$PWD/start_time"
else
# if we are on travis (the primary use case for setting a timeout)
# this file is created when the build starts
START_TIME_FILE="$TRAVIS_BUILD_DIR/start_time"
fi
if [ ! -f "${START_TIME_FILE}" ]; then
date +%s > "${START_TIME_FILE}"
fi
NEXT_TIME=60
function exit_at_45() {
if [ -f "${START_TIME_FILE}" ]; then
NOW=$(date +%s)
START=$(cat "${START_TIME_FILE}")
TIMEOUT_SECS=2700 # 45 * 60
TIME=$((NOW - START))
if (( TIME > NEXT_TIME )); then
echo "Build has taken $((TIME / 60)) minutes: $1"
NEXT_TIME=$((TIME + 60))
fi
if [ "$TIMEOUT" = true ] && (( TIME > TIMEOUT_SECS )); then
echo 'Exiting at 45 minutes to allow the cache to populate'
exit 1
fi
fi
}
# two arguments
# - pid (probably from $!)
# - echo message
function wait_and_echo() {
PID=$1
(set -o | grep xtrace | grep -q on)
TRACE_STATUS=$?
# disable xtrace or else this will get verbose, which is what
# I'm trying to avoid by going through all of this nonsense anyway
set +o xtrace
TIME=0
SLEEP=5
# loop until the process is no longer running
# check every $SLEEP seconds, echoing a message every minute
while (ps -p "${PID}" > /dev/null); do
exit_at_45 "$2"
sleep "${SLEEP}"
done
# restore the xtrace setting
if [ "${TRACE_STATUS}" -eq 0 ]; then
set -o xtrace
fi
wait "$PID"
return $?
}
# run a command ($1) in the background
# logging its stdout and stderr to $2
# and wait until it completed
function background() {
$1 >> "$2" 2>&1 &
wait_and_echo $! "$3"
}
function cleanup() {
rv=$?
# cat the log file if it exists
if [ -f "$2" ]; then
cat "$2"
fi
# delete the build directory
rm -rf "$1"
echo "Build failed. Removing $1"
exit $rv
}
function install_brew_packages() {
brew update > /dev/null
brew install autoconf
brew install automake
brew install libtool
brew install pkg-config
brew install protobuf
if ! brew ls | grep gmp --quiet; then
brew install gmp
fi
}
function install_apt_packages() {
if [ -z "${TRAVIS+x}" ]; then
# if not on travis, its nice to see progress # if not on travis, its nice to see progress
QUIET="" QUIET=""
else else
QUIET="-qq" QUIET="-qq"
fi fi
# get the required OS packages # get the required OS packages
$SUDO apt-get ${QUIET} update $SUDO apt-get ${QUIET} update
$SUDO apt-get ${QUIET} install -y --no-install-recommends \ $SUDO apt-get ${QUIET} install -y --no-install-recommends \
build-essential python-dev libbz2-dev libtool \ build-essential python-dev libbz2-dev libtool \
autotools-dev autoconf git pkg-config wget \ autotools-dev autoconf git pkg-config wget \
ca-certificates automake bsdmainutils ca-certificates automake bsdmainutils
}
mkdir dependencies function build_dependencies() {
cd dependencies if [ "${OS_NAME}" = "osx" ]; then
install_brew_packages
else
install_apt_packages
fi
wget http://download.oracle.com/berkeley-db/db-4.8.30.NC.tar.gz if [ "$CLEAN" = true ]; then
tar xf db-4.8.30.NC.tar.gz rm -rf "${LBRYCRD_DEPENDENCIES}"
export BDB_PREFIX="`pwd`/bdb" rm -rf "${OUTPUT_DIR}"
cd db-4.8.30.NC/build_unix fi
../dist/configure --prefix=$BDB_PREFIX --enable-cxx --disable-shared --with-pic
make
make install
cd ../../
wget https://www.openssl.org/source/openssl-1.0.1p.tar.gz if [ ! -d "${LBRYCRD_DEPENDENCIES}" ]; then
tar xf openssl-1.0.1p.tar.gz git clone https://github.com/lbryio/lbrycrd-dependencies.git "${LBRYCRD_DEPENDENCIES}"
export OPENSSL_PREFIX="`pwd`/openssl_build" fi
mkdir $OPENSSL_PREFIX mkdir -p "${LOG_DIR}"
cd openssl-1.0.1p
./Configure --prefix=$OPENSSL_PREFIX --openssldir=$OPENSSL_PREFIX/ssl linux-x86_64 -fPIC -static no-shared no-dso
make
make install
cd ..
build_dependency "${BDB_PREFIX}" "${LOG_DIR}/bdb_build.log" build_bdb
wget http://sourceforge.net/projects/boost/files/boost/1.59.0/boost_1_59_0.tar.bz2/download -O boost_1_59_0.tar.bz2 build_dependency "${OPENSSL_PREFIX}" "${LOG_DIR}/openssl_build.log" build_openssl
tar xf boost_1_59_0.tar.bz2
export BOOST_ROOT="`pwd`/boost_1_59_0"
cd boost_1_59_0
./bootstrap.sh
./b2 link=static cxxflags=-fPIC stage
cd ../../
mkdir libevent_build
git clone https://github.com/libevent/libevent.git
export LIBEVENT_PREFIX="`pwd`/libevent_build"
cd libevent
./autogen.sh
./configure --prefix=$LIBEVENT_PREFIX --enable-static --disable-shared --with-pic LDFLAGS="-L${OPENSSL_PREFIX}/lib/" CPPFLAGS="-I${OPENSSL_PREFIX}/include"
make
make install
cd ..
set +u set +u
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/:${LIBEVENT_PREFIX}/lib/pkgconfig" export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/"
set -u set -u
build_dependency "${BOOST_PREFIX}" "${LOG_DIR}/boost_build.log" build_boost
build_dependency "${LIBEVENT_PREFIX}" "${LOG_DIR}/libevent_build.log" build_libevent
}
function build_bdb() {
BDB_LOG="$1"
if [ "${OS_NAME}" = "osx" ]; then
patch db-4.8.30.NC/dbinc/atomic.h < atomic.patch
fi
cd db-4.8.30.NC/build_unix
echo "Building bdb. tail -f $BDB_LOG to see the details and monitor progress"
../dist/configure --prefix="${BDB_PREFIX}" --enable-cxx --disable-shared --with-pic > "${BDB_LOG}"
background make "${BDB_LOG}" "Waiting for bdb to finish building"
make install >> "${BDB_LOG}" 2>&1
}
function build_openssl() {
OPENSSL_LOG="$1"
mkdir -p "${OPENSSL_PREFIX}/ssl"
cd openssl-1.0.1p
echo "Building openssl. tail -f $OPENSSL_LOG to see the details and monitor progress"
if [ "${OS_NAME}" = "osx" ]; then
./Configure --prefix="${OPENSSL_PREFIX}" --openssldir="${OPENSSL_PREFIX}/ssl" \
-fPIC darwin64-x86_64-cc \
no-shared no-dso no-engines > "${OPENSSL_LOG}"
else
./Configure --prefix="${OPENSSL_PREFIX}" --openssldir="${OPENSSL_PREFIX}/ssl" \
linux-x86_64 -fPIC -static no-shared no-dso > "${OPENSSL_LOG}"
fi
background make "${OPENSSL_LOG}" "Waiting for openssl to finish building"
make install >> "${OPENSSL_LOG}" 2>&1
}
function build_boost() {
BOOST_LOG="$1"
cd boost_1_59_0
echo "Building Boost. tail -f ${BOOST_LOG} to see the details and monitor progress"
./bootstrap.sh --prefix="${BOOST_PREFIX}" > "${BOOST_LOG}" 2>&1
background "./b2 link=static cxxflags=-fPIC install" \
"${BOOST_LOG}" \
"Waiting for boost to finish building"
}
function build_libevent() {
LIBEVENT_LOG="$1"
git clone https://github.com/libevent/libevent.git
cd libevent
echo "Building libevent. tail -f ${LIBEVENT_LOG} to see the details and monitor progress"
./autogen.sh > "${LIBEVENT_LOG}" 2>&1
./configure --prefix="${LIBEVENT_PREFIX}" --enable-static --disable-shared --with-pic \
LDFLAGS="-L${OPENSSL_PREFIX}/lib/" \
CPPFLAGS="-I${OPENSSL_PREFIX}/include" >> "${LIBEVENT_LOG}" 2>&1
background make "${LIBEVENT_LOG}" "Waiting for libevent to finish building"
make install >> "${LIBEVENT_LOG}"
}
function build_dependency() {
PREFIX=$1
LOG=$2
BUILD=$3
if [ ! -d "${PREFIX}" ]; then
trap 'cleanup "${PREFIX}" "${LOG}"' INT TERM EXIT
cd "${LBRYCRD_DEPENDENCIES}"
mkdir -p "${PREFIX}"
"${BUILD}" "${LOG}"
trap - INT TERM EXIT
fi
}
function build_lbrycrd() {
if [ "$CLONE" == true ]; then if [ "$CLONE" == true ]; then
cd "${LBRYCRD_DEPENDENCIES}"
git clone https://github.com/lbryio/lbrycrd git clone https://github.com/lbryio/lbrycrd
cd lbrycrd cd lbrycrd
else
cd "${SOURCE_DIR}"
fi fi
./autogen.sh ./autogen.sh
./configure --without-gui LDFLAGS="-L${OPENSSL_PREFIX}/lib/ -L${BDB_PREFIX}/lib/ -L${LIBEVENT_PREFIX}/lib/ -static-libstdc++" CPPFLAGS="-I${OPENSSL_PREFIX}/include -I${BDB_PREFIX}/include -I${LIBEVENT_PREFIX}/include/" LDFLAGS="-L${OPENSSL_PREFIX}/lib/ -L${BDB_PREFIX}/lib/ -L${LIBEVENT_PREFIX}/lib/ -static-libstdc++"
CPPFLAGS="-I${OPENSSL_PREFIX}/include -I${BDB_PREFIX}/include -I${LIBEVENT_PREFIX}/include/"
if [ "${OS_NAME}" = "osx" ]; then
./configure --without-gui --enable-cxx --enable-static --disable-shared --with-pic \
--with-boost="${BOOST_PREFIX}" \
LDFLAGS="${LDFLAGS}" \
CPPFLAGS="${CPPFLAGS}"
else
./configure --without-gui --with-boost="${BOOST_PREFIX}" \
LDFLAGS="${LDFLAGS}" \
CPPFLAGS="${CPPFLAGS}"
fi
make make
# tests don't work on OSX. Should definitely figure out why
# that is but, for now, not letting that stop the rest
# of the build
if [ "${OS_NAME}" = "linux" ]; then
src/test/test_lbrycrd
fi
strip src/lbrycrdd strip src/lbrycrdd
strip src/lbrycrd-cli strip src/lbrycrd-cli
strip src/lbrycrd-tx strip src/lbrycrd-tx
strip src/test/test_lbrycrd strip src/test/test_lbrycrd
}
# these variables are needed in both functions
LBRYCRD_DEPENDENCIES="$(pwd)/lbrycrd-dependencies"
OUTPUT_DIR="$(pwd)/build"
LOG_DIR="$(pwd)/logs"
BDB_PREFIX="${OUTPUT_DIR}/bdb"
OPENSSL_PREFIX="${OUTPUT_DIR}/openssl"
BOOST_PREFIX="${OUTPUT_DIR}/boost"
LIBEVENT_PREFIX="${OUTPUT_DIR}/libevent"
if [ "${BUILD_DEPENDENCIES}" = true ]; then
build_dependencies
fi
set +u
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/:${LIBEVENT_PREFIX}/lib/pkgconfig/"
set -u
if [ "${BUILD_LBRYCRD}" = true ]; then
LBRYCRD_LOG="${LOG_DIR}/lbrycrd_build.log"
echo "Building lbrycrd. tail -f ${LBRYCRD_LOG} to see the details and monitor progress"
background build_lbrycrd "${LBRYCRD_LOG}" "Waiting for lbrycrd to finish building"
fi

View file

@ -1,72 +0,0 @@
brew update
brew install autoconf
brew install automake
brew install libtool
brew install pkg-config
brew install protobuf
brew install gmp
mkdir dependencies
cd dependencies
export LBRYCRD_DEPENDENCIES="`pwd`"
#download, patch, and build bdb
wget http://download.oracle.com/berkeley-db/db-4.8.30.NC.tar.gz
tar xf db-4.8.30.NC.tar.gz
export BDB_PREFIX="`pwd`/bdb"
curl -OL https://raw.github.com/narkoleptik/os-x-berkeleydb-patch/master/atomic.patch
patch db-4.8.30.NC/dbinc/atomic.h < atomic.patch
cd db-4.8.30.NC/build_unix
../dist/configure --prefix=$BDB_PREFIX --enable-cxx --disable-shared --with-pic
make
make install
#download and build openssl
cd $LBRYCRD_DEPENDENCIES
wget https://www.openssl.org/source/openssl-1.0.1p.tar.gz
tar xf openssl-1.0.1p.tar.gz
export OPENSSL_PREFIX="`pwd`/openssl_build"
mkdir $OPENSSL_PREFIX
mkdir $OPENSSL_PREFIX/ssl
cd openssl-1.0.1p
./Configure --prefix=$OPENSSL_PREFIX --openssldir=$OPENSSL_PREFIX/ssl -fPIC darwin64-x86_64-cc no-shared no-dso no-engines
make
make install
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/"
#download and build boost
cd $LBRYCRD_DEPENDENCIES
wget http://sourceforge.net/projects/boost/files/boost/1.59.0/boost_1_59_0.tar.bz2/download -O boost_1_59_0.tar.bz2
tar xf boost_1_59_0.tar.bz2
export BOOST_ROOT="`pwd`/boost_1_59_0"
cd boost_1_59_0
./bootstrap.sh
./b2 link=static cxxflags=-fPIC stage
#download and build libevent
cd $LBRYCRD_DEPENDENCIES
mkdir libevent_build
git clone https://github.com/libevent/libevent.git
export LIBEVENT_PREFIX="`pwd`/libevent_build"
cd libevent
./autogen.sh
./configure --prefix=$LIBEVENT_PREFIX --enable-static --disable-shared --with-pic LDFLAGS="-L${OPENSSL_PREFIX}/lib/" CPPFLAGS="-I${OPENSSL_PREFIX}/include"
make
make install
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${LIBEVENT_PREFIX}/lib/pkgconfig/"
#download and build lbrycrd
cd $LBRYCRD_DEPENDENCIES
git clone https://github.com/lbryio/lbrycrd
cd lbrycrd
git checkout real2
./autogen.sh
./configure --without-gui --enable-cxx --enable-static --disable-shared --with-pic \
LDFLAGS="-L${OPENSSL_PREFIX}/lib/ -L${BDB_PREFIX}/lib/ -L${LIBEVENT_PREFIX}/lib/ -static-libstdc++" \
CPPFLAGS="-I${OPENSSL_PREFIX}/include -I${BDB_PREFIX}/include -I${LIBEVENT_PREFIX}/include/"
make
strip src/lbrycrdd
strip src/lbrycrd-cli
strip src/lbrycrd-tx
strip src/test/test-lbrycrd