Add osx build to travis

This is a large commit.  Travis has some limitations that
the OSX build runs into so a few things needed to change:

- travis has a 4mb log limit
    - log stdout/stderr to log files
    - cat relevant log file if there is a build issue
- travis will kill the job if it is silent for ten minutes
    - echo messages every minute
- travis has a 50 minute job limit
    - add caching for dependencies
    - proactively abort a build after 45 minutes to ensure the
        cache will be populated

Caching creates its own set of problems so the ability to clear the
dependency folder was added and each dependency will delete its folder
if something goes wrong during the build of that dependency. This
prevents future runs from thinking a dependency has been cached when
it actually hasn't.

The build files for OSX and linux were unified into one script.
This commit is contained in:
Job Evers-Meltzer 2016-07-01 09:37:13 -05:00 committed by Job Evers
parent 786e9e13c3
commit 5b113e524c
3 changed files with 320 additions and 146 deletions

View file

@ -1,10 +1,31 @@
matrix:
include:
- os: linux
sudo: required
dist: trusty
language: c
- os: osx
language: c
osx_image: xcode7.3
cache:
ccache: true
directories:
- build
before_install:
- date +%s > "${TRAVIS_BUILD_DIR}/start_time"
- ls -lh build
- du -h -d 2 build
install: true
script:
- ./reproducible_build.sh -t
- zip -j lbrycrd-linux.zip src/lbrycrdd src/lbrycrd-cli src/lbrycrd-tx
- ./reproducible_build.sh -t -o -c
- if [[ "${TRAVIS_OS_NAME}" = "linux" ]]; then zip -j lbrycrd-linux.zip src/lbrycrdd src/lbrycrd-cli src/lbrycrd-tx; fi
before_cache:
- ls -lh build
- du -h -d 2 build
deploy:
- provider: releases
@ -12,5 +33,7 @@ deploy:
skip_cleanup: true
on:
tags: true
condition: "$TRAVIS_OS_NAME = linux"
api_key:
secure: "ibN+PQg84f0tgJLV3KaHjkHmgFNOPqBLIrSU42moen22xxT2VIn0j7DkFaLLts1fs/+/gxmrvhOQ0vUFJqmflZ3kCeyedP/CVgZz7D7HRVK0cYUMJ2F9VWMZAFIX6A67OUlypqZqzKAKrZHx3HXky6Z8NBU23JgWqDm8PcrbbgMLvB2lvWVASjJVKreaL9BOxFOOOrAj6xAIm9ogTOnFVI/AhufNOKOTZaiprcbqMngwHeaILbwEJVqv5P2YUurC6Sq+QaH26tX00DtsYOW+n8AA3Fe48+L6rqa6Q8ru+dhPuR/Apr2+DTZ+npzY8leV3A7mYUeLo4JLaEH6n1TLVt65cX7nuesS7KsSKBSxs7q7bsqeapDcMskZz5JR4EK4S96CuHZEnn6+efpH8IrxWpbeO3EibJfyT8tMJKw7Zym5mPr+sz6xSlVMoBVcCm+7z9e7Zcqukdvdy8jM7sGP9qusqEhpojh0o+gPgr9p13SfR+6OCpj9gwxPCnZO8RBfh1cbz9vFSVlDyUYcpI2CCCpPazA1glTsEDk+VtznnUMmWksXWfxBFV81jQB2SnoZKlnsiEuNMwhiMCMEkRFo74+AsUr2UjmKJ5KC8ACwsn475PoLZBh2J6fyXBHMdbrGfK+Zt5K2IHRRtDaZVT4eHqYzxh8KhalsuBNxCo3FygI="

View file

@ -3,26 +3,50 @@
set -euo pipefail
function HELP {
echo "Build a lbrycrd"
echo "Build lbrycrd"
echo "-----"
echo "When run without any arguments, this script expects the current directory"
echo "to be the lbrycrd repo and it builds what is in that directory"
echo
echo "This is a long build process so it can be split into two parts"
echo "Specify the -d flag to build only the dependencies"
echo "and the -l flag to build only lbrycrd. This will fail"
echo "if the dependencies weren't built earlier"
echo
echo "Optional arguments:"
echo
echo "-c: clone a fresh copy of the repo"
echo "-h: show help"
echo "-c: don't clone a fresh copy of the repo"
echo "-r: remove intermediate files."
echo "-l: build only lbrycrd"
echo "-d: build only the dependencies"
echo "-o: timeout build after 45 minutes"
echo "-t: turn trace on"
echo "-h: show help"
exit 1
}
CLONE=false
CLONE=true
CLEAN=false
BUILD_DEPENDENCIES=true
BUILD_LBRYCRD=true
TIMEOUT=false
while getopts :hctb:w:d: FLAG; do
while getopts :crldoth:w:d: FLAG; do
case $FLAG in
c)
CLONE=true
CLONE=false
;;
r)
CLEAN=true
;;
l)
BUILD_DEPENDENCIES=false
;;
d)
BUILD_LBRYCRD=false
;;
o)
TIMEOUT=true
;;
t)
set -o xtrace
@ -44,86 +68,285 @@ done
shift $((OPTIND-1))
SUDO=''
if (( $EUID != 0 )); then
if (( EUID != 0 )); then
SUDO='sudo'
fi
if [ "$CLONE" = false ]; then
if [ `basename $PWD` != "lbrycrd" ]; then
if [ "${CLONE}" = false ]; then
if [ "$(basename "$PWD")" != "lbrycrd" ]; then
echo "Not currently in the lbrycrd directory. Cowardly refusing to go forward"
exit 1
fi
SOURCE_DIR=$PWD
fi
if [ -z ${TRAVIS+x} ]; then
if [ -z "${TRAVIS_OS_NAME+x}" ]; then
if [ "$(uname -s)" = "Darwin" ]; then
OS_NAME="osx"
else
OS_NAME="linux"
fi
else
OS_NAME="${TRAVIS_OS_NAME}"
fi
if [ -z "${TRAVIS_BUILD_DIR+x}" ]; then
START_TIME_FILE="$PWD/start_time"
else
# if we are on travis (the primary use case for setting a timeout)
# this file is created when the build starts
START_TIME_FILE="$TRAVIS_BUILD_DIR/start_time"
fi
if [ ! -f "${START_TIME_FILE}" ]; then
date +%s > "${START_TIME_FILE}"
fi
NEXT_TIME=60
function exit_at_45() {
if [ -f "${START_TIME_FILE}" ]; then
NOW=$(date +%s)
START=$(cat "${START_TIME_FILE}")
TIMEOUT_SECS=2700 # 45 * 60
TIME=$((NOW - START))
if (( TIME > NEXT_TIME )); then
echo "Build has taken $((TIME / 60)) minutes: $1"
NEXT_TIME=$((TIME + 60))
fi
if [ "$TIMEOUT" = true ] && (( TIME > TIMEOUT_SECS )); then
echo 'Exiting at 45 minutes to allow the cache to populate'
exit 1
fi
fi
}
# two arguments
# - pid (probably from $!)
# - echo message
function wait_and_echo() {
PID=$1
(set -o | grep xtrace | grep -q on)
TRACE_STATUS=$?
# disable xtrace or else this will get verbose, which is what
# I'm trying to avoid by going through all of this nonsense anyway
set +o xtrace
TIME=0
SLEEP=5
# loop until the process is no longer running
# check every $SLEEP seconds, echoing a message every minute
while (ps -p "${PID}" > /dev/null); do
exit_at_45 "$2"
sleep "${SLEEP}"
done
# restore the xtrace setting
if [ "${TRACE_STATUS}" -eq 0 ]; then
set -o xtrace
fi
wait "$PID"
return $?
}
# run a command ($1) in the background
# logging its stdout and stderr to $2
# and wait until it completed
function background() {
$1 >> "$2" 2>&1 &
wait_and_echo $! "$3"
}
function cleanup() {
rv=$?
# cat the log file if it exists
if [ -f "$2" ]; then
cat "$2"
fi
# delete the build directory
rm -rf "$1"
echo "Build failed. Removing $1"
exit $rv
}
function install_brew_packages() {
brew update > /dev/null
brew install autoconf
brew install automake
brew install libtool
brew install pkg-config
brew install protobuf
if ! brew ls | grep gmp --quiet; then
brew install gmp
fi
}
function install_apt_packages() {
if [ -z "${TRAVIS+x}" ]; then
# if not on travis, its nice to see progress
QUIET=""
else
QUIET="-qq"
fi
# get the required OS packages
$SUDO apt-get ${QUIET} update
$SUDO apt-get ${QUIET} install -y --no-install-recommends \
build-essential python-dev libbz2-dev libtool \
autotools-dev autoconf git pkg-config wget \
ca-certificates automake bsdmainutils
}
mkdir dependencies
cd dependencies
function build_dependencies() {
if [ "${OS_NAME}" = "osx" ]; then
install_brew_packages
else
install_apt_packages
fi
wget http://download.oracle.com/berkeley-db/db-4.8.30.NC.tar.gz
tar xf db-4.8.30.NC.tar.gz
export BDB_PREFIX="`pwd`/bdb"
cd db-4.8.30.NC/build_unix
../dist/configure --prefix=$BDB_PREFIX --enable-cxx --disable-shared --with-pic
make
make install
cd ../../
if [ "$CLEAN" = true ]; then
rm -rf "${LBRYCRD_DEPENDENCIES}"
rm -rf "${OUTPUT_DIR}"
fi
wget https://www.openssl.org/source/openssl-1.0.1p.tar.gz
tar xf openssl-1.0.1p.tar.gz
export OPENSSL_PREFIX="`pwd`/openssl_build"
mkdir $OPENSSL_PREFIX
cd openssl-1.0.1p
./Configure --prefix=$OPENSSL_PREFIX --openssldir=$OPENSSL_PREFIX/ssl linux-x86_64 -fPIC -static no-shared no-dso
make
make install
cd ..
mkdir -p "${LBRYCRD_DEPENDENCIES}"
mkdir -p "${LOG_DIR}"
wget http://sourceforge.net/projects/boost/files/boost/1.59.0/boost_1_59_0.tar.bz2/download -O boost_1_59_0.tar.bz2
tar xf boost_1_59_0.tar.bz2
export BOOST_ROOT="`pwd`/boost_1_59_0"
cd boost_1_59_0
./bootstrap.sh
./b2 link=static cxxflags=-fPIC stage
cd ../../
mkdir libevent_build
git clone https://github.com/libevent/libevent.git
export LIBEVENT_PREFIX="`pwd`/libevent_build"
cd libevent
./autogen.sh
./configure --prefix=$LIBEVENT_PREFIX --enable-static --disable-shared --with-pic LDFLAGS="-L${OPENSSL_PREFIX}/lib/" CPPFLAGS="-I${OPENSSL_PREFIX}/include"
make
make install
cd ..
build_dependency "${BDB_PREFIX}" "${LOG_DIR}/bdb_build.log" build_bdb
build_dependency "${OPENSSL_PREFIX}" "${LOG_DIR}/openssl_build.log" build_openssl
set +u
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/:${LIBEVENT_PREFIX}/lib/pkgconfig"
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/"
set -u
build_dependency "${BOOST_PREFIX}" "${LOG_DIR}/boost_build.log" build_boost
build_dependency "${LIBEVENT_PREFIX}" "${LOG_DIR}/libevent_build.log" build_libevent
}
function build_bdb() {
BDB_LOG="$1"
if [ ! -f db-4.8.30.NC.tar.gz ]; then
wget http://download.oracle.com/berkeley-db/db-4.8.30.NC.tar.gz
fi
tar xf db-4.8.30.NC.tar.gz
if [ "${OS_NAME}" = "osx" ]; then
curl -OL https://raw.github.com/narkoleptik/os-x-berkeleydb-patch/master/atomic.patch
patch db-4.8.30.NC/dbinc/atomic.h < atomic.patch
fi
cd db-4.8.30.NC/build_unix
echo "Building bdb. tail -f $BDB_LOG to see the details and monitor progress"
../dist/configure --prefix="${BDB_PREFIX}" --enable-cxx --disable-shared --with-pic > "${BDB_LOG}"
background make "${BDB_LOG}" "Waiting for bdb to finish building"
make install >> "${BDB_LOG}" 2>&1
}
function build_openssl() {
OPENSSL_LOG="$1"
wget https://www.openssl.org/source/openssl-1.0.1p.tar.gz
tar xf openssl-1.0.1p.tar.gz
mkdir -p "${OPENSSL_PREFIX}/ssl"
cd openssl-1.0.1p
echo "Building openssl. tail -f $OPENSSL_LOG to see the details and monitor progress"
if [ "${OS_NAME}" = "osx" ]; then
./Configure --prefix="${OPENSSL_PREFIX}" --openssldir="${OPENSSL_PREFIX}/ssl" \
-fPIC darwin64-x86_64-cc \
no-shared no-dso no-engines > "${OPENSSL_LOG}"
else
./Configure --prefix="${OPENSSL_PREFIX}" --openssldir="${OPENSSL_PREFIX}/ssl" \
linux-x86_64 -fPIC -static no-shared no-dso > "${OPENSSL_LOG}"
fi
background make "${OPENSSL_LOG}" "Waiting for openssl to finish building"
make install >> "${OPENSSL_LOG}" 2>&1
}
function build_boost() {
BOOST_LOG="$1"
wget http://sourceforge.net/projects/boost/files/boost/1.59.0/boost_1_59_0.tar.bz2/download \
-O boost_1_59_0.tar.bz2
tar xf boost_1_59_0.tar.bz2
cd boost_1_59_0
echo "Building Boost. tail -f ${BOOST_LOG} to see the details and monitor progress"
./bootstrap.sh --prefix="${BOOST_PREFIX}" > "${BOOST_LOG}" 2>&1
background "./b2 link=static cxxflags=-fPIC install" \
"${BOOST_LOG}" \
"Waiting for boost to finish building"
}
function build_libevent() {
LIBEVENT_LOG="$1"
git clone https://github.com/libevent/libevent.git
cd libevent
echo "Building libevent. tail -f ${LIBEVENT_LOG} to see the details and monitor progress"
./autogen.sh > "${LIBEVENT_LOG}" 2>&1
./configure --prefix="${LIBEVENT_PREFIX}" --enable-static --disable-shared --with-pic \
LDFLAGS="-L${OPENSSL_PREFIX}/lib/" \
CPPFLAGS="-I${OPENSSL_PREFIX}/include" >> "${LIBEVENT_LOG}" 2>&1
background make "${LIBEVENT_LOG}" "Waiting for libevent to finish building"
make install >> "${LIBEVENT_LOG}"
}
function build_dependency() {
PREFIX=$1
LOG=$2
BUILD=$3
if [ ! -d "${PREFIX}" ]; then
trap 'cleanup "${PREFIX}" "${LOG}"' INT TERM EXIT
cd "${LBRYCRD_DEPENDENCIES}"
mkdir -p "${PREFIX}"
"${BUILD}" "${LOG}"
trap - INT TERM EXIT
fi
}
function build_lbrycrd() {
if [ "$CLONE" == true ]; then
cd "${LBRYCRD_DEPENDENCIES}"
git clone https://github.com/lbryio/lbrycrd
cd lbrycrd
else
cd "${SOURCE_DIR}"
fi
./autogen.sh
./configure --without-gui LDFLAGS="-L${OPENSSL_PREFIX}/lib/ -L${BDB_PREFIX}/lib/ -L${LIBEVENT_PREFIX}/lib/ -static-libstdc++" CPPFLAGS="-I${OPENSSL_PREFIX}/include -I${BDB_PREFIX}/include -I${LIBEVENT_PREFIX}/include/"
LDFLAGS="-L${OPENSSL_PREFIX}/lib/ -L${BDB_PREFIX}/lib/ -L${LIBEVENT_PREFIX}/lib/ -static-libstdc++"
CPPFLAGS="-I${OPENSSL_PREFIX}/include -I${BDB_PREFIX}/include -I${LIBEVENT_PREFIX}/include/"
if [ "${OS_NAME}" = "osx" ]; then
./configure --without-gui --enable-cxx --enable-static --disable-shared --with-pic \
--with-boost="${BOOST_PREFIX}" \
LDFLAGS="${LDFLAGS}" \
CPPFLAGS="${CPPFLAGS}"
else
./configure --without-gui --with-boost="${BOOST_PREFIX}" \
LDFLAGS="${LDFLAGS}" \
CPPFLAGS="${CPPFLAGS}"
fi
make
# tests don't work on OSX. Should definitely figure out why
# that is but, for now, not letting that stop the rest
# of the build
if [ "${OS_NAME}" = "linux" ]; then
src/test/test_lbrycrd
fi
strip src/lbrycrdd
strip src/lbrycrd-cli
strip src/lbrycrd-tx
strip src/test/test_lbrycrd
}
# these variables are needed in both functions
LBRYCRD_DEPENDENCIES="$(pwd)/lbrycrd-dependencies"
OUTPUT_DIR="$(pwd)/build"
LOG_DIR="$(pwd)/logs"
BDB_PREFIX="${OUTPUT_DIR}/bdb"
OPENSSL_PREFIX="${OUTPUT_DIR}/openssl"
BOOST_PREFIX="${OUTPUT_DIR}/boost"
LIBEVENT_PREFIX="${OUTPUT_DIR}/libevent"
if [ "${BUILD_DEPENDENCIES}" = true ]; then
build_dependencies
fi
set +u
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/:${LIBEVENT_PREFIX}/lib/pkgconfig/"
set -u
if [ "${BUILD_LBRYCRD}" = true ]; then
LBRYCRD_LOG="${LOG_DIR}/lbrycrd_build.log"
echo "Building lbrycrd. tail -f ${LBRYCRD_LOG} to see the details and monitor progress"
background build_lbrycrd "${LBRYCRD_LOG}" "Waiting for lbrycrd to finish building"
fi

View file

@ -1,72 +0,0 @@
brew update
brew install autoconf
brew install automake
brew install libtool
brew install pkg-config
brew install protobuf
brew install gmp
mkdir dependencies
cd dependencies
export LBRYCRD_DEPENDENCIES="`pwd`"
#download, patch, and build bdb
wget http://download.oracle.com/berkeley-db/db-4.8.30.NC.tar.gz
tar xf db-4.8.30.NC.tar.gz
export BDB_PREFIX="`pwd`/bdb"
curl -OL https://raw.github.com/narkoleptik/os-x-berkeleydb-patch/master/atomic.patch
patch db-4.8.30.NC/dbinc/atomic.h < atomic.patch
cd db-4.8.30.NC/build_unix
../dist/configure --prefix=$BDB_PREFIX --enable-cxx --disable-shared --with-pic
make
make install
#download and build openssl
cd $LBRYCRD_DEPENDENCIES
wget https://www.openssl.org/source/openssl-1.0.1p.tar.gz
tar xf openssl-1.0.1p.tar.gz
export OPENSSL_PREFIX="`pwd`/openssl_build"
mkdir $OPENSSL_PREFIX
mkdir $OPENSSL_PREFIX/ssl
cd openssl-1.0.1p
./Configure --prefix=$OPENSSL_PREFIX --openssldir=$OPENSSL_PREFIX/ssl -fPIC darwin64-x86_64-cc no-shared no-dso no-engines
make
make install
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/"
#download and build boost
cd $LBRYCRD_DEPENDENCIES
wget http://sourceforge.net/projects/boost/files/boost/1.59.0/boost_1_59_0.tar.bz2/download -O boost_1_59_0.tar.bz2
tar xf boost_1_59_0.tar.bz2
export BOOST_ROOT="`pwd`/boost_1_59_0"
cd boost_1_59_0
./bootstrap.sh
./b2 link=static cxxflags=-fPIC stage
#download and build libevent
cd $LBRYCRD_DEPENDENCIES
mkdir libevent_build
git clone https://github.com/libevent/libevent.git
export LIBEVENT_PREFIX="`pwd`/libevent_build"
cd libevent
./autogen.sh
./configure --prefix=$LIBEVENT_PREFIX --enable-static --disable-shared --with-pic LDFLAGS="-L${OPENSSL_PREFIX}/lib/" CPPFLAGS="-I${OPENSSL_PREFIX}/include"
make
make install
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${LIBEVENT_PREFIX}/lib/pkgconfig/"
#download and build lbrycrd
cd $LBRYCRD_DEPENDENCIES
git clone https://github.com/lbryio/lbrycrd
cd lbrycrd
git checkout real2
./autogen.sh
./configure --without-gui --enable-cxx --enable-static --disable-shared --with-pic \
LDFLAGS="-L${OPENSSL_PREFIX}/lib/ -L${BDB_PREFIX}/lib/ -L${LIBEVENT_PREFIX}/lib/ -static-libstdc++" \
CPPFLAGS="-I${OPENSSL_PREFIX}/include -I${BDB_PREFIX}/include -I${LIBEVENT_PREFIX}/include/"
make
strip src/lbrycrdd
strip src/lbrycrd-cli
strip src/lbrycrd-tx
strip src/test/test-lbrycrd