From 5b113e524c41918ce126c94aed8da3ac51b94b00 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Fri, 1 Jul 2016 09:37:13 -0500 Subject: [PATCH] Add osx build to travis This is a large commit. Travis has some limitations that the OSX build runs into so a few things needed to change: - travis has a 4mb log limit - log stdout/stderr to log files - cat relevant log file if there is a build issue - travis will kill the job if it is silent for ten minutes - echo messages every minute - travis has a 50 minute job limit - add caching for dependencies - proactively abort a build after 45 minutes to ensure the cache will be populated Caching creates its own set of problems so the ability to clear the dependency folder was added and each dependency will delete its folder if something goes wrong during the build of that dependency. This prevents future runs from thinking a dependency has been cached when it actually hasn't. The build files for OSX and linux were unified into one script. --- .travis.yml | 33 +++- reproducible_build.sh | 361 ++++++++++++++++++++++++++++++-------- reproducible_build_osx.sh | 72 -------- 3 files changed, 320 insertions(+), 146 deletions(-) delete mode 100644 reproducible_build_osx.sh diff --git a/.travis.yml b/.travis.yml index 8dcde0ca4..fccfbeee6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,31 @@ -sudo: required -dist: trusty +matrix: + include: + - os: linux + sudo: required + dist: trusty + language: c + - os: osx + language: c + osx_image: xcode7.3 + +cache: + ccache: true + directories: + - build + +before_install: + - date +%s > "${TRAVIS_BUILD_DIR}/start_time" + - ls -lh build + - du -h -d 2 build install: true -script: - - ./reproducible_build.sh -t - - zip -j lbrycrd-linux.zip src/lbrycrdd src/lbrycrd-cli src/lbrycrd-tx +script: + - ./reproducible_build.sh -t -o -c + - if [[ "${TRAVIS_OS_NAME}" = "linux" ]]; then zip -j lbrycrd-linux.zip src/lbrycrdd src/lbrycrd-cli src/lbrycrd-tx; fi + +before_cache: + - ls -lh build + - du -h -d 2 build deploy: - provider: releases @@ -12,5 +33,7 @@ deploy: skip_cleanup: true on: tags: true + condition: "$TRAVIS_OS_NAME = linux" api_key: secure: "ibN+PQg84f0tgJLV3KaHjkHmgFNOPqBLIrSU42moen22xxT2VIn0j7DkFaLLts1fs/+/gxmrvhOQ0vUFJqmflZ3kCeyedP/CVgZz7D7HRVK0cYUMJ2F9VWMZAFIX6A67OUlypqZqzKAKrZHx3HXky6Z8NBU23JgWqDm8PcrbbgMLvB2lvWVASjJVKreaL9BOxFOOOrAj6xAIm9ogTOnFVI/AhufNOKOTZaiprcbqMngwHeaILbwEJVqv5P2YUurC6Sq+QaH26tX00DtsYOW+n8AA3Fe48+L6rqa6Q8ru+dhPuR/Apr2+DTZ+npzY8leV3A7mYUeLo4JLaEH6n1TLVt65cX7nuesS7KsSKBSxs7q7bsqeapDcMskZz5JR4EK4S96CuHZEnn6+efpH8IrxWpbeO3EibJfyT8tMJKw7Zym5mPr+sz6xSlVMoBVcCm+7z9e7Zcqukdvdy8jM7sGP9qusqEhpojh0o+gPgr9p13SfR+6OCpj9gwxPCnZO8RBfh1cbz9vFSVlDyUYcpI2CCCpPazA1glTsEDk+VtznnUMmWksXWfxBFV81jQB2SnoZKlnsiEuNMwhiMCMEkRFo74+AsUr2UjmKJ5KC8ACwsn475PoLZBh2J6fyXBHMdbrGfK+Zt5K2IHRRtDaZVT4eHqYzxh8KhalsuBNxCo3FygI=" + diff --git a/reproducible_build.sh b/reproducible_build.sh index 2af7a04d8..d660645ff 100755 --- a/reproducible_build.sh +++ b/reproducible_build.sh @@ -3,26 +3,50 @@ set -euo pipefail function HELP { - echo "Build a lbrycrd" + echo "Build lbrycrd" echo "-----" echo "When run without any arguments, this script expects the current directory" echo "to be the lbrycrd repo and it builds what is in that directory" echo + echo "This is a long build process so it can be split into two parts" + echo "Specify the -d flag to build only the dependencies" + echo "and the -l flag to build only lbrycrd. This will fail" + echo "if the dependencies weren't built earlier" + echo echo "Optional arguments:" echo - echo "-c: clone a fresh copy of the repo" - echo "-h: show help" + echo "-c: don't clone a fresh copy of the repo" + echo "-r: remove intermediate files." + echo "-l: build only lbrycrd" + echo "-d: build only the dependencies" + echo "-o: timeout build after 45 minutes" echo "-t: turn trace on" + echo "-h: show help" exit 1 } -CLONE=false +CLONE=true +CLEAN=false +BUILD_DEPENDENCIES=true +BUILD_LBRYCRD=true +TIMEOUT=false - -while getopts :hctb:w:d: FLAG; do +while getopts :crldoth:w:d: FLAG; do case $FLAG in c) - CLONE=true + CLONE=false + ;; + r) + CLEAN=true + ;; + l) + BUILD_DEPENDENCIES=false + ;; + d) + BUILD_LBRYCRD=false + ;; + o) + TIMEOUT=true ;; t) set -o xtrace @@ -44,86 +68,285 @@ done shift $((OPTIND-1)) SUDO='' -if (( $EUID != 0 )); then +if (( EUID != 0 )); then SUDO='sudo' fi -if [ "$CLONE" = false ]; then - if [ `basename $PWD` != "lbrycrd" ]; then +if [ "${CLONE}" = false ]; then + if [ "$(basename "$PWD")" != "lbrycrd" ]; then echo "Not currently in the lbrycrd directory. Cowardly refusing to go forward" exit 1 fi SOURCE_DIR=$PWD fi -if [ -z ${TRAVIS+x} ]; then - # if not on travis, its nice to see progress - QUIET="" +if [ -z "${TRAVIS_OS_NAME+x}" ]; then + if [ "$(uname -s)" = "Darwin" ]; then + OS_NAME="osx" + else + OS_NAME="linux" + fi else - QUIET="-qq" + OS_NAME="${TRAVIS_OS_NAME}" fi -# get the required OS packages -$SUDO apt-get ${QUIET} update -$SUDO apt-get ${QUIET} install -y --no-install-recommends \ - build-essential python-dev libbz2-dev libtool \ - autotools-dev autoconf git pkg-config wget \ - ca-certificates automake bsdmainutils - -mkdir dependencies -cd dependencies - -wget http://download.oracle.com/berkeley-db/db-4.8.30.NC.tar.gz -tar xf db-4.8.30.NC.tar.gz -export BDB_PREFIX="`pwd`/bdb" -cd db-4.8.30.NC/build_unix -../dist/configure --prefix=$BDB_PREFIX --enable-cxx --disable-shared --with-pic -make -make install -cd ../../ - -wget https://www.openssl.org/source/openssl-1.0.1p.tar.gz -tar xf openssl-1.0.1p.tar.gz -export OPENSSL_PREFIX="`pwd`/openssl_build" -mkdir $OPENSSL_PREFIX -cd openssl-1.0.1p -./Configure --prefix=$OPENSSL_PREFIX --openssldir=$OPENSSL_PREFIX/ssl linux-x86_64 -fPIC -static no-shared no-dso -make -make install -cd .. +if [ -z "${TRAVIS_BUILD_DIR+x}" ]; then + START_TIME_FILE="$PWD/start_time" +else + # if we are on travis (the primary use case for setting a timeout) + # this file is created when the build starts + START_TIME_FILE="$TRAVIS_BUILD_DIR/start_time" +fi +if [ ! -f "${START_TIME_FILE}" ]; then + date +%s > "${START_TIME_FILE}" +fi -wget http://sourceforge.net/projects/boost/files/boost/1.59.0/boost_1_59_0.tar.bz2/download -O boost_1_59_0.tar.bz2 -tar xf boost_1_59_0.tar.bz2 -export BOOST_ROOT="`pwd`/boost_1_59_0" -cd boost_1_59_0 -./bootstrap.sh -./b2 link=static cxxflags=-fPIC stage -cd ../../ +NEXT_TIME=60 +function exit_at_45() { + if [ -f "${START_TIME_FILE}" ]; then + NOW=$(date +%s) + START=$(cat "${START_TIME_FILE}") + TIMEOUT_SECS=2700 # 45 * 60 + TIME=$((NOW - START)) + if (( TIME > NEXT_TIME )); then + echo "Build has taken $((TIME / 60)) minutes: $1" + NEXT_TIME=$((TIME + 60)) + fi + if [ "$TIMEOUT" = true ] && (( TIME > TIMEOUT_SECS )); then + echo 'Exiting at 45 minutes to allow the cache to populate' + exit 1 + fi + fi +} +# two arguments +# - pid (probably from $!) +# - echo message +function wait_and_echo() { + PID=$1 + (set -o | grep xtrace | grep -q on) + TRACE_STATUS=$? + # disable xtrace or else this will get verbose, which is what + # I'm trying to avoid by going through all of this nonsense anyway + set +o xtrace + TIME=0 + SLEEP=5 + # loop until the process is no longer running + # check every $SLEEP seconds, echoing a message every minute + while (ps -p "${PID}" > /dev/null); do + exit_at_45 "$2" + sleep "${SLEEP}" + done + # restore the xtrace setting + if [ "${TRACE_STATUS}" -eq 0 ]; then + set -o xtrace + fi + wait "$PID" + return $? +} -mkdir libevent_build -git clone https://github.com/libevent/libevent.git -export LIBEVENT_PREFIX="`pwd`/libevent_build" -cd libevent -./autogen.sh -./configure --prefix=$LIBEVENT_PREFIX --enable-static --disable-shared --with-pic LDFLAGS="-L${OPENSSL_PREFIX}/lib/" CPPFLAGS="-I${OPENSSL_PREFIX}/include" -make -make install -cd .. +# run a command ($1) in the background +# logging its stdout and stderr to $2 +# and wait until it completed +function background() { + $1 >> "$2" 2>&1 & + wait_and_echo $! "$3" +} + +function cleanup() { + rv=$? + # cat the log file if it exists + if [ -f "$2" ]; then + cat "$2" + fi + # delete the build directory + rm -rf "$1" + echo "Build failed. Removing $1" + exit $rv +} + +function install_brew_packages() { + brew update > /dev/null + brew install autoconf + brew install automake + brew install libtool + brew install pkg-config + brew install protobuf + if ! brew ls | grep gmp --quiet; then + brew install gmp + fi +} + +function install_apt_packages() { + if [ -z "${TRAVIS+x}" ]; then + # if not on travis, its nice to see progress + QUIET="" + else + QUIET="-qq" + fi + # get the required OS packages + $SUDO apt-get ${QUIET} update + $SUDO apt-get ${QUIET} install -y --no-install-recommends \ + build-essential python-dev libbz2-dev libtool \ + autotools-dev autoconf git pkg-config wget \ + ca-certificates automake bsdmainutils +} + +function build_dependencies() { + if [ "${OS_NAME}" = "osx" ]; then + install_brew_packages + else + install_apt_packages + fi + + if [ "$CLEAN" = true ]; then + rm -rf "${LBRYCRD_DEPENDENCIES}" + rm -rf "${OUTPUT_DIR}" + fi + + mkdir -p "${LBRYCRD_DEPENDENCIES}" + mkdir -p "${LOG_DIR}" + + build_dependency "${BDB_PREFIX}" "${LOG_DIR}/bdb_build.log" build_bdb + build_dependency "${OPENSSL_PREFIX}" "${LOG_DIR}/openssl_build.log" build_openssl + + set +u + export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/" + set -u + + build_dependency "${BOOST_PREFIX}" "${LOG_DIR}/boost_build.log" build_boost + build_dependency "${LIBEVENT_PREFIX}" "${LOG_DIR}/libevent_build.log" build_libevent +} + +function build_bdb() { + BDB_LOG="$1" + if [ ! -f db-4.8.30.NC.tar.gz ]; then + wget http://download.oracle.com/berkeley-db/db-4.8.30.NC.tar.gz + fi + tar xf db-4.8.30.NC.tar.gz + if [ "${OS_NAME}" = "osx" ]; then + curl -OL https://raw.github.com/narkoleptik/os-x-berkeleydb-patch/master/atomic.patch + patch db-4.8.30.NC/dbinc/atomic.h < atomic.patch + fi + cd db-4.8.30.NC/build_unix + echo "Building bdb. tail -f $BDB_LOG to see the details and monitor progress" + ../dist/configure --prefix="${BDB_PREFIX}" --enable-cxx --disable-shared --with-pic > "${BDB_LOG}" + background make "${BDB_LOG}" "Waiting for bdb to finish building" + make install >> "${BDB_LOG}" 2>&1 +} + +function build_openssl() { + OPENSSL_LOG="$1" + wget https://www.openssl.org/source/openssl-1.0.1p.tar.gz + tar xf openssl-1.0.1p.tar.gz + mkdir -p "${OPENSSL_PREFIX}/ssl" + cd openssl-1.0.1p + echo "Building openssl. tail -f $OPENSSL_LOG to see the details and monitor progress" + if [ "${OS_NAME}" = "osx" ]; then + ./Configure --prefix="${OPENSSL_PREFIX}" --openssldir="${OPENSSL_PREFIX}/ssl" \ + -fPIC darwin64-x86_64-cc \ + no-shared no-dso no-engines > "${OPENSSL_LOG}" + else + ./Configure --prefix="${OPENSSL_PREFIX}" --openssldir="${OPENSSL_PREFIX}/ssl" \ + linux-x86_64 -fPIC -static no-shared no-dso > "${OPENSSL_LOG}" + fi + background make "${OPENSSL_LOG}" "Waiting for openssl to finish building" + make install >> "${OPENSSL_LOG}" 2>&1 +} + +function build_boost() { + BOOST_LOG="$1" + wget http://sourceforge.net/projects/boost/files/boost/1.59.0/boost_1_59_0.tar.bz2/download \ + -O boost_1_59_0.tar.bz2 + tar xf boost_1_59_0.tar.bz2 + cd boost_1_59_0 + echo "Building Boost. tail -f ${BOOST_LOG} to see the details and monitor progress" + ./bootstrap.sh --prefix="${BOOST_PREFIX}" > "${BOOST_LOG}" 2>&1 + background "./b2 link=static cxxflags=-fPIC install" \ + "${BOOST_LOG}" \ + "Waiting for boost to finish building" +} + +function build_libevent() { + LIBEVENT_LOG="$1" + git clone https://github.com/libevent/libevent.git + cd libevent + echo "Building libevent. tail -f ${LIBEVENT_LOG} to see the details and monitor progress" + ./autogen.sh > "${LIBEVENT_LOG}" 2>&1 + ./configure --prefix="${LIBEVENT_PREFIX}" --enable-static --disable-shared --with-pic \ + LDFLAGS="-L${OPENSSL_PREFIX}/lib/" \ + CPPFLAGS="-I${OPENSSL_PREFIX}/include" >> "${LIBEVENT_LOG}" 2>&1 + background make "${LIBEVENT_LOG}" "Waiting for libevent to finish building" + make install >> "${LIBEVENT_LOG}" +} + +function build_dependency() { + PREFIX=$1 + LOG=$2 + BUILD=$3 + if [ ! -d "${PREFIX}" ]; then + trap 'cleanup "${PREFIX}" "${LOG}"' INT TERM EXIT + cd "${LBRYCRD_DEPENDENCIES}" + mkdir -p "${PREFIX}" + "${BUILD}" "${LOG}" + trap - INT TERM EXIT + fi +} + +function build_lbrycrd() { + if [ "$CLONE" == true ]; then + cd "${LBRYCRD_DEPENDENCIES}" + git clone https://github.com/lbryio/lbrycrd + cd lbrycrd + else + cd "${SOURCE_DIR}" + fi + ./autogen.sh + LDFLAGS="-L${OPENSSL_PREFIX}/lib/ -L${BDB_PREFIX}/lib/ -L${LIBEVENT_PREFIX}/lib/ -static-libstdc++" + CPPFLAGS="-I${OPENSSL_PREFIX}/include -I${BDB_PREFIX}/include -I${LIBEVENT_PREFIX}/include/" + if [ "${OS_NAME}" = "osx" ]; then + ./configure --without-gui --enable-cxx --enable-static --disable-shared --with-pic \ + --with-boost="${BOOST_PREFIX}" \ + LDFLAGS="${LDFLAGS}" \ + CPPFLAGS="${CPPFLAGS}" + else + ./configure --without-gui --with-boost="${BOOST_PREFIX}" \ + LDFLAGS="${LDFLAGS}" \ + CPPFLAGS="${CPPFLAGS}" + fi + make + # tests don't work on OSX. Should definitely figure out why + # that is but, for now, not letting that stop the rest + # of the build + if [ "${OS_NAME}" = "linux" ]; then + src/test/test_lbrycrd + fi + strip src/lbrycrdd + strip src/lbrycrd-cli + strip src/lbrycrd-tx + strip src/test/test_lbrycrd +} + +# these variables are needed in both functions +LBRYCRD_DEPENDENCIES="$(pwd)/lbrycrd-dependencies" +OUTPUT_DIR="$(pwd)/build" +LOG_DIR="$(pwd)/logs" +BDB_PREFIX="${OUTPUT_DIR}/bdb" +OPENSSL_PREFIX="${OUTPUT_DIR}/openssl" +BOOST_PREFIX="${OUTPUT_DIR}/boost" +LIBEVENT_PREFIX="${OUTPUT_DIR}/libevent" + +if [ "${BUILD_DEPENDENCIES}" = true ]; then + build_dependencies +fi set +u -export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/:${LIBEVENT_PREFIX}/lib/pkgconfig" +export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/:${LIBEVENT_PREFIX}/lib/pkgconfig/" set -u -if [ "$CLONE" == true ]; then - git clone https://github.com/lbryio/lbrycrd - cd lbrycrd +if [ "${BUILD_LBRYCRD}" = true ]; then + LBRYCRD_LOG="${LOG_DIR}/lbrycrd_build.log" + echo "Building lbrycrd. tail -f ${LBRYCRD_LOG} to see the details and monitor progress" + background build_lbrycrd "${LBRYCRD_LOG}" "Waiting for lbrycrd to finish building" fi -./autogen.sh -./configure --without-gui LDFLAGS="-L${OPENSSL_PREFIX}/lib/ -L${BDB_PREFIX}/lib/ -L${LIBEVENT_PREFIX}/lib/ -static-libstdc++" CPPFLAGS="-I${OPENSSL_PREFIX}/include -I${BDB_PREFIX}/include -I${LIBEVENT_PREFIX}/include/" -make -strip src/lbrycrdd -strip src/lbrycrd-cli -strip src/lbrycrd-tx -strip src/test/test_lbrycrd diff --git a/reproducible_build_osx.sh b/reproducible_build_osx.sh deleted file mode 100644 index f56f125b9..000000000 --- a/reproducible_build_osx.sh +++ /dev/null @@ -1,72 +0,0 @@ -brew update -brew install autoconf -brew install automake -brew install libtool -brew install pkg-config -brew install protobuf -brew install gmp - -mkdir dependencies -cd dependencies -export LBRYCRD_DEPENDENCIES="`pwd`" - -#download, patch, and build bdb -wget http://download.oracle.com/berkeley-db/db-4.8.30.NC.tar.gz -tar xf db-4.8.30.NC.tar.gz -export BDB_PREFIX="`pwd`/bdb" -curl -OL https://raw.github.com/narkoleptik/os-x-berkeleydb-patch/master/atomic.patch -patch db-4.8.30.NC/dbinc/atomic.h < atomic.patch -cd db-4.8.30.NC/build_unix -../dist/configure --prefix=$BDB_PREFIX --enable-cxx --disable-shared --with-pic -make -make install - -#download and build openssl -cd $LBRYCRD_DEPENDENCIES -wget https://www.openssl.org/source/openssl-1.0.1p.tar.gz -tar xf openssl-1.0.1p.tar.gz -export OPENSSL_PREFIX="`pwd`/openssl_build" -mkdir $OPENSSL_PREFIX -mkdir $OPENSSL_PREFIX/ssl -cd openssl-1.0.1p -./Configure --prefix=$OPENSSL_PREFIX --openssldir=$OPENSSL_PREFIX/ssl -fPIC darwin64-x86_64-cc no-shared no-dso no-engines -make -make install -export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/" - -#download and build boost -cd $LBRYCRD_DEPENDENCIES -wget http://sourceforge.net/projects/boost/files/boost/1.59.0/boost_1_59_0.tar.bz2/download -O boost_1_59_0.tar.bz2 -tar xf boost_1_59_0.tar.bz2 -export BOOST_ROOT="`pwd`/boost_1_59_0" -cd boost_1_59_0 -./bootstrap.sh -./b2 link=static cxxflags=-fPIC stage - -#download and build libevent -cd $LBRYCRD_DEPENDENCIES -mkdir libevent_build -git clone https://github.com/libevent/libevent.git -export LIBEVENT_PREFIX="`pwd`/libevent_build" -cd libevent -./autogen.sh -./configure --prefix=$LIBEVENT_PREFIX --enable-static --disable-shared --with-pic LDFLAGS="-L${OPENSSL_PREFIX}/lib/" CPPFLAGS="-I${OPENSSL_PREFIX}/include" -make -make install -export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${LIBEVENT_PREFIX}/lib/pkgconfig/" - -#download and build lbrycrd -cd $LBRYCRD_DEPENDENCIES -git clone https://github.com/lbryio/lbrycrd -cd lbrycrd -git checkout real2 -./autogen.sh -./configure --without-gui --enable-cxx --enable-static --disable-shared --with-pic \ - LDFLAGS="-L${OPENSSL_PREFIX}/lib/ -L${BDB_PREFIX}/lib/ -L${LIBEVENT_PREFIX}/lib/ -static-libstdc++" \ - CPPFLAGS="-I${OPENSSL_PREFIX}/include -I${BDB_PREFIX}/include -I${LIBEVENT_PREFIX}/include/" - -make -strip src/lbrycrdd -strip src/lbrycrd-cli -strip src/lbrycrd-tx -strip src/test/test-lbrycrd \ No newline at end of file