diff --git a/.travis.yml b/.travis.yml index 8dcde0ca4..fccfbeee6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,31 @@ -sudo: required -dist: trusty +matrix: + include: + - os: linux + sudo: required + dist: trusty + language: c + - os: osx + language: c + osx_image: xcode7.3 + +cache: + ccache: true + directories: + - build + +before_install: + - date +%s > "${TRAVIS_BUILD_DIR}/start_time" + - ls -lh build + - du -h -d 2 build install: true -script: - - ./reproducible_build.sh -t - - zip -j lbrycrd-linux.zip src/lbrycrdd src/lbrycrd-cli src/lbrycrd-tx +script: + - ./reproducible_build.sh -t -o -c + - if [[ "${TRAVIS_OS_NAME}" = "linux" ]]; then zip -j lbrycrd-linux.zip src/lbrycrdd src/lbrycrd-cli src/lbrycrd-tx; fi + +before_cache: + - ls -lh build + - du -h -d 2 build deploy: - provider: releases @@ -12,5 +33,7 @@ deploy: skip_cleanup: true on: tags: true + condition: "$TRAVIS_OS_NAME = linux" api_key: secure: "ibN+PQg84f0tgJLV3KaHjkHmgFNOPqBLIrSU42moen22xxT2VIn0j7DkFaLLts1fs/+/gxmrvhOQ0vUFJqmflZ3kCeyedP/CVgZz7D7HRVK0cYUMJ2F9VWMZAFIX6A67OUlypqZqzKAKrZHx3HXky6Z8NBU23JgWqDm8PcrbbgMLvB2lvWVASjJVKreaL9BOxFOOOrAj6xAIm9ogTOnFVI/AhufNOKOTZaiprcbqMngwHeaILbwEJVqv5P2YUurC6Sq+QaH26tX00DtsYOW+n8AA3Fe48+L6rqa6Q8ru+dhPuR/Apr2+DTZ+npzY8leV3A7mYUeLo4JLaEH6n1TLVt65cX7nuesS7KsSKBSxs7q7bsqeapDcMskZz5JR4EK4S96CuHZEnn6+efpH8IrxWpbeO3EibJfyT8tMJKw7Zym5mPr+sz6xSlVMoBVcCm+7z9e7Zcqukdvdy8jM7sGP9qusqEhpojh0o+gPgr9p13SfR+6OCpj9gwxPCnZO8RBfh1cbz9vFSVlDyUYcpI2CCCpPazA1glTsEDk+VtznnUMmWksXWfxBFV81jQB2SnoZKlnsiEuNMwhiMCMEkRFo74+AsUr2UjmKJ5KC8ACwsn475PoLZBh2J6fyXBHMdbrGfK+Zt5K2IHRRtDaZVT4eHqYzxh8KhalsuBNxCo3FygI=" + diff --git a/reproducible_build.sh b/reproducible_build.sh index 2af7a04d8..e6008f8a4 100755 --- a/reproducible_build.sh +++ b/reproducible_build.sh @@ -3,26 +3,50 @@ set -euo pipefail function HELP { - echo "Build a lbrycrd" + echo "Build lbrycrd" echo "-----" echo "When run without any arguments, this script expects the current directory" echo "to be the lbrycrd repo and it builds what is in that directory" echo + echo "This is a long build process so it can be split into two parts" + echo "Specify the -d flag to build only the dependencies" + echo "and the -l flag to build only lbrycrd. This will fail" + echo "if the dependencies weren't built earlier" + echo echo "Optional arguments:" echo - echo "-c: clone a fresh copy of the repo" - echo "-h: show help" + echo "-c: don't clone a fresh copy of the repo" + echo "-r: remove intermediate files." + echo "-l: build only lbrycrd" + echo "-d: build only the dependencies" + echo "-o: timeout build after 45 minutes" echo "-t: turn trace on" + echo "-h: show help" exit 1 } -CLONE=false +CLONE=true +CLEAN=false +BUILD_DEPENDENCIES=true +BUILD_LBRYCRD=true +TIMEOUT=false - -while getopts :hctb:w:d: FLAG; do +while getopts :crldoth:w:d: FLAG; do case $FLAG in c) - CLONE=true + CLONE=false + ;; + r) + CLEAN=true + ;; + l) + BUILD_DEPENDENCIES=false + ;; + d) + BUILD_LBRYCRD=false + ;; + o) + TIMEOUT=true ;; t) set -o xtrace @@ -44,86 +68,277 @@ done shift $((OPTIND-1)) SUDO='' -if (( $EUID != 0 )); then +if (( EUID != 0 )); then SUDO='sudo' fi -if [ "$CLONE" = false ]; then - if [ `basename $PWD` != "lbrycrd" ]; then +if [ "${CLONE}" = false ]; then + if [ "$(basename "$PWD")" != "lbrycrd" ]; then echo "Not currently in the lbrycrd directory. Cowardly refusing to go forward" exit 1 fi SOURCE_DIR=$PWD fi -if [ -z ${TRAVIS+x} ]; then - # if not on travis, its nice to see progress - QUIET="" +if [ -z "${TRAVIS_OS_NAME+x}" ]; then + if [ "$(uname -s)" = "Darwin" ]; then + OS_NAME="osx" + else + OS_NAME="linux" + fi else - QUIET="-qq" + OS_NAME="${TRAVIS_OS_NAME}" fi -# get the required OS packages -$SUDO apt-get ${QUIET} update -$SUDO apt-get ${QUIET} install -y --no-install-recommends \ - build-essential python-dev libbz2-dev libtool \ - autotools-dev autoconf git pkg-config wget \ - ca-certificates automake bsdmainutils - -mkdir dependencies -cd dependencies - -wget http://download.oracle.com/berkeley-db/db-4.8.30.NC.tar.gz -tar xf db-4.8.30.NC.tar.gz -export BDB_PREFIX="`pwd`/bdb" -cd db-4.8.30.NC/build_unix -../dist/configure --prefix=$BDB_PREFIX --enable-cxx --disable-shared --with-pic -make -make install -cd ../../ - -wget https://www.openssl.org/source/openssl-1.0.1p.tar.gz -tar xf openssl-1.0.1p.tar.gz -export OPENSSL_PREFIX="`pwd`/openssl_build" -mkdir $OPENSSL_PREFIX -cd openssl-1.0.1p -./Configure --prefix=$OPENSSL_PREFIX --openssldir=$OPENSSL_PREFIX/ssl linux-x86_64 -fPIC -static no-shared no-dso -make -make install -cd .. +if [ -z "${TRAVIS_BUILD_DIR+x}" ]; then + START_TIME_FILE="$PWD/start_time" +else + # if we are on travis (the primary use case for setting a timeout) + # this file is created when the build starts + START_TIME_FILE="$TRAVIS_BUILD_DIR/start_time" +fi +if [ ! -f "${START_TIME_FILE}" ]; then + date +%s > "${START_TIME_FILE}" +fi -wget http://sourceforge.net/projects/boost/files/boost/1.59.0/boost_1_59_0.tar.bz2/download -O boost_1_59_0.tar.bz2 -tar xf boost_1_59_0.tar.bz2 -export BOOST_ROOT="`pwd`/boost_1_59_0" -cd boost_1_59_0 -./bootstrap.sh -./b2 link=static cxxflags=-fPIC stage -cd ../../ +NEXT_TIME=60 +function exit_at_45() { + if [ -f "${START_TIME_FILE}" ]; then + NOW=$(date +%s) + START=$(cat "${START_TIME_FILE}") + TIMEOUT_SECS=2700 # 45 * 60 + TIME=$((NOW - START)) + if (( TIME > NEXT_TIME )); then + echo "Build has taken $((TIME / 60)) minutes: $1" + NEXT_TIME=$((TIME + 60)) + fi + if [ "$TIMEOUT" = true ] && (( TIME > TIMEOUT_SECS )); then + echo 'Exiting at 45 minutes to allow the cache to populate' + exit 1 + fi + fi +} +# two arguments +# - pid (probably from $!) +# - echo message +function wait_and_echo() { + PID=$1 + (set -o | grep xtrace | grep -q on) + TRACE_STATUS=$? + # disable xtrace or else this will get verbose, which is what + # I'm trying to avoid by going through all of this nonsense anyway + set +o xtrace + TIME=0 + SLEEP=5 + # loop until the process is no longer running + # check every $SLEEP seconds, echoing a message every minute + while (ps -p "${PID}" > /dev/null); do + exit_at_45 "$2" + sleep "${SLEEP}" + done + # restore the xtrace setting + if [ "${TRACE_STATUS}" -eq 0 ]; then + set -o xtrace + fi + wait "$PID" + return $? +} -mkdir libevent_build -git clone https://github.com/libevent/libevent.git -export LIBEVENT_PREFIX="`pwd`/libevent_build" -cd libevent -./autogen.sh -./configure --prefix=$LIBEVENT_PREFIX --enable-static --disable-shared --with-pic LDFLAGS="-L${OPENSSL_PREFIX}/lib/" CPPFLAGS="-I${OPENSSL_PREFIX}/include" -make -make install -cd .. +# run a command ($1) in the background +# logging its stdout and stderr to $2 +# and wait until it completed +function background() { + $1 >> "$2" 2>&1 & + wait_and_echo $! "$3" +} + +function cleanup() { + rv=$? + # cat the log file if it exists + if [ -f "$2" ]; then + cat "$2" + fi + # delete the build directory + rm -rf "$1" + echo "Build failed. Removing $1" + exit $rv +} + +function install_brew_packages() { + brew update > /dev/null + brew install autoconf + brew install automake + brew install libtool + brew install pkg-config + brew install protobuf + if ! brew ls | grep gmp --quiet; then + brew install gmp + fi +} + +function install_apt_packages() { + if [ -z "${TRAVIS+x}" ]; then + # if not on travis, its nice to see progress + QUIET="" + else + QUIET="-qq" + fi + # get the required OS packages + $SUDO apt-get ${QUIET} update + $SUDO apt-get ${QUIET} install -y --no-install-recommends \ + build-essential python-dev libbz2-dev libtool \ + autotools-dev autoconf git pkg-config wget \ + ca-certificates automake bsdmainutils +} + +function build_dependencies() { + if [ "${OS_NAME}" = "osx" ]; then + install_brew_packages + else + install_apt_packages + fi + + if [ "$CLEAN" = true ]; then + rm -rf "${LBRYCRD_DEPENDENCIES}" + rm -rf "${OUTPUT_DIR}" + fi + + if [ ! -d "${LBRYCRD_DEPENDENCIES}" ]; then + git clone https://github.com/lbryio/lbrycrd-dependencies.git "${LBRYCRD_DEPENDENCIES}" + fi + mkdir -p "${LOG_DIR}" + + build_dependency "${BDB_PREFIX}" "${LOG_DIR}/bdb_build.log" build_bdb + build_dependency "${OPENSSL_PREFIX}" "${LOG_DIR}/openssl_build.log" build_openssl + + set +u + export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/" + set -u + + build_dependency "${BOOST_PREFIX}" "${LOG_DIR}/boost_build.log" build_boost + build_dependency "${LIBEVENT_PREFIX}" "${LOG_DIR}/libevent_build.log" build_libevent +} + +function build_bdb() { + BDB_LOG="$1" + if [ "${OS_NAME}" = "osx" ]; then + patch db-4.8.30.NC/dbinc/atomic.h < atomic.patch + fi + cd db-4.8.30.NC/build_unix + echo "Building bdb. tail -f $BDB_LOG to see the details and monitor progress" + ../dist/configure --prefix="${BDB_PREFIX}" --enable-cxx --disable-shared --with-pic > "${BDB_LOG}" + background make "${BDB_LOG}" "Waiting for bdb to finish building" + make install >> "${BDB_LOG}" 2>&1 +} + +function build_openssl() { + OPENSSL_LOG="$1" + mkdir -p "${OPENSSL_PREFIX}/ssl" + cd openssl-1.0.1p + echo "Building openssl. tail -f $OPENSSL_LOG to see the details and monitor progress" + if [ "${OS_NAME}" = "osx" ]; then + ./Configure --prefix="${OPENSSL_PREFIX}" --openssldir="${OPENSSL_PREFIX}/ssl" \ + -fPIC darwin64-x86_64-cc \ + no-shared no-dso no-engines > "${OPENSSL_LOG}" + else + ./Configure --prefix="${OPENSSL_PREFIX}" --openssldir="${OPENSSL_PREFIX}/ssl" \ + linux-x86_64 -fPIC -static no-shared no-dso > "${OPENSSL_LOG}" + fi + background make "${OPENSSL_LOG}" "Waiting for openssl to finish building" + make install >> "${OPENSSL_LOG}" 2>&1 +} + +function build_boost() { + BOOST_LOG="$1" + cd boost_1_59_0 + echo "Building Boost. tail -f ${BOOST_LOG} to see the details and monitor progress" + ./bootstrap.sh --prefix="${BOOST_PREFIX}" > "${BOOST_LOG}" 2>&1 + background "./b2 link=static cxxflags=-fPIC install" \ + "${BOOST_LOG}" \ + "Waiting for boost to finish building" +} + +function build_libevent() { + LIBEVENT_LOG="$1" + git clone https://github.com/libevent/libevent.git + cd libevent + echo "Building libevent. tail -f ${LIBEVENT_LOG} to see the details and monitor progress" + ./autogen.sh > "${LIBEVENT_LOG}" 2>&1 + ./configure --prefix="${LIBEVENT_PREFIX}" --enable-static --disable-shared --with-pic \ + LDFLAGS="-L${OPENSSL_PREFIX}/lib/" \ + CPPFLAGS="-I${OPENSSL_PREFIX}/include" >> "${LIBEVENT_LOG}" 2>&1 + background make "${LIBEVENT_LOG}" "Waiting for libevent to finish building" + make install >> "${LIBEVENT_LOG}" +} + +function build_dependency() { + PREFIX=$1 + LOG=$2 + BUILD=$3 + if [ ! -d "${PREFIX}" ]; then + trap 'cleanup "${PREFIX}" "${LOG}"' INT TERM EXIT + cd "${LBRYCRD_DEPENDENCIES}" + mkdir -p "${PREFIX}" + "${BUILD}" "${LOG}" + trap - INT TERM EXIT + fi +} + +function build_lbrycrd() { + if [ "$CLONE" == true ]; then + cd "${LBRYCRD_DEPENDENCIES}" + git clone https://github.com/lbryio/lbrycrd + cd lbrycrd + else + cd "${SOURCE_DIR}" + fi + ./autogen.sh + LDFLAGS="-L${OPENSSL_PREFIX}/lib/ -L${BDB_PREFIX}/lib/ -L${LIBEVENT_PREFIX}/lib/ -static-libstdc++" + CPPFLAGS="-I${OPENSSL_PREFIX}/include -I${BDB_PREFIX}/include -I${LIBEVENT_PREFIX}/include/" + if [ "${OS_NAME}" = "osx" ]; then + ./configure --without-gui --enable-cxx --enable-static --disable-shared --with-pic \ + --with-boost="${BOOST_PREFIX}" \ + LDFLAGS="${LDFLAGS}" \ + CPPFLAGS="${CPPFLAGS}" + else + ./configure --without-gui --with-boost="${BOOST_PREFIX}" \ + LDFLAGS="${LDFLAGS}" \ + CPPFLAGS="${CPPFLAGS}" + fi + make + # tests don't work on OSX. Should definitely figure out why + # that is but, for now, not letting that stop the rest + # of the build + if [ "${OS_NAME}" = "linux" ]; then + src/test/test_lbrycrd + fi + strip src/lbrycrdd + strip src/lbrycrd-cli + strip src/lbrycrd-tx + strip src/test/test_lbrycrd +} + +# these variables are needed in both functions +LBRYCRD_DEPENDENCIES="$(pwd)/lbrycrd-dependencies" +OUTPUT_DIR="$(pwd)/build" +LOG_DIR="$(pwd)/logs" +BDB_PREFIX="${OUTPUT_DIR}/bdb" +OPENSSL_PREFIX="${OUTPUT_DIR}/openssl" +BOOST_PREFIX="${OUTPUT_DIR}/boost" +LIBEVENT_PREFIX="${OUTPUT_DIR}/libevent" + +if [ "${BUILD_DEPENDENCIES}" = true ]; then + build_dependencies +fi set +u -export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/:${LIBEVENT_PREFIX}/lib/pkgconfig" +export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/:${LIBEVENT_PREFIX}/lib/pkgconfig/" set -u -if [ "$CLONE" == true ]; then - git clone https://github.com/lbryio/lbrycrd - cd lbrycrd +if [ "${BUILD_LBRYCRD}" = true ]; then + LBRYCRD_LOG="${LOG_DIR}/lbrycrd_build.log" + echo "Building lbrycrd. tail -f ${LBRYCRD_LOG} to see the details and monitor progress" + background build_lbrycrd "${LBRYCRD_LOG}" "Waiting for lbrycrd to finish building" fi -./autogen.sh -./configure --without-gui LDFLAGS="-L${OPENSSL_PREFIX}/lib/ -L${BDB_PREFIX}/lib/ -L${LIBEVENT_PREFIX}/lib/ -static-libstdc++" CPPFLAGS="-I${OPENSSL_PREFIX}/include -I${BDB_PREFIX}/include -I${LIBEVENT_PREFIX}/include/" -make -strip src/lbrycrdd -strip src/lbrycrd-cli -strip src/lbrycrd-tx -strip src/test/test_lbrycrd diff --git a/reproducible_build_osx.sh b/reproducible_build_osx.sh deleted file mode 100644 index f56f125b9..000000000 --- a/reproducible_build_osx.sh +++ /dev/null @@ -1,72 +0,0 @@ -brew update -brew install autoconf -brew install automake -brew install libtool -brew install pkg-config -brew install protobuf -brew install gmp - -mkdir dependencies -cd dependencies -export LBRYCRD_DEPENDENCIES="`pwd`" - -#download, patch, and build bdb -wget http://download.oracle.com/berkeley-db/db-4.8.30.NC.tar.gz -tar xf db-4.8.30.NC.tar.gz -export BDB_PREFIX="`pwd`/bdb" -curl -OL https://raw.github.com/narkoleptik/os-x-berkeleydb-patch/master/atomic.patch -patch db-4.8.30.NC/dbinc/atomic.h < atomic.patch -cd db-4.8.30.NC/build_unix -../dist/configure --prefix=$BDB_PREFIX --enable-cxx --disable-shared --with-pic -make -make install - -#download and build openssl -cd $LBRYCRD_DEPENDENCIES -wget https://www.openssl.org/source/openssl-1.0.1p.tar.gz -tar xf openssl-1.0.1p.tar.gz -export OPENSSL_PREFIX="`pwd`/openssl_build" -mkdir $OPENSSL_PREFIX -mkdir $OPENSSL_PREFIX/ssl -cd openssl-1.0.1p -./Configure --prefix=$OPENSSL_PREFIX --openssldir=$OPENSSL_PREFIX/ssl -fPIC darwin64-x86_64-cc no-shared no-dso no-engines -make -make install -export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/" - -#download and build boost -cd $LBRYCRD_DEPENDENCIES -wget http://sourceforge.net/projects/boost/files/boost/1.59.0/boost_1_59_0.tar.bz2/download -O boost_1_59_0.tar.bz2 -tar xf boost_1_59_0.tar.bz2 -export BOOST_ROOT="`pwd`/boost_1_59_0" -cd boost_1_59_0 -./bootstrap.sh -./b2 link=static cxxflags=-fPIC stage - -#download and build libevent -cd $LBRYCRD_DEPENDENCIES -mkdir libevent_build -git clone https://github.com/libevent/libevent.git -export LIBEVENT_PREFIX="`pwd`/libevent_build" -cd libevent -./autogen.sh -./configure --prefix=$LIBEVENT_PREFIX --enable-static --disable-shared --with-pic LDFLAGS="-L${OPENSSL_PREFIX}/lib/" CPPFLAGS="-I${OPENSSL_PREFIX}/include" -make -make install -export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${LIBEVENT_PREFIX}/lib/pkgconfig/" - -#download and build lbrycrd -cd $LBRYCRD_DEPENDENCIES -git clone https://github.com/lbryio/lbrycrd -cd lbrycrd -git checkout real2 -./autogen.sh -./configure --without-gui --enable-cxx --enable-static --disable-shared --with-pic \ - LDFLAGS="-L${OPENSSL_PREFIX}/lib/ -L${BDB_PREFIX}/lib/ -L${LIBEVENT_PREFIX}/lib/ -static-libstdc++" \ - CPPFLAGS="-I${OPENSSL_PREFIX}/include -I${BDB_PREFIX}/include -I${LIBEVENT_PREFIX}/include/" - -make -strip src/lbrycrdd -strip src/lbrycrd-cli -strip src/lbrycrd-tx -strip src/test/test-lbrycrd \ No newline at end of file