3 changed files with 320 additions and 146 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -1,10 +1,31 @@
-sudo: required
-dist: trusty
+matrix:
+    include:
+        - os: linux
+          sudo: required
+          dist: trusty
+          language: c
+        - os: osx
+          language: c
+          osx_image: xcode7.3
+
+cache:
+  ccache: true
+  directories:
+    - build
+
+before_install:
+  - date +%s > "${TRAVIS_BUILD_DIR}/start_time"
+  - ls -lh build
+  - du -h -d 2 build

 install: true
-script: 
-  - ./reproducible_build.sh -t
-  - zip -j lbrycrd-linux.zip src/lbrycrdd src/lbrycrd-cli src/lbrycrd-tx
+script:
+  - ./reproducible_build.sh -t -o -c
+  - if [[ "${TRAVIS_OS_NAME}" = "linux" ]]; then zip -j lbrycrd-linux.zip src/lbrycrdd src/lbrycrd-cli src/lbrycrd-tx; fi
+
+before_cache:
+  - ls -lh build
+  - du -h -d 2 build

 deploy:
  - provider: releases
@ -12,5 +33,7 @@ deploy:
    skip_cleanup: true
    on:
      tags: true
+      condition: "$TRAVIS_OS_NAME = linux"
    api_key:
      secure: "ibN+PQg84f0tgJLV3KaHjkHmgFNOPqBLIrSU42moen22xxT2VIn0j7DkFaLLts1fs/+/gxmrvhOQ0vUFJqmflZ3kCeyedP/CVgZz7D7HRVK0cYUMJ2F9VWMZAFIX6A67OUlypqZqzKAKrZHx3HXky6Z8NBU23JgWqDm8PcrbbgMLvB2lvWVASjJVKreaL9BOxFOOOrAj6xAIm9ogTOnFVI/AhufNOKOTZaiprcbqMngwHeaILbwEJVqv5P2YUurC6Sq+QaH26tX00DtsYOW+n8AA3Fe48+L6rqa6Q8ru+dhPuR/Apr2+DTZ+npzY8leV3A7mYUeLo4JLaEH6n1TLVt65cX7nuesS7KsSKBSxs7q7bsqeapDcMskZz5JR4EK4S96CuHZEnn6+efpH8IrxWpbeO3EibJfyT8tMJKw7Zym5mPr+sz6xSlVMoBVcCm+7z9e7Zcqukdvdy8jM7sGP9qusqEhpojh0o+gPgr9p13SfR+6OCpj9gwxPCnZO8RBfh1cbz9vFSVlDyUYcpI2CCCpPazA1glTsEDk+VtznnUMmWksXWfxBFV81jQB2SnoZKlnsiEuNMwhiMCMEkRFo74+AsUr2UjmKJ5KC8ACwsn475PoLZBh2J6fyXBHMdbrGfK+Zt5K2IHRRtDaZVT4eHqYzxh8KhalsuBNxCo3FygI="
+
--- a/reproducible_build.sh
+++ b/reproducible_build.sh
@ -3,26 +3,50 @@
 set -euo pipefail

 function HELP {
-    echo "Build a lbrycrd"
+    echo "Build lbrycrd"
    echo "-----"
    echo "When run without any arguments, this script expects the current directory"
    echo "to be the lbrycrd repo and it builds what is in that directory"
    echo
+    echo "This is a long build process so it can be split into two parts"
+    echo "Specify the -d flag to build only the dependencies"
+    echo "and the -l flag to build only lbrycrd. This will fail"
+    echo "if the dependencies weren't built earlier"
+    echo
    echo "Optional arguments:"
    echo
-    echo "-c: clone a fresh copy of the repo"
-    echo "-h: show help"
+    echo "-c: don't clone a fresh copy of the repo"
+    echo "-r: remove intermediate files."
+    echo "-l: build only lbrycrd"
+    echo "-d: build only the dependencies"
+    echo "-o: timeout build after 45 minutes"
    echo "-t: turn trace on"
+    echo "-h: show help"
    exit 1
 }

-CLONE=false
+CLONE=true
+CLEAN=false
+BUILD_DEPENDENCIES=true
+BUILD_LBRYCRD=true
+TIMEOUT=false

-
-while getopts :hctb:w:d: FLAG; do
+while getopts :crldoth:w:d: FLAG; do
    case $FLAG in
 	c)
-	    CLONE=true
+	    CLONE=false
+	    ;;
+	r)
+	    CLEAN=true
+	    ;;
+	l)
+	    BUILD_DEPENDENCIES=false
+	    ;;
+	d)
+	    BUILD_LBRYCRD=false
+	    ;;
+	o)
+	    TIMEOUT=true
 	    ;;
 	t)
 	    set -o xtrace
@ -44,86 +68,285 @@ done
 shift $((OPTIND-1))

 SUDO=''
-if (( $EUID != 0 )); then
+if (( EUID != 0 )); then
    SUDO='sudo'
 fi

-if [ "$CLONE" = false ]; then
-    if [ `basename $PWD` != "lbrycrd" ]; then
+if [ "${CLONE}" = false ]; then
+    if [ "$(basename "$PWD")" != "lbrycrd" ]; then
 	echo "Not currently in the lbrycrd directory. Cowardly refusing to go forward"
 	exit 1
    fi
    SOURCE_DIR=$PWD
 fi

-if [ -z ${TRAVIS+x} ]; then
-  # if not on travis, its nice to see progress
-  QUIET=""
+if [ -z "${TRAVIS_OS_NAME+x}" ]; then
+    if [ "$(uname -s)" = "Darwin" ]; then
+	OS_NAME="osx"
+    else
+	OS_NAME="linux"
+    fi
 else
-    QUIET="-qq"
+    OS_NAME="${TRAVIS_OS_NAME}"
 fi

-# get the required OS packages
-$SUDO apt-get ${QUIET} update
-$SUDO apt-get ${QUIET} install -y --no-install-recommends \
-    build-essential python-dev libbz2-dev libtool \
-    autotools-dev autoconf git pkg-config wget \
-    ca-certificates automake bsdmainutils
-
-mkdir dependencies
-cd dependencies
-
-wget http://download.oracle.com/berkeley-db/db-4.8.30.NC.tar.gz
-tar xf db-4.8.30.NC.tar.gz
-export BDB_PREFIX="`pwd`/bdb"
-cd db-4.8.30.NC/build_unix
-../dist/configure --prefix=$BDB_PREFIX --enable-cxx --disable-shared --with-pic
-make
-make install
-cd ../../
-
-wget https://www.openssl.org/source/openssl-1.0.1p.tar.gz
-tar xf openssl-1.0.1p.tar.gz
-export OPENSSL_PREFIX="`pwd`/openssl_build"
-mkdir $OPENSSL_PREFIX
-cd openssl-1.0.1p
-./Configure --prefix=$OPENSSL_PREFIX --openssldir=$OPENSSL_PREFIX/ssl linux-x86_64 -fPIC -static no-shared no-dso
-make
-make install
-cd ..
+if [ -z "${TRAVIS_BUILD_DIR+x}" ]; then
+    START_TIME_FILE="$PWD/start_time"
+else
+    # if we are on travis (the primary use case for setting a timeout)
+    # this file is created when the build starts
+    START_TIME_FILE="$TRAVIS_BUILD_DIR/start_time"
+fi
+if [ ! -f "${START_TIME_FILE}" ]; then
+    date +%s > "${START_TIME_FILE}"
+fi


-wget http://sourceforge.net/projects/boost/files/boost/1.59.0/boost_1_59_0.tar.bz2/download -O boost_1_59_0.tar.bz2
-tar xf boost_1_59_0.tar.bz2
-export BOOST_ROOT="`pwd`/boost_1_59_0"
-cd boost_1_59_0
-./bootstrap.sh
-./b2 link=static cxxflags=-fPIC stage
-cd ../../
+NEXT_TIME=60
+function exit_at_45() {
+    if [ -f "${START_TIME_FILE}" ]; then
+	NOW=$(date +%s)
+	START=$(cat "${START_TIME_FILE}")
+	TIMEOUT_SECS=2700 # 45 * 60
+	TIME=$((NOW - START))
+	if (( TIME > NEXT_TIME )); then
+	    echo "Build has taken $((TIME / 60)) minutes: $1"
+	    NEXT_TIME=$((TIME + 60))
+	fi
+	if [ "$TIMEOUT" = true ] && (( TIME > TIMEOUT_SECS )); then
+	    echo 'Exiting at 45 minutes to allow the cache to populate'
+	    exit 1
+	fi
+    fi
+}

+# two arguments
+#  - pid (probably from $!)
+#  - echo message
+function wait_and_echo() {
+    PID=$1
+    (set -o | grep xtrace | grep -q on)
+    TRACE_STATUS=$?
+    # disable xtrace or else this will get verbose, which is what
+    # I'm trying to avoid by going through all of this nonsense anyway
+    set +o xtrace
+    TIME=0
+    SLEEP=5
+    # loop until the process is no longer running
+    # check every $SLEEP seconds, echoing a message every minute
+    while (ps -p "${PID}" > /dev/null); do
+	exit_at_45 "$2"
+	sleep "${SLEEP}"
+    done
+    # restore the xtrace setting
+    if [ "${TRACE_STATUS}" -eq 0 ]; then
+	set -o xtrace
+    fi
+    wait "$PID"
+    return $?
+}

-mkdir libevent_build
-git clone https://github.com/libevent/libevent.git
-export LIBEVENT_PREFIX="`pwd`/libevent_build"
-cd libevent
-./autogen.sh
-./configure --prefix=$LIBEVENT_PREFIX --enable-static --disable-shared --with-pic LDFLAGS="-L${OPENSSL_PREFIX}/lib/" CPPFLAGS="-I${OPENSSL_PREFIX}/include"
-make
-make install
-cd ..
+# run a command ($1) in the background
+# logging its stdout and stderr to $2
+# and wait until it completed
+function background() {
+    $1 >> "$2" 2>&1 &
+    wait_and_echo $! "$3"
+}
+
+function cleanup() {
+    rv=$?
+    # cat the log file if it exists
+    if [ -f "$2" ]; then
+	cat "$2"
+    fi
+    # delete the build directory
+    rm -rf "$1"
+    echo "Build failed. Removing $1"
+    exit $rv
+}
+
+function install_brew_packages() {
+    brew update > /dev/null
+    brew install autoconf
+    brew install automake
+    brew install libtool
+    brew install pkg-config
+    brew install protobuf
+    if ! brew ls | grep gmp --quiet; then
+	brew install gmp
+    fi
+}
+
+function install_apt_packages() {
+    if [ -z "${TRAVIS+x}" ]; then
+	# if not on travis, its nice to see progress
+	QUIET=""
+    else
+	QUIET="-qq"
+    fi
+    # get the required OS packages
+    $SUDO apt-get ${QUIET} update
+    $SUDO apt-get ${QUIET} install -y --no-install-recommends \
+	  build-essential python-dev libbz2-dev libtool \
+	  autotools-dev autoconf git pkg-config wget \
+	  ca-certificates automake bsdmainutils
+}
+
+function build_dependencies() {
+    if [ "${OS_NAME}" = "osx" ]; then
+	install_brew_packages
+    else
+	install_apt_packages
+    fi
+    
+    if [ "$CLEAN" = true ]; then
+	rm -rf "${LBRYCRD_DEPENDENCIES}"
+	rm -rf "${OUTPUT_DIR}"
+    fi
+
+    mkdir -p "${LBRYCRD_DEPENDENCIES}"
+    mkdir -p "${LOG_DIR}"
+
+    build_dependency "${BDB_PREFIX}" "${LOG_DIR}/bdb_build.log" build_bdb
+    build_dependency "${OPENSSL_PREFIX}" "${LOG_DIR}/openssl_build.log" build_openssl
+    
+    set +u
+    export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/"
+    set -u
+
+    build_dependency "${BOOST_PREFIX}" "${LOG_DIR}/boost_build.log" build_boost
+    build_dependency "${LIBEVENT_PREFIX}" "${LOG_DIR}/libevent_build.log" build_libevent
+}
+
+function build_bdb() {
+    BDB_LOG="$1"
+    if [ ! -f db-4.8.30.NC.tar.gz ]; then
+	wget http://download.oracle.com/berkeley-db/db-4.8.30.NC.tar.gz
+    fi
+    tar xf db-4.8.30.NC.tar.gz
+    if [ "${OS_NAME}" = "osx" ]; then
+	curl -OL https://raw.github.com/narkoleptik/os-x-berkeleydb-patch/master/atomic.patch
+	patch db-4.8.30.NC/dbinc/atomic.h < atomic.patch
+    fi
+    cd db-4.8.30.NC/build_unix
+    echo "Building bdb.  tail -f $BDB_LOG to see the details and monitor progress"
+    ../dist/configure --prefix="${BDB_PREFIX}" --enable-cxx --disable-shared --with-pic > "${BDB_LOG}"
+    background make "${BDB_LOG}" "Waiting for bdb to finish building"
+    make install >> "${BDB_LOG}" 2>&1
+}
+
+function build_openssl() {
+    OPENSSL_LOG="$1"
+    wget https://www.openssl.org/source/openssl-1.0.1p.tar.gz
+    tar xf openssl-1.0.1p.tar.gz
+    mkdir -p "${OPENSSL_PREFIX}/ssl"
+    cd openssl-1.0.1p
+    echo "Building openssl.  tail -f $OPENSSL_LOG to see the details and monitor progress"
+    if [ "${OS_NAME}" = "osx" ]; then
+	./Configure --prefix="${OPENSSL_PREFIX}" --openssldir="${OPENSSL_PREFIX}/ssl" \
+		    -fPIC darwin64-x86_64-cc \
+		    no-shared no-dso no-engines > "${OPENSSL_LOG}"
+    else
+	./Configure --prefix="${OPENSSL_PREFIX}" --openssldir="${OPENSSL_PREFIX}/ssl" \
+		    linux-x86_64 -fPIC -static no-shared no-dso > "${OPENSSL_LOG}"
+    fi
+    background make "${OPENSSL_LOG}" "Waiting for openssl to finish building"
+    make install >> "${OPENSSL_LOG}" 2>&1
+}
+
+function build_boost() {
+    BOOST_LOG="$1"
+    wget http://sourceforge.net/projects/boost/files/boost/1.59.0/boost_1_59_0.tar.bz2/download \
+	 -O boost_1_59_0.tar.bz2
+    tar xf boost_1_59_0.tar.bz2
+    cd boost_1_59_0
+    echo "Building Boost.  tail -f ${BOOST_LOG} to see the details and monitor progress"
+    ./bootstrap.sh --prefix="${BOOST_PREFIX}" > "${BOOST_LOG}" 2>&1
+    background "./b2 link=static cxxflags=-fPIC install" \
+	       "${BOOST_LOG}" \
+	       "Waiting for boost to finish building"
+}
+
+function build_libevent() {
+    LIBEVENT_LOG="$1"
+    git clone https://github.com/libevent/libevent.git
+    cd libevent
+    echo "Building libevent.  tail -f ${LIBEVENT_LOG} to see the details and monitor progress"
+    ./autogen.sh > "${LIBEVENT_LOG}" 2>&1
+    ./configure --prefix="${LIBEVENT_PREFIX}" --enable-static --disable-shared --with-pic \
+		LDFLAGS="-L${OPENSSL_PREFIX}/lib/" \
+		CPPFLAGS="-I${OPENSSL_PREFIX}/include" >> "${LIBEVENT_LOG}" 2>&1
+    background make "${LIBEVENT_LOG}" "Waiting for libevent to finish building"
+    make install >> "${LIBEVENT_LOG}"
+}
+
+function build_dependency() {
+    PREFIX=$1
+    LOG=$2
+    BUILD=$3
+    if [ ! -d "${PREFIX}" ]; then
+	trap 'cleanup "${PREFIX}" "${LOG}"' INT TERM EXIT
+	cd "${LBRYCRD_DEPENDENCIES}"
+	mkdir -p "${PREFIX}"
+	"${BUILD}" "${LOG}"
+	trap - INT TERM EXIT
+    fi
+}
+
+function build_lbrycrd() {
+    if [ "$CLONE" == true ]; then
+	cd "${LBRYCRD_DEPENDENCIES}"
+	git clone https://github.com/lbryio/lbrycrd
+	cd lbrycrd
+    else
+	cd "${SOURCE_DIR}"
+    fi
+    ./autogen.sh
+    LDFLAGS="-L${OPENSSL_PREFIX}/lib/ -L${BDB_PREFIX}/lib/ -L${LIBEVENT_PREFIX}/lib/ -static-libstdc++"
+    CPPFLAGS="-I${OPENSSL_PREFIX}/include -I${BDB_PREFIX}/include -I${LIBEVENT_PREFIX}/include/"
+    if [ "${OS_NAME}" = "osx" ]; then
+	./configure --without-gui --enable-cxx --enable-static --disable-shared --with-pic \
+		    --with-boost="${BOOST_PREFIX}" \
+		    LDFLAGS="${LDFLAGS}" \
+		    CPPFLAGS="${CPPFLAGS}"
+    else
+	./configure --without-gui --with-boost="${BOOST_PREFIX}" \
+		    LDFLAGS="${LDFLAGS}" \
+		    CPPFLAGS="${CPPFLAGS}"
+    fi
+    make
+    # tests don't work on OSX. Should definitely figure out why
+    # that is but, for now, not letting that stop the rest
+    # of the build
+    if [ "${OS_NAME}" = "linux" ]; then
+	src/test/test_lbrycrd
+    fi
+    strip src/lbrycrdd
+    strip src/lbrycrd-cli
+    strip src/lbrycrd-tx
+    strip src/test/test_lbrycrd
+}
+
+# these variables are needed in both functions
+LBRYCRD_DEPENDENCIES="$(pwd)/lbrycrd-dependencies"
+OUTPUT_DIR="$(pwd)/build"
+LOG_DIR="$(pwd)/logs"
+BDB_PREFIX="${OUTPUT_DIR}/bdb"
+OPENSSL_PREFIX="${OUTPUT_DIR}/openssl"
+BOOST_PREFIX="${OUTPUT_DIR}/boost"
+LIBEVENT_PREFIX="${OUTPUT_DIR}/libevent"
+
+if [ "${BUILD_DEPENDENCIES}" = true ]; then
+    build_dependencies
+fi

 set +u
-export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/:${LIBEVENT_PREFIX}/lib/pkgconfig"
+export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/:${LIBEVENT_PREFIX}/lib/pkgconfig/"
 set -u

-if [ "$CLONE" == true ]; then
-  git clone https://github.com/lbryio/lbrycrd
-  cd lbrycrd
+if [ "${BUILD_LBRYCRD}" = true ]; then
+    LBRYCRD_LOG="${LOG_DIR}/lbrycrd_build.log"
+    echo "Building lbrycrd.  tail -f ${LBRYCRD_LOG} to see the details and monitor progress"
+    background build_lbrycrd "${LBRYCRD_LOG}" "Waiting for lbrycrd to finish building"
 fi
-./autogen.sh
-./configure --without-gui LDFLAGS="-L${OPENSSL_PREFIX}/lib/ -L${BDB_PREFIX}/lib/ -L${LIBEVENT_PREFIX}/lib/ -static-libstdc++" CPPFLAGS="-I${OPENSSL_PREFIX}/include -I${BDB_PREFIX}/include -I${LIBEVENT_PREFIX}/include/"
-make
-strip src/lbrycrdd
-strip src/lbrycrd-cli
-strip src/lbrycrd-tx
-strip src/test/test_lbrycrd
--- a/reproducible_build_osx.sh
+++ b/reproducible_build_osx.sh
@ -1,72 +0,0 @@
-brew update
-brew install autoconf
-brew install automake
-brew install libtool
-brew install pkg-config
-brew install protobuf
-brew install gmp
-
-mkdir dependencies
-cd dependencies
-export LBRYCRD_DEPENDENCIES="`pwd`"
-
-#download, patch, and build bdb
-wget http://download.oracle.com/berkeley-db/db-4.8.30.NC.tar.gz
-tar xf db-4.8.30.NC.tar.gz
-export BDB_PREFIX="`pwd`/bdb"
-curl -OL https://raw.github.com/narkoleptik/os-x-berkeleydb-patch/master/atomic.patch
-patch db-4.8.30.NC/dbinc/atomic.h < atomic.patch
-cd db-4.8.30.NC/build_unix
-../dist/configure --prefix=$BDB_PREFIX --enable-cxx --disable-shared --with-pic
-make
-make install
-
-#download and build openssl
-cd $LBRYCRD_DEPENDENCIES
-wget https://www.openssl.org/source/openssl-1.0.1p.tar.gz
-tar xf openssl-1.0.1p.tar.gz
-export OPENSSL_PREFIX="`pwd`/openssl_build"
-mkdir $OPENSSL_PREFIX
-mkdir $OPENSSL_PREFIX/ssl
-cd openssl-1.0.1p
-./Configure --prefix=$OPENSSL_PREFIX --openssldir=$OPENSSL_PREFIX/ssl -fPIC darwin64-x86_64-cc no-shared no-dso no-engines
-make
-make install
-export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig/"
-
-#download and build boost
-cd $LBRYCRD_DEPENDENCIES
-wget http://sourceforge.net/projects/boost/files/boost/1.59.0/boost_1_59_0.tar.bz2/download -O boost_1_59_0.tar.bz2
-tar xf boost_1_59_0.tar.bz2
-export BOOST_ROOT="`pwd`/boost_1_59_0"
-cd boost_1_59_0
-./bootstrap.sh
-./b2 link=static cxxflags=-fPIC stage
-
-#download and build libevent
-cd $LBRYCRD_DEPENDENCIES
-mkdir libevent_build
-git clone https://github.com/libevent/libevent.git
-export LIBEVENT_PREFIX="`pwd`/libevent_build"
-cd libevent
-./autogen.sh
-./configure --prefix=$LIBEVENT_PREFIX --enable-static --disable-shared --with-pic LDFLAGS="-L${OPENSSL_PREFIX}/lib/" CPPFLAGS="-I${OPENSSL_PREFIX}/include"
-make
-make install
-export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${LIBEVENT_PREFIX}/lib/pkgconfig/"
-
-#download and build lbrycrd
-cd $LBRYCRD_DEPENDENCIES
-git clone https://github.com/lbryio/lbrycrd
-cd lbrycrd
-git checkout real2
-./autogen.sh
-./configure --without-gui --enable-cxx --enable-static --disable-shared --with-pic \
-            LDFLAGS="-L${OPENSSL_PREFIX}/lib/ -L${BDB_PREFIX}/lib/ -L${LIBEVENT_PREFIX}/lib/ -static-libstdc++" \
-            CPPFLAGS="-I${OPENSSL_PREFIX}/include -I${BDB_PREFIX}/include -I${LIBEVENT_PREFIX}/include/"
-
-make
-strip src/lbrycrdd
-strip src/lbrycrd-cli
-strip src/lbrycrd-tx
-strip src/test/test-lbrycrd