439 lines
13 KiB
Bash
Executable file
439 lines
13 KiB
Bash
Executable file
#!/bin/bash
|
|
|
|
set -euo pipefail
|
|
|
|
function HELP {
|
|
echo "Build lbrycrd"
|
|
echo "-----"
|
|
echo "When run without any arguments, this script expects the current directory"
|
|
echo "to be the lbrycrd repo and it builds what is in that directory"
|
|
echo
|
|
echo "This is a long build process so it can be split into two parts"
|
|
echo "Specify the -d flag to build only the dependencies"
|
|
echo "and the -l flag to build only lbrycrd. This will fail"
|
|
echo "if the dependencies weren't built earlier"
|
|
echo
|
|
echo "Optional arguments:"
|
|
echo
|
|
echo "-f: check formatting of committed code relative to master"
|
|
echo "-r: remove intermediate files."
|
|
echo "-l: build only lbrycrd"
|
|
echo "-d: build only the dependencies"
|
|
echo "-o: timeout build after 40 minutes"
|
|
echo "-t: turn trace on"
|
|
echo "-h: show help"
|
|
exit 1
|
|
}
|
|
|
|
CLEAN=false
|
|
CHECK_CODE_FORMAT=false
|
|
BUILD_DEPENDENCIES=true
|
|
BUILD_LBRYCRD=true
|
|
TIMEOUT=false
|
|
THREE_MB=3145728
|
|
# this flag gets set to False if
|
|
# the script exits due to a timeout
|
|
OUTPUT_LOG=true
|
|
|
|
while getopts :crfldoth:w:d: FLAG; do
|
|
case $FLAG in
|
|
r)
|
|
CLEAN=true
|
|
;;
|
|
f)
|
|
CHECK_CODE_FORMAT=true
|
|
;;
|
|
l)
|
|
BUILD_DEPENDENCIES=false
|
|
;;
|
|
d)
|
|
BUILD_LBRYCRD=false
|
|
;;
|
|
o)
|
|
TIMEOUT=true
|
|
;;
|
|
t)
|
|
set -o xtrace
|
|
;;
|
|
h)
|
|
HELP
|
|
;;
|
|
\?) #unrecognized option - show help
|
|
echo "Option -$OPTARG not allowed."
|
|
HELP
|
|
;;
|
|
:)
|
|
echo "Option -$OPTARG requires an argument."
|
|
HELP
|
|
;;
|
|
esac
|
|
done
|
|
|
|
shift $((OPTIND-1))
|
|
|
|
SUDO=''
|
|
if (( EUID != 0 )); then
|
|
SUDO='sudo'
|
|
fi
|
|
|
|
if [ "$(basename "$PWD")" != "lbrycrd" ]; then
|
|
echo "Not currently in the lbrycrd directory. Cowardly refusing to go forward"
|
|
exit 1
|
|
fi
|
|
SOURCE_DIR=$PWD
|
|
|
|
if [ -z "${TRAVIS_OS_NAME+x}" ]; then
|
|
if [ "$(uname -s)" = "Darwin" ]; then
|
|
OS_NAME="osx"
|
|
else
|
|
OS_NAME="linux"
|
|
fi
|
|
else
|
|
OS_NAME="${TRAVIS_OS_NAME}"
|
|
fi
|
|
|
|
if [ -z "${TRAVIS_BUILD_DIR+x}" ]; then
|
|
START_TIME_FILE="$PWD/start_time"
|
|
else
|
|
# if we are on travis (the primary use case for setting a timeout)
|
|
# this file is created when the build starts
|
|
START_TIME_FILE="$TRAVIS_BUILD_DIR/start_time"
|
|
fi
|
|
rm -f ${START_TIME_FILE}
|
|
date +%s > ${START_TIME_FILE}
|
|
|
|
NEXT_TIME=60
|
|
function exit_at_60() {
|
|
if [ -f "${START_TIME_FILE}" ]; then
|
|
NOW=$(date +%s)
|
|
START=$(cat "${START_TIME_FILE}")
|
|
TIMEOUT_SECS=3600 # 60 * 60
|
|
TIME=$((NOW - START))
|
|
if (( TIME > NEXT_TIME )); then
|
|
echo "Build has taken $((TIME / 60)) minutes: $1"
|
|
NEXT_TIME=$((TIME + 60))
|
|
fi
|
|
if [ "$TIMEOUT" = true ] && (( TIME > TIMEOUT_SECS )); then
|
|
echo 'Exiting at 60 minutes to allow the cache to populate'
|
|
OUTPUT_LOG=false
|
|
exit 1
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# two arguments
|
|
# - pid (probably from $!)
|
|
# - echo message
|
|
function wait_and_echo() {
|
|
PID=$1
|
|
TIME=0
|
|
SLEEP=3
|
|
# loop until the process is no longer running
|
|
# check every $SLEEP seconds, echoing a message every minute
|
|
while (ps -p "${PID}" > /dev/null); do
|
|
exit_at_60 "$2"
|
|
sleep "${SLEEP}"
|
|
done
|
|
}
|
|
|
|
# run a command ($1) in the background
|
|
# logging its stdout and stderr to $2
|
|
# and wait until it completed
|
|
function background() {
|
|
eval $1 >> "$2" 2>&1 &
|
|
BACKGROUND_PID=$!
|
|
(
|
|
set +xe # do not echo each sleep call in trace mode
|
|
wait_and_echo $BACKGROUND_PID "$3"
|
|
)
|
|
wait $BACKGROUND_PID
|
|
}
|
|
|
|
function cleanup() {
|
|
rv=$?
|
|
if [ $rv -eq 0 ]; then
|
|
return $rv
|
|
fi
|
|
# cat the log file if it exists
|
|
if [ -f "$2" ] && [ "${OUTPUT_LOG}" = true ]; then
|
|
echo
|
|
echo "Output of log file $2"
|
|
echo
|
|
tail -n 200 "$2"
|
|
echo
|
|
fi
|
|
# delete the build directory
|
|
rm -rf "$1"
|
|
echo "Build failed. Removing $1"
|
|
exit $rv
|
|
}
|
|
|
|
function cat_and_exit() {
|
|
rv=$?
|
|
# cat the log file if it exists
|
|
if [ -f "$1" ] && [ "${OUTPUT_LOG}" = true ]; then
|
|
echo
|
|
echo "Output of log file $1"
|
|
echo
|
|
# This used to be the last 3MB but outputing that
|
|
# caused problems on travis.
|
|
# Hopefully the last 1000 lines is enough
|
|
# to debug whatever went wrong
|
|
tail -n 1000 "$1"
|
|
echo
|
|
fi
|
|
exit $rv
|
|
}
|
|
|
|
function brew_if_not_installed() {
|
|
if ! brew ls | grep $1 --quiet; then
|
|
brew install $1
|
|
fi
|
|
}
|
|
|
|
function install_brew_packages() {
|
|
brew update > /dev/null
|
|
brew unlink python
|
|
brew_if_not_installed autoconf
|
|
brew_if_not_installed automake
|
|
# something weird happened where glibtoolize was failing to find
|
|
# sed, and reinstalling fixes it.
|
|
brew reinstall libtool
|
|
brew_if_not_installed pkg-config
|
|
brew_if_not_installed protobuf
|
|
brew_if_not_installed gmp
|
|
|
|
if [ "${CHECK_CODE_FORMAT}" = true ]; then
|
|
brew_if_not_installed clang-format
|
|
fi
|
|
}
|
|
|
|
function install_apt_packages() {
|
|
if [ -d "${OUTPUT_DIR}" ]; then
|
|
return 0
|
|
fi
|
|
|
|
if [ -z "${TRAVIS+x}" ]; then
|
|
# if not on travis, its nice to see progress
|
|
QUIET=""
|
|
else
|
|
QUIET="-qq"
|
|
fi
|
|
# get the required OS packages
|
|
$SUDO apt-get ${QUIET} update
|
|
$SUDO apt-get ${QUIET} install -y --no-install-recommends \
|
|
build-essential python-dev libbz2-dev libtool \
|
|
autotools-dev autoconf git pkg-config wget \
|
|
ca-certificates automake bsdmainutils
|
|
|
|
if [ "${CHECK_CODE_FORMAT}" = true ]; then
|
|
$SUDO apt-get ${QUIET} install -y --no-install-recommends \
|
|
clang-format-3.9
|
|
fi
|
|
|
|
}
|
|
|
|
function build_dependencies() {
|
|
if [ "${OS_NAME}" = "osx" ]; then
|
|
PARALLEL="-j $(sysctl -n hw.ncpu)"
|
|
install_brew_packages
|
|
else
|
|
PARALLEL="-j $(grep -c processor /proc/cpuinfo)"
|
|
install_apt_packages
|
|
fi
|
|
|
|
if [ "$CLEAN" = true ]; then
|
|
rm -rf "${LBRYCRD_DEPENDENCIES}"
|
|
rm -rf "${OUTPUT_DIR}"
|
|
fi
|
|
|
|
if [ ! -d "${LBRYCRD_DEPENDENCIES}" ]; then
|
|
git clone https://github.com/lbryio/lbrycrd-dependencies.git "${LBRYCRD_DEPENDENCIES}"
|
|
fi
|
|
# TODO: if the repo exists, make sure its clean: revert to head.
|
|
mkdir -p "${LOG_DIR}"
|
|
|
|
build_dependency "${OPENSSL_PREFIX}" "${LOG_DIR}/openssl_build.log" build_openssl
|
|
build_dependency "${ICU_PREFIX}" "${LOG_DIR}/icu_build.log" build_icu
|
|
build_dependency "${BDB_PREFIX}" "${LOG_DIR}/bdb_build.log" build_bdb
|
|
|
|
set +u
|
|
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig"
|
|
set -u
|
|
|
|
build_dependency "${BOOST_PREFIX}" "${LOG_DIR}/boost_build.log" build_boost
|
|
build_dependency "${LIBEVENT_PREFIX}" "${LOG_DIR}/libevent_build.log" build_libevent
|
|
}
|
|
|
|
function build_bdb() {
|
|
BDB_LOG="$1"
|
|
if [ "${OS_NAME}" = "osx" ]; then
|
|
# TODO: make this handle already patched files
|
|
patch db-4.8.30.NC/dbinc/atomic.h < atomic.patch
|
|
fi
|
|
cd db-4.8.30.NC/build_unix
|
|
echo "Building bdb. tail -f $BDB_LOG to see the details and monitor progress"
|
|
../dist/configure --prefix="${BDB_PREFIX}" --enable-cxx --disable-shared --with-pic > "${BDB_LOG}"
|
|
background "make ${PARALLEL}" "${BDB_LOG}" "Waiting for bdb to finish building"
|
|
make install >> "${BDB_LOG}" 2>&1
|
|
}
|
|
|
|
function build_openssl() {
|
|
OPENSSL_LOG="$1"
|
|
mkdir -p "${OPENSSL_PREFIX}/ssl"
|
|
cd openssl-1.0.1p
|
|
echo "Building openssl. tail -f $OPENSSL_LOG to see the details and monitor progress"
|
|
if [ "${OS_NAME}" = "osx" ]; then
|
|
./Configure --prefix="${OPENSSL_PREFIX}" --openssldir="${OPENSSL_PREFIX}/ssl" \
|
|
-fPIC darwin64-x86_64-cc \
|
|
no-shared no-dso no-engines > "${OPENSSL_LOG}"
|
|
else
|
|
[[ $(uname -m) = 'i686' ]] && OS_ARCH="linux-generic32" || OS_ARCH="linux-x86_64"
|
|
./Configure --prefix="${OPENSSL_PREFIX}" --openssldir="${OPENSSL_PREFIX}/ssl" \
|
|
${OS_ARCH} -fPIC -static no-shared no-dso > "${OPENSSL_LOG}"
|
|
fi
|
|
background "make ${PARALLEL}" "${OPENSSL_LOG}" "Waiting for openssl to finish building"
|
|
make install >> "${OPENSSL_LOG}" 2>&1
|
|
}
|
|
|
|
function build_boost() {
|
|
BOOST_LOG="$1"
|
|
cd boost_1_59_0
|
|
|
|
echo "int main() { return 0; }" > libs/regex/build/has_icu_test.cpp
|
|
echo "int main() { return 0; }" > libs/locale/build/has_icu_test.cpp
|
|
|
|
export BOOST_ICU_LIBS="-L${ICU_PREFIX}/lib -licui18n -licuuc -licudata -dl"
|
|
export BOOST_LDFLAGS="-L${BOOST_PREFIX}/lib ${BOOST_ICU_LIBS}"
|
|
|
|
echo "BOOST_LDFLAGS: $BOOST_LDFLAGS"
|
|
|
|
echo "Building Boost. tail -f ${BOOST_LOG} to see the details and monitor progress"
|
|
./bootstrap.sh --prefix="${BOOST_PREFIX}" --with-icu="${ICU_PREFIX}" > "${BOOST_LOG}" 2>&1
|
|
b2cmd="./b2 --reconfigure ${PARALLEL} link=static cxxflags=-fPIC install boost.locale.iconv=off boost.locale.posix=off -sICU_PATH=\"${ICU_PREFIX}\" -sICU_LINK=\"${BOOST_ICU_LIBS}\""
|
|
background "${b2cmd}" "${BOOST_LOG}" "Waiting for boost to finish building"
|
|
if grep -q "icu[[:space:]]*:[[:space:]]*no$" "${BOOST_LOG}"; then
|
|
echo "Failed to find ICU dependencies. Exiting..."
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
function build_icu() {
|
|
ICU_LOG="$1"
|
|
mkdir -p "${ICU_PREFIX}/icu"
|
|
wget -c http://download.icu-project.org/files/icu4c/57.1/icu4c-57_1-src.tgz
|
|
tar -xf icu4c-57_1-src.tgz
|
|
pushd icu/source > /dev/null
|
|
echo "Building icu. tail -f $ICU_LOG to see the details and monitor progress"
|
|
./configure --prefix="${ICU_PREFIX}" --enable-draft --enable-tools \
|
|
--disable-shared --enable-static --disable-extras --disable-icuio --disable-dyload \
|
|
--disable-layout --disable-layoutex --disable-tests --disable-samples CFLAGS=-fPIC CPPFLAGS=-fPIC > "${ICU_LOG}"
|
|
if [ ! -z ${TARGET+x} ]; then
|
|
TMP_TARGET="${TARGET}"
|
|
unset TARGET
|
|
fi
|
|
set +e
|
|
background "make ${PARALLEL} VERBOSE=1" "${ICU_LOG}" "Waiting for icu to finish building"
|
|
make install >> "${ICU_LOG}" 2>&1
|
|
if [ ! -z ${TARGET+x} ]; then
|
|
TARGET="${TMP_TARGET}"
|
|
fi
|
|
set -e
|
|
popd > /dev/null
|
|
}
|
|
|
|
function build_libevent() {
|
|
LIBEVENT_LOG="$1"
|
|
if [ ! -d libevent ]; then
|
|
git clone --branch release-2.0.22-stable https://github.com/libevent/libevent.git
|
|
fi
|
|
cd libevent
|
|
echo "Building libevent. tail -f ${LIBEVENT_LOG} to see the details and monitor progress"
|
|
./autogen.sh > "${LIBEVENT_LOG}" 2>&1
|
|
./configure --prefix="${LIBEVENT_PREFIX}" --enable-static --disable-shared --with-pic \
|
|
LDFLAGS="-L${OPENSSL_PREFIX}/lib" \
|
|
CPPFLAGS="-I${OPENSSL_PREFIX}/include" >> "${LIBEVENT_LOG}" 2>&1
|
|
background "make ${PARALLEL}" "${LIBEVENT_LOG}" "Waiting for libevent to finish building"
|
|
make install >> "${LIBEVENT_LOG}"
|
|
}
|
|
|
|
function build_dependency() {
|
|
pushd .
|
|
PREFIX=$1
|
|
LOG=$2
|
|
BUILD=$3
|
|
cd "${LBRYCRD_DEPENDENCIES}"
|
|
mkdir -p "${PREFIX}"
|
|
trap 'cleanup "${PREFIX}" "${LOG}"' INT TERM EXIT
|
|
"${BUILD}" "${LOG}"
|
|
trap - INT TERM EXIT
|
|
popd
|
|
}
|
|
|
|
function build_lbrycrd() {
|
|
cd "${SOURCE_DIR}"
|
|
./autogen.sh > "${LBRYCRD_LOG}" 2>&1
|
|
LDFLAGS="-L${OPENSSL_PREFIX}/lib -L${BDB_PREFIX}/lib -L${LIBEVENT_PREFIX}/lib -L${ICU_PREFIX}/lib -static-libstdc++ -licui18n -licuuc -licudata -dl"
|
|
OPTIONS="--enable-cxx --enable-static --disable-shared --with-pic"
|
|
if [ "${OS_NAME}" = "osx" ]; then
|
|
CPPFLAGS="-I${OPENSSL_PREFIX}/include -I${BDB_PREFIX}/include -I${LIBEVENT_PREFIX}/include -I${ICU_PREFIX}/include"
|
|
else
|
|
CPPFLAGS="-I${OPENSSL_PREFIX}/include -I${BDB_PREFIX}/include -I${LIBEVENT_PREFIX}/include -I${ICU_PREFIX}/include -Wno-unused-local-typedefs -Wno-deprecated -Wno-implicit-fallthrough"
|
|
fi
|
|
|
|
CPPFLAGS="${CPPFLAGS}" LDFLAGS="${LDFLAGS}" \
|
|
./configure --without-gui ${OPTIONS} \
|
|
--with-boost="${BOOST_PREFIX}" \
|
|
--with-icu="${ICU_PREFIX}" >> "${LBRYCRD_LOG}" 2>&1
|
|
|
|
background "make ${PARALLEL}" "${LBRYCRD_LOG}" "Waiting for lbrycrd to finish building"
|
|
}
|
|
|
|
function clang_format_diff(){
|
|
# run a code formatting check on any commits not in master
|
|
# requires clang-format
|
|
git diff -U0 origin/master -- '*.h' '*.cpp' | ./contrib/devtools/clang-format-diff.py -p1
|
|
}
|
|
|
|
# these variables are needed in both functions
|
|
LBRYCRD_DEPENDENCIES="$(pwd)/lbrycrd-dependencies"
|
|
OUTPUT_DIR="$(pwd)/build"
|
|
LOG_DIR="$(pwd)/logs"
|
|
ICU_PREFIX="${OUTPUT_DIR}/icu"
|
|
BDB_PREFIX="${OUTPUT_DIR}/bdb"
|
|
OPENSSL_PREFIX="${OUTPUT_DIR}/openssl"
|
|
BOOST_PREFIX="${OUTPUT_DIR}/boost"
|
|
LIBEVENT_PREFIX="${OUTPUT_DIR}/libevent"
|
|
|
|
if [ "${BUILD_DEPENDENCIES}" = true ]; then
|
|
build_dependencies
|
|
fi
|
|
|
|
if [ "${CHECK_CODE_FORMAT}" = true ]; then
|
|
LINES_W_FORMAT_REQUIRED=$(clang_format_diff | wc -l)
|
|
if [ ${LINES_W_FORMAT_REQUIRED} -ne 0 ]; then
|
|
echo "Failed to pass clang format diff: See below for the diff"
|
|
clang_format_diff
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
set +u
|
|
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:${OPENSSL_PREFIX}/lib/pkgconfig:${LIBEVENT_PREFIX}/lib/pkgconfig:${ICU_PREFIX}/lib/pkgconfig"
|
|
set -u
|
|
|
|
if [ "${BUILD_LBRYCRD}" = true ]; then
|
|
LBRYCRD_LOG="${LOG_DIR}/lbrycrd_build.log"
|
|
echo "Building lbrycrd. tail -f ${LBRYCRD_LOG} to see the details and monitor progress"
|
|
trap 'cat_and_exit "${LBRYCRD_LOG}"' INT TERM EXIT
|
|
build_lbrycrd
|
|
trap - INT TERM EXIT
|
|
|
|
./src/test/test_lbrycrd
|
|
set +u
|
|
if [[ ! $CXXFLAGS =~ -g ]]; then
|
|
strip src/lbrycrdd
|
|
strip src/lbrycrd-cli
|
|
strip src/lbrycrd-tx
|
|
fi
|
|
fi
|