integration testing scripts

some scripts for integration testing and a docker file for an action.
Still need to figure out how to properly run a more realistic version
in ci.

docker/Dockerfile.action.integration

@@ -0,0 +1,13 @@
+FROM jeffreypicard/hub-github-env:dev
+
+COPY scripts/integration_tests.sh /integration_tests.sh
+COPY scripts/cicd_integration_test_runner.sh /cicd_integration_test_runner.sh
+COPY herald /herald
+
+RUN apt install -y jq curl
+
+ENV CGO_LDFLAGS "-L/usr/local/lib -lrocksdb -lstdc++ -lm -lz -lsnappy -llz4 -lzstd"
+ENV CGO_CFLAGS "-I/usr/local/include/rocksdb"
+ENV LD_LIBRARY_PATH /usr/local/lib
+
+ENTRYPOINT ["/cicd_integration_test_runner.sh"]

scripts/cicd_integration_test_runner.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+#
+# cicd_integration_test_runner.sh
+#
+# simple script to kick off herald and call the integration testing
+# script
+#
+
+

scripts/integration_tests.sh

@@ -0,0 +1,208 @@
+#!/bin/bash
+#
+# integration_testing.sh
+#
+# GitHub Action CI/CD based integration tests for herald.go
+# These are smoke / sanity tests for the server behaving correctly on a "live"
+# system, and looks for reasonable response codes, not specific correct
+# behavior. Those are covered in unit tests.
+#
+# N.B.
+# For the curl based json tests the `id` field existing is needed.
+#
+
+# global variables
+
+RES=(0)
+FINALRES=0
+CHUNK_TEST_RES="010000000000000000000000000000000000000000000000000000000000000000000000cc59e59ff97ac092b55e423aa549"
+
+# functions
+
+
+function logical_or {
+	for res in ${RES[@]}; do
+		if [ $res -eq 1 -o $FINALRES -eq 1 ]; then
+			FINALRES=1
+			return
+		fi
+	done
+}
+
+function want_got {
+	if [ "${WANT}" != "${GOT}" ]; then
+		echo "WANT: ${WANT}"
+		echo "GOT: ${GOT}"
+		RES+=(1)
+	else
+		RES+=(0)
+	fi
+}
+
+function want_greater {
+	if [ "${WANT}" -gt "${GOT}" ]; then
+		echo "WANT: ${WANT}"
+		echo "GOT: ${GOT}"
+		RES+=(1)
+	else
+		RES+=(0)
+	fi
+}
+
+function test_command_with_want {
+	echo $CMD
+	GOT=`eval $CMD`
+
+	want_got
+}
+
+# grpc endpoint testing
+
+
+read -r -d '' CMD <<- EOM
+	grpcurl -plaintext -d '{"value": ["@Styxhexenhammer666:2"]}' 127.0.0.1:50051 pb.Hub.Resolve
+	| jq .txos[0].txHash | sed 's/"//g'
+EOM
+WANT="VOFP8MQEwps9Oa5NJJQ18WfVzUzlpCjst0Wz3xyOPd4="
+test_command_with_want
+
+# GOT=`eval $CMD`
+
+#want_got
+
+##
+## N.B. This is a degenerate case that takes a long time to run.
+## The runtime should be fixed, but in the meantime, we definitely should
+## ensure this behaves as expected.
+##
+## TODO: Test runtime doesn't exceed worst case.
+##
+
+#WANT=806389
+#read -r -d '' CMD <<- EOM
+#	grpcurl -plaintext -d '{"value": ["foo"]}' 127.0.0.1:50051 pb.Hub.Resolve | jq .txos[0].height
+#EOM
+# test_command_with_want
+
+# json rpc endpoint testing
+
+## blockchain.block
+
+### blockchain.block.get_chunk
+read -r -d '' CMD <<- EOM
+	curl http://127.0.0.1:50001/rpc -s -H "Content-Type: application/json"
+	--data '{"id": 1, "method": "blockchain.block.get_chunk", "params": [0]}'
+	| jq .result | sed 's/"//g' | head -c 100
+EOM
+WANT="${CHUNK_TEST_RES}"
+test_command_with_want
+
+### blockchain.block.get_header
+read -r -d '' CMD <<- EOM
+	curl http://127.0.0.1:50001/rpc -s -H "Content-Type: application/json"
+	--data '{"id": 1, "method": "blockchain.block.get_header", "params": []}'
+	| jq .result.timestamp
+EOM
+WANT=1446058291
+test_command_with_want
+
+### blockchain.block.headers
+read -r -d '' CMD <<- EOM
+	curl http://127.0.0.1:50001/rpc -s -H "Content-Type: application/json"
+	--data '{"id": 1, "method": "blockchain.block.headers", "params": []}'
+	| jq .result.count
+EOM
+WANT=0
+test_command_with_want
+
+## blockchain.claimtrie
+
+read -r -d '' CMD <<- EOM
+	curl http://127.0.0.1:50001/rpc -s -H "Content-Type: application/json"
+	--data '{"id": 1, "method": "blockchain.claimtrie.resolve", "params":[{"Data": ["@Styxhexenhammer666:2"]}]}'
+	| jq .result.txos[0].tx_hash | sed 's/"//g'
+EOM
+WANT="VOFP8MQEwps9Oa5NJJQ18WfVzUzlpCjst0Wz3xyOPd4="
+test_command_with_want
+
+## blockchain.address
+
+### blockchain.address.get_balance
+
+read -r -d '' CMD <<- EOM
+	curl http://127.0.0.1:50001/rpc -s -H "Content-Type: application/json"
+	--data '{"id": 1, "method": "blockchain.address.get_balance", "params":[{"Address": "bGqWuXRVm5bBqLvLPEQQpvsNxJ5ubc6bwN"}]}'
+	| jq .result.confirmed
+EOM
+WANT=44415602186
+test_command_with_want
+
+## blockchain.address.get_history
+
+read -r -d '' CMD <<- EOM
+	curl http://127.0.0.1:50001/rpc -s -H "Content-Type: application/json"
+	--data '{"id": 1, "method": "blockchain.address.get_history", "params":[{"Address": "bGqWuXRVm5bBqLvLPEQQpvsNxJ5ubc6bwN"}]}'
+	| jq '.result.confirmed | length'
+EOM
+WANT=82
+test_command_with_want
+
+## blockchain.address.listunspent
+
+read -r -d '' CMD <<- EOM
+	curl http://127.0.0.1:50001/rpc -s -H "Content-Type: application/json"
+	--data '{"id": 1, "method": "blockchain.address.listunspent", "params":[{"Address": "bGqWuXRVm5bBqLvLPEQQpvsNxJ5ubc6bwN"}]}'
+	| jq '.result | length'
+EOM
+WANT=32
+test_command_with_want
+
+# blockchain.scripthash
+
+## blockchain.scripthash.get_mempool
+
+read -r -d '' CMD <<- EOM
+	curl http://127.0.0.1:50001/rpc -s -H "Content-Type: application/json"
+	--data '{"id": 1, "method": "blockchain.scripthash.get_mempool", "params":[{"scripthash": "bGqWuXRVm5bBqLvLPEQQpvsNxJ5ubc6bwN"}]}'
+	| jq .error | sed 's/"//g'
+EOM
+WANT="encoding/hex: invalid byte: U+0047 'G'"
+test_command_with_want
+
+## blockchain.scripthash.get_history
+
+read -r -d '' CMD <<- EOM
+	curl http://127.0.0.1:50001/rpc -s -H "Content-Type: application/json"
+	--data '{"id": 1, "method": "blockchain.scripthash.get_history", "params":[{"scripthash": "bGqWuXRVm5bBqLvLPEQQpvsNxJ5ubc6bwN"}]}'
+	| jq .error | sed 's/"//g'
+EOM
+WANT="encoding/hex: invalid byte: U+0047 'G'"
+test_command_with_want
+
+## blockchain.scripthash.listunspent
+
+read -r -d '' CMD <<- EOM
+	curl http://127.0.0.1:50001/rpc -s -H "Content-Type: application/json"
+	--data '{"id": 1, "method": "blockchain.scripthash.listunspent", "params":[{"scripthash": "bGqWuXRVm5bBqLvLPEQQpvsNxJ5ubc6bwN"}]}'
+	| jq .error | sed 's/"//g'
+EOM
+WANT="encoding/hex: invalid byte: U+0047 'G'"
+test_command_with_want
+
+# metrics endpoint testing
+
+WANT=0
+GOT=$(curl http://127.0.0.1:2112/metrics -s | grep requests | grep resolve | awk '{print $NF}')
+want_greater
+
+# caclulate return value
+
+logical_or $RES
+
+if [ $FINALRES -eq 1 ]; then
+	echo "Failed!"
+	exit 1
+else
+	echo "Passed!"
+	exit 0
+fi

server/jsonrpc_blockchain.go

@@ -18,6 +18,8 @@ import (
 	"github.com/lbryio/lbcd/wire"
 	"github.com/lbryio/lbcutil"
 	"golang.org/x/exp/constraints"
+
+	log "github.com/sirupsen/logrus"
 )
 
 // BlockchainBlockService methods handle "blockchain.block.*" RPCs
@@ -120,6 +122,7 @@ func (s *BlockchainBlockService) Get_chunk(req *BlockGetChunkReq, resp **BlockGe
 	index := uint32(*req)
 	db_headers, err := s.DB.GetHeaders(index*CHUNK_SIZE, CHUNK_SIZE)
 	if err != nil {
+		log.Warn(err)
 		return err
 	}
 	raw := make([]byte, 0, HEADER_SIZE*len(db_headers))
@@ -141,6 +144,7 @@ func (s *BlockchainBlockService) Get_header(req *BlockGetHeaderReq, resp **Block
 	height := uint32(*req)
 	headers, err := s.DB.GetHeaders(height, 1)
 	if err != nil {
+		log.Warn(err)
 		return err
 	}
 	if len(headers) < 1 {
@@ -171,6 +175,7 @@ func (s *BlockchainBlockService) Headers(req *BlockHeadersReq, resp **BlockHeade
 	count := min(req.Count, MAX_CHUNK_SIZE)
 	db_headers, err := s.DB.GetHeaders(req.StartHeight, count)
 	if err != nil {
+		log.Warn(err)
 		return err
 	}
 	count = uint32(len(db_headers))
@@ -283,18 +288,22 @@ type AddressGetBalanceResp struct {
 func (s *BlockchainAddressService) Get_balance(req *AddressGetBalanceReq, resp **AddressGetBalanceResp) error {
 	address, err := lbcutil.DecodeAddress(req.Address, s.Chain)
 	if err != nil {
+		log.Warn(err)
 		return err
 	}
 	script, err := txscript.PayToAddrScript(address)
 	if err != nil {
+		log.Warn(err)
 		return err
 	}
 	hashX := hashXScript(script, s.Chain)
 	confirmed, unconfirmed, err := s.DB.GetBalance(hashX)
 	if err != nil {
+		log.Warn(err)
 		return err
 	}
 	*resp = &AddressGetBalanceResp{confirmed, unconfirmed}
+
 	return err
 }
 
@@ -310,11 +319,13 @@ type ScripthashGetBalanceResp struct {
 func (s *BlockchainScripthashService) Get_balance(req *scripthashGetBalanceReq, resp **ScripthashGetBalanceResp) error {
 	scripthash, err := decodeScriptHash(req.ScriptHash)
 	if err != nil {
+		log.Warn(err)
 		return err
 	}
 	hashX := hashX(scripthash)
 	confirmed, unconfirmed, err := s.DB.GetBalance(hashX)
 	if err != nil {
+		log.Warn(err)
 		return err
 	}
 	*resp = &ScripthashGetBalanceResp{confirmed, unconfirmed}
@@ -341,15 +352,18 @@ type AddressGetHistoryResp struct {
 func (s *BlockchainAddressService) Get_history(req *AddressGetHistoryReq, resp **AddressGetHistoryResp) error {
 	address, err := lbcutil.DecodeAddress(req.Address, s.Chain)
 	if err != nil {
+		log.Warn(err)
 		return err
 	}
 	script, err := txscript.PayToAddrScript(address)
 	if err != nil {
+		log.Warn(err)
 		return err
 	}
 	hashX := hashXScript(script, s.Chain)
 	dbTXs, err := s.DB.GetHistory(hashX)
 	if err != nil {
+		log.Warn(err)
 		return err
 	}
 	confirmed := make([]TxInfo, 0, len(dbTXs))
@@ -380,11 +394,13 @@ type ScripthashGetHistoryResp struct {
 func (s *BlockchainScripthashService) Get_history(req *ScripthashGetHistoryReq, resp **ScripthashGetHistoryResp) error {
 	scripthash, err := decodeScriptHash(req.ScriptHash)
 	if err != nil {
+		log.Warn(err)
 		return err
 	}
 	hashX := hashX(scripthash)
 	dbTXs, err := s.DB.GetHistory(hashX)
 	if err != nil {
+		log.Warn(err)
 		return err
 	}
 	confirmed := make([]TxInfo, 0, len(dbTXs))
@@ -412,10 +428,12 @@ type AddressGetMempoolResp []TxInfoFee
 func (s *BlockchainAddressService) Get_mempool(req *AddressGetMempoolReq, resp **AddressGetMempoolResp) error {
 	address, err := lbcutil.DecodeAddress(req.Address, s.Chain)
 	if err != nil {
+		log.Warn(err)
 		return err
 	}
 	script, err := txscript.PayToAddrScript(address)
 	if err != nil {
+		log.Warn(err)
 		return err
 	}
 	hashX := hashXScript(script, s.Chain)
@@ -436,6 +454,7 @@ type ScripthashGetMempoolResp []TxInfoFee
 func (s *BlockchainScripthashService) Get_mempool(req *ScripthashGetMempoolReq, resp **ScripthashGetMempoolResp) error {
 	scripthash, err := decodeScriptHash(req.ScriptHash)
 	if err != nil {
+		log.Warn(err)
 		return err
 	}
 	hashX := hashX(scripthash)
@@ -462,10 +481,12 @@ type AddressListUnspentResp []TXOInfo
 func (s *BlockchainAddressService) Listunspent(req *AddressListUnspentReq, resp **AddressListUnspentResp) error {
 	address, err := lbcutil.DecodeAddress(req.Address, s.Chain)
 	if err != nil {
+		log.Warn(err)
 		return err
 	}
 	script, err := txscript.PayToAddrScript(address)
 	if err != nil {
+		log.Warn(err)
 		return err
 	}
 	hashX := hashXScript(script, s.Chain)
@@ -494,6 +515,7 @@ type ScripthashListUnspentResp []TXOInfo
 func (s *BlockchainScripthashService) Listunspent(req *ScripthashListUnspentReq, resp **ScripthashListUnspentResp) error {
 	scripthash, err := decodeScriptHash(req.ScriptHash)
 	if err != nil {
+		log.Warn(err)
 		return err
 	}
 	hashX := hashX(scripthash)