chain: request pruned blocks from backend peers

At the moment, this is only done for the BitcoindClient, as the other
backends don't support block pruning.

chain/bitcoind_client.go

@@ -140,7 +140,7 @@ func (c *BitcoindClient) GetBlockHeight(hash *chainhash.Hash) (int32, error) {
 
 // GetBlock returns a block from the hash.
 func (c *BitcoindClient) GetBlock(hash *chainhash.Hash) (*wire.MsgBlock, error) {
-	return c.chainConn.client.GetBlock(hash)
+	return c.chainConn.GetBlock(hash)
 }
 
 // GetBlockVerbose returns a verbose block from the hash.

chain/bitcoind_conn.go

@@ -9,11 +9,13 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/btcsuite/btcd/btcjson"
 	"github.com/btcsuite/btcd/chaincfg"
 	"github.com/btcsuite/btcd/chaincfg/chainhash"
 	"github.com/btcsuite/btcd/rpcclient"
 	"github.com/btcsuite/btcd/wire"
 	"github.com/lightninglabs/gozmq"
+	"github.com/lightningnetwork/lnd/ticker"
 )
 
 const (
@@ -36,6 +38,10 @@ const (
 	// seqNumLen is the length of the sequence number of a message sent from
 	// bitcoind through ZMQ.
 	seqNumLen = 4
+
+	// errBlockPrunedStr is the error message returned by bitcoind upon
+	// calling GetBlock on a pruned block.
+	errBlockPrunedStr = "Block not available (pruned data)"
 )
 
 // BitcoindConfig contains all of the parameters required to establish a
@@ -66,6 +72,17 @@ type BitcoindConfig struct {
 	// ZMQReadDeadline represents the read deadline we'll apply when reading
 	// ZMQ messages from either subscription.
 	ZMQReadDeadline time.Duration
+
+	// Dialer is a closure we'll use to dial Bitcoin peers. If the chain
+	// backend is running over Tor, this must support dialing peers over Tor
+	// as well.
+	Dialer Dialer
+
+	// PrunedModeMaxPeers is the maximum number of peers we'll attempt to
+	// retrieve pruned blocks from.
+	//
+	// NOTE: This only applies for pruned bitcoind nodes.
+	PrunedModeMaxPeers int
 }
 
 // BitcoindConn represents a persistent client connection to a bitcoind node
@@ -84,6 +101,11 @@ type BitcoindConn struct {
 	// client is the RPC client to the bitcoind node.
 	client *rpcclient.Client
 
+	// prunedBlockDispatcher handles all of the pruned block requests.
+	//
+	// NOTE: This is nil when the bitcoind node is not pruned.
+	prunedBlockDispatcher *PrunedBlockDispatcher
+
 	// zmqBlockConn is the ZMQ connection we'll use to read raw block
 	// events.
 	zmqBlockConn *gozmq.Conn
@@ -101,6 +123,10 @@ type BitcoindConn struct {
 	wg   sync.WaitGroup
 }
 
+// Dialer represents a way to dial Bitcoin peers. If the chain backend is
+// running over Tor, this must support dialing peers over Tor as well.
+type Dialer = func(string) (net.Conn, error)
+
 // NewBitcoindConn creates a client connection to the node described by the host
 // string. The ZMQ connections are established immediately to ensure liveness.
 // If the remote node does not operate on the same bitcoin network as described
@@ -120,6 +146,24 @@ func NewBitcoindConn(cfg *BitcoindConfig) (*BitcoindConn, error) {
 		return nil, err
 	}
 
+	// Verify that the node is running on the expected network.
+	net, err := getCurrentNet(client)
+	if err != nil {
+		return nil, err
+	}
+	if net != cfg.ChainParams.Net {
+		return nil, fmt.Errorf("expected network %v, got %v",
+			cfg.ChainParams.Net, net)
+	}
+
+	// Check if the node is pruned, as we'll need to perform additional
+	// operations if so.
+	chainInfo, err := client.GetBlockChainInfo()
+	if err != nil {
+		return nil, fmt.Errorf("unable to determine if bitcoind is "+
+			"pruned: %v", err)
+	}
+
 	// Establish two different ZMQ connections to bitcoind to retrieve block
 	// and transaction event notifications. We'll use two as a separation of
 	// concern to ensure one type of event isn't dropped from the connection
@@ -142,16 +186,37 @@ func NewBitcoindConn(cfg *BitcoindConfig) (*BitcoindConn, error) {
 			"events: %v", err)
 	}
 
-	conn := &BitcoindConn{
+	// Only initialize the PrunedBlockDispatcher when the connected bitcoind
-		cfg:           *cfg,
+	// node is pruned.
-		client:        client,
+	var prunedBlockDispatcher *PrunedBlockDispatcher
-		zmqBlockConn:  zmqBlockConn,
+	if chainInfo.Pruned {
-		zmqTxConn:     zmqTxConn,
+		prunedBlockDispatcher, err = NewPrunedBlockDispatcher(
-		rescanClients: make(map[uint64]*BitcoindClient),
+			&PrunedBlockDispatcherConfig{
-		quit:          make(chan struct{}),
+				ChainParams:      cfg.ChainParams,
+				NumTargetPeers:   cfg.PrunedModeMaxPeers,
+				Dial:             cfg.Dialer,
+				GetPeers:         client.GetPeerInfo,
+				PeerReadyTimeout: defaultPeerReadyTimeout,
+				RefreshPeersTicker: ticker.New(
+					defaultRefreshPeersInterval,
+				),
+				MaxRequestInvs: wire.MaxInvPerMsg,
+			},
+		)
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	return conn, nil
+	return &BitcoindConn{
+		cfg:                   *cfg,
+		client:                client,
+		prunedBlockDispatcher: prunedBlockDispatcher,
+		zmqBlockConn:          zmqBlockConn,
+		zmqTxConn:             zmqTxConn,
+		rescanClients:         make(map[uint64]*BitcoindClient),
+		quit:                  make(chan struct{}),
+	}, nil
 }
 
 // Start attempts to establish a RPC and ZMQ connection to a bitcoind node. If
@@ -164,14 +229,13 @@ func (c *BitcoindConn) Start() error {
 		return nil
 	}
 
-	// Verify that the node is running on the expected network.
+	// If we're connected to a pruned backend, we'll need to also start our
-	net, err := c.getCurrentNet()
+	// pruned block dispatcher to handle pruned block requests.
-	if err != nil {
+	if c.prunedBlockDispatcher != nil {
-		return err
+		log.Debug("Detected pruned bitcoind backend")
-	}
+		if err := c.prunedBlockDispatcher.Start(); err != nil {
-	if net != c.cfg.ChainParams.Net {
+			return err
-		return fmt.Errorf("expected network %v, got %v",
+		}
-			c.cfg.ChainParams.Net, net)
 	}
 
 	c.wg.Add(2)
@@ -197,6 +261,10 @@ func (c *BitcoindConn) Stop() {
 	c.zmqBlockConn.Close()
 	c.zmqTxConn.Close()
 
+	if c.prunedBlockDispatcher != nil {
+		c.prunedBlockDispatcher.Stop()
+	}
+
 	c.client.WaitForShutdown()
 	c.wg.Wait()
 }
@@ -405,8 +473,8 @@ func (c *BitcoindConn) txEventHandler() {
 }
 
 // getCurrentNet returns the network on which the bitcoind node is running.
-func (c *BitcoindConn) getCurrentNet() (wire.BitcoinNet, error) {
+func getCurrentNet(client *rpcclient.Client) (wire.BitcoinNet, error) {
-	hash, err := c.client.GetBlockHash(0)
+	hash, err := client.GetBlockHash(0)
 	if err != nil {
 		return 0, err
 	}
@@ -472,6 +540,55 @@ func (c *BitcoindConn) RemoveClient(id uint64) {
 	delete(c.rescanClients, id)
 }
 
+// isBlockPrunedErr determines if the error returned by the GetBlock RPC
+// corresponds to the requested block being pruned.
+func isBlockPrunedErr(err error) bool {
+	rpcErr, ok := err.(*btcjson.RPCError)
+	return ok && rpcErr.Code == btcjson.ErrRPCMisc &&
+		rpcErr.Message == errBlockPrunedStr
+}
+
+// GetBlock returns a raw block from the server given its hash. If the server
+// has already pruned the block, it will be retrieved from one of its peers.
+func (c *BitcoindConn) GetBlock(hash *chainhash.Hash) (*wire.MsgBlock, error) {
+	block, err := c.client.GetBlock(hash)
+	// Got the block from the backend successfully, return it.
+	if err == nil {
+		return block, nil
+	}
+
+	// We failed getting the block from the backend for whatever reason. If
+	// it wasn't due to the block being pruned, return the error
+	// immediately.
+	if !isBlockPrunedErr(err) || c.prunedBlockDispatcher == nil {
+		return nil, err
+	}
+
+	// Now that we know the block has been pruned for sure, request it from
+	// our backend peers.
+	blockChan, errChan := c.prunedBlockDispatcher.Query(
+		[]*chainhash.Hash{hash},
+	)
+
+	for {
+		select {
+		case block := <-blockChan:
+			return block, nil
+
+		case err := <-errChan:
+			if err != nil {
+				return nil, err
+			}
+
+			// errChan fired before blockChan with a nil error, wait
+			// for the block now.
+
+		case <-c.quit:
+			return nil, ErrBitcoindClientShuttingDown
+		}
+	}
+}
+
 // isASCII is a helper method that checks whether all bytes in `data` would be
 // printable ASCII characters if interpreted as a string.
 func isASCII(s string) bool {