From 20c02df1e31b734bc5b0c2a91e46d844be6b74f7 Mon Sep 17 00:00:00 2001 From: Wilmer Paulino Date: Mon, 22 Mar 2021 17:46:29 -0700 Subject: [PATCH] chain: request pruned blocks from backend peers At the moment, this is only done for the BitcoindClient, as the other backends don't support block pruning. --- chain/bitcoind_client.go | 2 +- chain/bitcoind_conn.go | 153 ++++++++++++++++++++++++++++++++++----- 2 files changed, 136 insertions(+), 19 deletions(-) diff --git a/chain/bitcoind_client.go b/chain/bitcoind_client.go index 5deb1f6..caaa3a7 100644 --- a/chain/bitcoind_client.go +++ b/chain/bitcoind_client.go @@ -140,7 +140,7 @@ func (c *BitcoindClient) GetBlockHeight(hash *chainhash.Hash) (int32, error) { // GetBlock returns a block from the hash. func (c *BitcoindClient) GetBlock(hash *chainhash.Hash) (*wire.MsgBlock, error) { - return c.chainConn.client.GetBlock(hash) + return c.chainConn.GetBlock(hash) } // GetBlockVerbose returns a verbose block from the hash. diff --git a/chain/bitcoind_conn.go b/chain/bitcoind_conn.go index f5e598d..0487167 100644 --- a/chain/bitcoind_conn.go +++ b/chain/bitcoind_conn.go @@ -9,11 +9,13 @@ import ( "sync/atomic" "time" + "github.com/btcsuite/btcd/btcjson" "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/chaincfg/chainhash" "github.com/btcsuite/btcd/rpcclient" "github.com/btcsuite/btcd/wire" "github.com/lightninglabs/gozmq" + "github.com/lightningnetwork/lnd/ticker" ) const ( @@ -36,6 +38,10 @@ const ( // seqNumLen is the length of the sequence number of a message sent from // bitcoind through ZMQ. seqNumLen = 4 + + // errBlockPrunedStr is the error message returned by bitcoind upon + // calling GetBlock on a pruned block. + errBlockPrunedStr = "Block not available (pruned data)" ) // BitcoindConfig contains all of the parameters required to establish a @@ -66,6 +72,17 @@ type BitcoindConfig struct { // ZMQReadDeadline represents the read deadline we'll apply when reading // ZMQ messages from either subscription. ZMQReadDeadline time.Duration + + // Dialer is a closure we'll use to dial Bitcoin peers. If the chain + // backend is running over Tor, this must support dialing peers over Tor + // as well. + Dialer Dialer + + // PrunedModeMaxPeers is the maximum number of peers we'll attempt to + // retrieve pruned blocks from. + // + // NOTE: This only applies for pruned bitcoind nodes. + PrunedModeMaxPeers int } // BitcoindConn represents a persistent client connection to a bitcoind node @@ -84,6 +101,11 @@ type BitcoindConn struct { // client is the RPC client to the bitcoind node. client *rpcclient.Client + // prunedBlockDispatcher handles all of the pruned block requests. + // + // NOTE: This is nil when the bitcoind node is not pruned. + prunedBlockDispatcher *PrunedBlockDispatcher + // zmqBlockConn is the ZMQ connection we'll use to read raw block // events. zmqBlockConn *gozmq.Conn @@ -101,6 +123,10 @@ type BitcoindConn struct { wg sync.WaitGroup } +// Dialer represents a way to dial Bitcoin peers. If the chain backend is +// running over Tor, this must support dialing peers over Tor as well. +type Dialer = func(string) (net.Conn, error) + // NewBitcoindConn creates a client connection to the node described by the host // string. The ZMQ connections are established immediately to ensure liveness. // If the remote node does not operate on the same bitcoin network as described @@ -120,6 +146,24 @@ func NewBitcoindConn(cfg *BitcoindConfig) (*BitcoindConn, error) { return nil, err } + // Verify that the node is running on the expected network. + net, err := getCurrentNet(client) + if err != nil { + return nil, err + } + if net != cfg.ChainParams.Net { + return nil, fmt.Errorf("expected network %v, got %v", + cfg.ChainParams.Net, net) + } + + // Check if the node is pruned, as we'll need to perform additional + // operations if so. + chainInfo, err := client.GetBlockChainInfo() + if err != nil { + return nil, fmt.Errorf("unable to determine if bitcoind is "+ + "pruned: %v", err) + } + // Establish two different ZMQ connections to bitcoind to retrieve block // and transaction event notifications. We'll use two as a separation of // concern to ensure one type of event isn't dropped from the connection @@ -142,16 +186,37 @@ func NewBitcoindConn(cfg *BitcoindConfig) (*BitcoindConn, error) { "events: %v", err) } - conn := &BitcoindConn{ - cfg: *cfg, - client: client, - zmqBlockConn: zmqBlockConn, - zmqTxConn: zmqTxConn, - rescanClients: make(map[uint64]*BitcoindClient), - quit: make(chan struct{}), + // Only initialize the PrunedBlockDispatcher when the connected bitcoind + // node is pruned. + var prunedBlockDispatcher *PrunedBlockDispatcher + if chainInfo.Pruned { + prunedBlockDispatcher, err = NewPrunedBlockDispatcher( + &PrunedBlockDispatcherConfig{ + ChainParams: cfg.ChainParams, + NumTargetPeers: cfg.PrunedModeMaxPeers, + Dial: cfg.Dialer, + GetPeers: client.GetPeerInfo, + PeerReadyTimeout: defaultPeerReadyTimeout, + RefreshPeersTicker: ticker.New( + defaultRefreshPeersInterval, + ), + MaxRequestInvs: wire.MaxInvPerMsg, + }, + ) + if err != nil { + return nil, err + } } - return conn, nil + return &BitcoindConn{ + cfg: *cfg, + client: client, + prunedBlockDispatcher: prunedBlockDispatcher, + zmqBlockConn: zmqBlockConn, + zmqTxConn: zmqTxConn, + rescanClients: make(map[uint64]*BitcoindClient), + quit: make(chan struct{}), + }, nil } // Start attempts to establish a RPC and ZMQ connection to a bitcoind node. If @@ -164,14 +229,13 @@ func (c *BitcoindConn) Start() error { return nil } - // Verify that the node is running on the expected network. - net, err := c.getCurrentNet() - if err != nil { - return err - } - if net != c.cfg.ChainParams.Net { - return fmt.Errorf("expected network %v, got %v", - c.cfg.ChainParams.Net, net) + // If we're connected to a pruned backend, we'll need to also start our + // pruned block dispatcher to handle pruned block requests. + if c.prunedBlockDispatcher != nil { + log.Debug("Detected pruned bitcoind backend") + if err := c.prunedBlockDispatcher.Start(); err != nil { + return err + } } c.wg.Add(2) @@ -197,6 +261,10 @@ func (c *BitcoindConn) Stop() { c.zmqBlockConn.Close() c.zmqTxConn.Close() + if c.prunedBlockDispatcher != nil { + c.prunedBlockDispatcher.Stop() + } + c.client.WaitForShutdown() c.wg.Wait() } @@ -405,8 +473,8 @@ func (c *BitcoindConn) txEventHandler() { } // getCurrentNet returns the network on which the bitcoind node is running. -func (c *BitcoindConn) getCurrentNet() (wire.BitcoinNet, error) { - hash, err := c.client.GetBlockHash(0) +func getCurrentNet(client *rpcclient.Client) (wire.BitcoinNet, error) { + hash, err := client.GetBlockHash(0) if err != nil { return 0, err } @@ -472,6 +540,55 @@ func (c *BitcoindConn) RemoveClient(id uint64) { delete(c.rescanClients, id) } +// isBlockPrunedErr determines if the error returned by the GetBlock RPC +// corresponds to the requested block being pruned. +func isBlockPrunedErr(err error) bool { + rpcErr, ok := err.(*btcjson.RPCError) + return ok && rpcErr.Code == btcjson.ErrRPCMisc && + rpcErr.Message == errBlockPrunedStr +} + +// GetBlock returns a raw block from the server given its hash. If the server +// has already pruned the block, it will be retrieved from one of its peers. +func (c *BitcoindConn) GetBlock(hash *chainhash.Hash) (*wire.MsgBlock, error) { + block, err := c.client.GetBlock(hash) + // Got the block from the backend successfully, return it. + if err == nil { + return block, nil + } + + // We failed getting the block from the backend for whatever reason. If + // it wasn't due to the block being pruned, return the error + // immediately. + if !isBlockPrunedErr(err) || c.prunedBlockDispatcher == nil { + return nil, err + } + + // Now that we know the block has been pruned for sure, request it from + // our backend peers. + blockChan, errChan := c.prunedBlockDispatcher.Query( + []*chainhash.Hash{hash}, + ) + + for { + select { + case block := <-blockChan: + return block, nil + + case err := <-errChan: + if err != nil { + return nil, err + } + + // errChan fired before blockChan with a nil error, wait + // for the block now. + + case <-c.quit: + return nil, ErrBitcoindClientShuttingDown + } + } +} + // isASCII is a helper method that checks whether all bytes in `data` would be // printable ASCII characters if interpreted as a string. func isASCII(s string) bool {