2014-01-01 17:16:15 +01:00
|
|
|
// Copyright (c) 2013-2014 Conformal Systems LLC.
|
2013-08-06 23:55:22 +02:00
|
|
|
// Use of this source code is governed by an ISC
|
|
|
|
// license that can be found in the LICENSE file.
|
|
|
|
|
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
2013-09-03 23:55:14 +02:00
|
|
|
"container/list"
|
2013-08-06 23:55:22 +02:00
|
|
|
"github.com/conformal/btcchain"
|
|
|
|
"github.com/conformal/btcdb"
|
|
|
|
"github.com/conformal/btcutil"
|
|
|
|
"github.com/conformal/btcwire"
|
2013-10-04 05:31:00 +02:00
|
|
|
"net"
|
2013-08-06 23:55:22 +02:00
|
|
|
"os"
|
|
|
|
"path/filepath"
|
|
|
|
"sync"
|
2013-10-03 01:33:42 +02:00
|
|
|
"sync/atomic"
|
2013-08-06 23:55:22 +02:00
|
|
|
"time"
|
|
|
|
)
|
|
|
|
|
|
|
|
const (
|
|
|
|
chanBufferSize = 50
|
2013-09-15 19:08:42 +02:00
|
|
|
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
// minInFlightBlocks is the minimum number of blocks that should be
|
|
|
|
// in the request queue for headers-first mode before requesting
|
|
|
|
// more.
|
|
|
|
minInFlightBlocks = 10
|
|
|
|
|
2013-09-15 19:08:42 +02:00
|
|
|
// blockDbNamePrefix is the prefix for the block database name. The
|
|
|
|
// database type is appended to this value to form the full block
|
|
|
|
// database name.
|
|
|
|
blockDbNamePrefix = "blocks"
|
2013-08-06 23:55:22 +02:00
|
|
|
)
|
|
|
|
|
2013-10-01 00:53:21 +02:00
|
|
|
// newPeerMsg signifies a newly connected peer to the block handler.
|
|
|
|
type newPeerMsg struct {
|
|
|
|
peer *peer
|
|
|
|
}
|
|
|
|
|
2013-08-06 23:55:22 +02:00
|
|
|
// blockMsg packages a bitcoin block message and the peer it came from together
|
|
|
|
// so the block handler has access to that information.
|
|
|
|
type blockMsg struct {
|
|
|
|
block *btcutil.Block
|
2013-08-16 20:35:38 +02:00
|
|
|
peer *peer
|
2013-08-06 23:55:22 +02:00
|
|
|
}
|
|
|
|
|
2013-09-27 02:41:02 +02:00
|
|
|
// invMsg packages a bitcoin inv message and the peer it came from together
|
|
|
|
// so the block handler has access to that information.
|
|
|
|
type invMsg struct {
|
|
|
|
inv *btcwire.MsgInv
|
|
|
|
peer *peer
|
|
|
|
}
|
|
|
|
|
2013-11-16 00:16:51 +01:00
|
|
|
// blockMsg packages a bitcoin block message and the peer it came from together
|
|
|
|
// so the block handler has access to that information.
|
|
|
|
type headersMsg struct {
|
|
|
|
headers *btcwire.MsgHeaders
|
|
|
|
peer *peer
|
|
|
|
}
|
|
|
|
|
2013-10-01 00:53:21 +02:00
|
|
|
// donePeerMsg signifies a newly disconnected peer to the block handler.
|
|
|
|
type donePeerMsg struct {
|
|
|
|
peer *peer
|
|
|
|
}
|
|
|
|
|
2013-08-06 23:55:22 +02:00
|
|
|
// txMsg packages a bitcoin tx message and the peer it came from together
|
|
|
|
// so the block handler has access to that information.
|
|
|
|
type txMsg struct {
|
2013-10-28 21:44:38 +01:00
|
|
|
tx *btcutil.Tx
|
2013-08-06 23:55:22 +02:00
|
|
|
peer *peer
|
|
|
|
}
|
|
|
|
|
2014-02-04 16:39:00 +01:00
|
|
|
// getSyncPeerMsg is a message type to be sent across the query channel for
|
|
|
|
// retrieving the current sync peer.
|
|
|
|
type getSyncPeerMsg struct {
|
|
|
|
reply chan *peer
|
|
|
|
}
|
|
|
|
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
// headerNode is used as a node in a list of headers that are linked together
|
|
|
|
// between checkpoints.
|
|
|
|
type headerNode struct {
|
|
|
|
height int64
|
|
|
|
sha *btcwire.ShaHash
|
|
|
|
}
|
|
|
|
|
2013-08-06 23:55:22 +02:00
|
|
|
// blockManager provides a concurrency safe block manager for handling all
|
2013-08-29 21:44:43 +02:00
|
|
|
// incoming blocks.
|
2013-08-06 23:55:22 +02:00
|
|
|
type blockManager struct {
|
|
|
|
server *server
|
2013-10-03 01:33:42 +02:00
|
|
|
started int32
|
|
|
|
shutdown int32
|
2013-08-06 23:55:22 +02:00
|
|
|
blockChain *btcchain.BlockChain
|
2013-08-29 21:44:43 +02:00
|
|
|
blockPeer map[btcwire.ShaHash]*peer
|
2013-10-08 17:47:00 +02:00
|
|
|
requestedTxns map[btcwire.ShaHash]bool
|
2013-10-01 00:53:21 +02:00
|
|
|
requestedBlocks map[btcwire.ShaHash]bool
|
2013-08-06 23:55:22 +02:00
|
|
|
receivedLogBlocks int64
|
|
|
|
receivedLogTx int64
|
|
|
|
lastBlockLogTime time.Time
|
|
|
|
processingReqs bool
|
2013-09-03 23:55:14 +02:00
|
|
|
syncPeer *peer
|
2013-10-01 00:53:21 +02:00
|
|
|
msgChan chan interface{}
|
2014-02-04 16:39:00 +01:00
|
|
|
query chan interface{}
|
2013-08-06 23:55:22 +02:00
|
|
|
wg sync.WaitGroup
|
|
|
|
quit chan bool
|
2013-11-16 00:16:51 +01:00
|
|
|
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
// The following fields are used for headers-first mode.
|
|
|
|
headersFirstMode bool
|
|
|
|
headerList *list.List
|
|
|
|
startHeader *list.Element
|
|
|
|
nextCheckpoint *btcchain.Checkpoint
|
2013-11-16 00:16:51 +01:00
|
|
|
}
|
|
|
|
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
// resetHeaderState sets the headers-first mode state to values appropriate for
|
|
|
|
// syncing from a new peer.
|
|
|
|
func (b *blockManager) resetHeaderState(newestHash *btcwire.ShaHash, newestHeight int64) {
|
|
|
|
b.headersFirstMode = false
|
|
|
|
b.headerList.Init()
|
|
|
|
b.startHeader = nil
|
|
|
|
|
|
|
|
// When there is a next checkpoint, add an entry for the latest known
|
|
|
|
// block into the header pool. This allows the next downloaded header
|
|
|
|
// to prove it links to the chain properly.
|
|
|
|
if b.nextCheckpoint != nil {
|
|
|
|
node := headerNode{height: newestHeight, sha: newestHash}
|
|
|
|
b.headerList.PushBack(&node)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// findNextHeaderCheckpoint returns the next checkpoint after the passed height.
|
|
|
|
// It returns nil when there is not one either because the height is already
|
|
|
|
// later than the final checkpoint or some other reason such as disabled
|
|
|
|
// checkpoints.
|
|
|
|
func (b *blockManager) findNextHeaderCheckpoint(height int64) *btcchain.Checkpoint {
|
|
|
|
checkpoints := b.blockChain.Checkpoints()
|
|
|
|
if checkpoints == nil {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// There is no next checkpoint if the height is already after the final
|
|
|
|
// checkpoint.
|
|
|
|
finalCheckpoint := &checkpoints[len(checkpoints)-1]
|
|
|
|
if height >= finalCheckpoint.Height {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Find the next checkpoint.
|
|
|
|
nextCheckpoint := finalCheckpoint
|
|
|
|
for i := len(checkpoints) - 2; i >= 0; i-- {
|
|
|
|
if height >= checkpoints[i].Height {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
nextCheckpoint = &checkpoints[i]
|
|
|
|
}
|
|
|
|
return nextCheckpoint
|
2013-08-06 23:55:22 +02:00
|
|
|
}
|
|
|
|
|
2013-09-03 23:55:14 +02:00
|
|
|
// startSync will choose the best peer among the available candidate peers to
|
|
|
|
// download/sync the blockchain from. When syncing is already running, it
|
|
|
|
// simply returns. It also examines the candidates for any which are no longer
|
|
|
|
// candidates and removes them as needed.
|
|
|
|
func (b *blockManager) startSync(peers *list.List) {
|
|
|
|
// Return now if we're already syncing.
|
|
|
|
if b.syncPeer != nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// Find the height of the current known best block.
|
|
|
|
_, height, err := b.server.db.NewestSha()
|
|
|
|
if err != nil {
|
2013-11-21 19:03:56 +01:00
|
|
|
bmgrLog.Errorf("%v", err)
|
2013-09-03 23:55:14 +02:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
var bestPeer *peer
|
2013-10-07 16:04:11 +02:00
|
|
|
var enext *list.Element
|
|
|
|
for e := peers.Front(); e != nil; e = enext {
|
|
|
|
enext = e.Next()
|
2013-09-03 23:55:14 +02:00
|
|
|
p := e.Value.(*peer)
|
|
|
|
|
|
|
|
// Remove sync candidate peers that are no longer candidates due
|
2013-10-04 05:31:00 +02:00
|
|
|
// to passing their latest known block. NOTE: The < is
|
|
|
|
// intentional as opposed to <=. While techcnically the peer
|
|
|
|
// doesn't have a later block when it's equal, it will likely
|
|
|
|
// have one soon so it is a reasonable choice. It also allows
|
|
|
|
// the case where both are at 0 such as during regression test.
|
|
|
|
if p.lastBlock < int32(height) {
|
2013-09-03 23:55:14 +02:00
|
|
|
peers.Remove(e)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
// TODO(davec): Use a better algorithm to choose the best peer.
|
|
|
|
// For now, just pick the first available candidate.
|
|
|
|
bestPeer = p
|
|
|
|
}
|
|
|
|
|
|
|
|
// Start syncing from the best peer if one was selected.
|
|
|
|
if bestPeer != nil {
|
|
|
|
locator, err := b.blockChain.LatestBlockLocator()
|
|
|
|
if err != nil {
|
2013-11-21 19:03:56 +01:00
|
|
|
bmgrLog.Errorf("Failed to get block locator for the "+
|
2013-09-03 23:55:14 +02:00
|
|
|
"latest block: %v", err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2013-11-21 19:03:56 +01:00
|
|
|
bmgrLog.Infof("Syncing to block height %d from peer %v",
|
2013-10-03 00:06:29 +02:00
|
|
|
bestPeer.lastBlock, bestPeer.addr)
|
2013-11-16 00:16:51 +01:00
|
|
|
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
// When the current height is less than a known checkpoint we
|
|
|
|
// can use block headers to learn about which blocks comprise
|
|
|
|
// the chain up to the checkpoint and perform less validation
|
|
|
|
// for them. This is possible since each header contains the
|
|
|
|
// hash of the previous header and a merkle root. Therefore if
|
|
|
|
// we validate all of the received headers link together
|
|
|
|
// properly and the checkpoint hashes match, we can be sure the
|
|
|
|
// hashes for the blocks in between are accurate. Further, once
|
|
|
|
// the full blocks are downloaded, the merkle root is computed
|
|
|
|
// and compared against the value in the header which proves the
|
|
|
|
// full block hasn't been tampered with.
|
|
|
|
//
|
|
|
|
// Once we have passed the final checkpoint, or checkpoints are
|
|
|
|
// disabled, use standard inv messages learn about the blocks
|
|
|
|
// and fully validate them. Finally, regression test mode does
|
|
|
|
// not support the headers-first approach so do normal block
|
|
|
|
// downloads when in regression test mode.
|
|
|
|
if b.nextCheckpoint != nil && height < b.nextCheckpoint.Height &&
|
|
|
|
!cfg.RegressionTest && !cfg.DisableCheckpoints {
|
|
|
|
|
|
|
|
bestPeer.PushGetHeadersMsg(locator, b.nextCheckpoint.Hash)
|
|
|
|
b.headersFirstMode = true
|
|
|
|
bmgrLog.Infof("Downloading headers for blocks %d to "+
|
|
|
|
"%d from peer %s", height+1,
|
|
|
|
b.nextCheckpoint.Height, bestPeer.addr)
|
2013-11-16 00:16:51 +01:00
|
|
|
} else {
|
|
|
|
bestPeer.PushGetBlocksMsg(locator, &zeroHash)
|
|
|
|
}
|
2013-09-03 23:55:14 +02:00
|
|
|
b.syncPeer = bestPeer
|
2013-10-08 00:09:33 +02:00
|
|
|
} else {
|
2013-11-21 19:03:56 +01:00
|
|
|
bmgrLog.Warnf("No sync peer candidates available")
|
2013-09-03 23:55:14 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-10-04 05:31:00 +02:00
|
|
|
// isSyncCandidate returns whether or not the peer is a candidate to consider
|
|
|
|
// syncing from.
|
|
|
|
func (b *blockManager) isSyncCandidate(p *peer) bool {
|
|
|
|
// Typically a peer is not a candidate for sync if it's not a full node,
|
|
|
|
// however regression test is special in that the regression tool is
|
|
|
|
// not a full node and still needs to be considered a sync candidate.
|
|
|
|
if cfg.RegressionTest {
|
|
|
|
// The peer is not a candidate if it's not coming from localhost
|
|
|
|
// or the hostname can't be determined for some reason.
|
|
|
|
host, _, err := net.SplitHostPort(p.addr)
|
|
|
|
if err != nil {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
if host != "127.0.0.1" && host != "localhost" {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
// The peer is not a candidate for sync if it's not a full node.
|
|
|
|
if p.services&btcwire.SFNodeNetwork != btcwire.SFNodeNetwork {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Candidate if all checks passed.
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2013-10-01 00:53:21 +02:00
|
|
|
// handleNewPeerMsg deals with new peers that have signalled they may
|
2013-09-03 23:55:14 +02:00
|
|
|
// be considered as a sync peer (they have already successfully negotiated). It
|
2013-09-29 22:26:03 +02:00
|
|
|
// also starts syncing if needed. It is invoked from the syncHandler goroutine.
|
2013-10-01 00:53:21 +02:00
|
|
|
func (b *blockManager) handleNewPeerMsg(peers *list.List, p *peer) {
|
2013-09-03 23:55:14 +02:00
|
|
|
// Ignore if in the process of shutting down.
|
2013-10-03 01:33:42 +02:00
|
|
|
if atomic.LoadInt32(&b.shutdown) != 0 {
|
2013-09-03 23:55:14 +02:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2014-01-07 17:34:34 +01:00
|
|
|
bmgrLog.Infof("New valid peer %s (%s)", p, p.userAgent)
|
2013-10-02 23:49:31 +02:00
|
|
|
|
2013-10-04 05:31:00 +02:00
|
|
|
// Ignore the peer if it's not a sync candidate.
|
|
|
|
if !b.isSyncCandidate(p) {
|
2013-09-03 23:55:14 +02:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// Add the peer as a candidate to sync from.
|
|
|
|
peers.PushBack(p)
|
|
|
|
|
|
|
|
// Start syncing by choosing the best candidate if needed.
|
|
|
|
b.startSync(peers)
|
|
|
|
}
|
|
|
|
|
|
|
|
// handleDonePeerMsg deals with peers that have signalled they are done. It
|
|
|
|
// removes the peer as a candidate for syncing and in the case where it was
|
|
|
|
// the current sync peer, attempts to select a new best peer to sync from. It
|
|
|
|
// is invoked from the syncHandler goroutine.
|
|
|
|
func (b *blockManager) handleDonePeerMsg(peers *list.List, p *peer) {
|
|
|
|
// Remove the peer from the list of candidate peers.
|
2013-10-07 17:10:10 +02:00
|
|
|
for e := peers.Front(); e != nil; e = e.Next() {
|
2013-09-03 23:55:14 +02:00
|
|
|
if e.Value == p {
|
|
|
|
peers.Remove(e)
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-11-21 19:03:56 +01:00
|
|
|
bmgrLog.Infof("Lost peer %s", p)
|
2013-10-02 23:49:31 +02:00
|
|
|
|
2013-10-08 17:47:00 +02:00
|
|
|
// Remove requested transactions from the global map so that they will
|
|
|
|
// be fetched from elsewhere next time we get an inv.
|
|
|
|
for k := range p.requestedTxns {
|
|
|
|
delete(b.requestedTxns, k)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Remove requested blocks from the global map so that they will be
|
2013-10-03 18:10:26 +02:00
|
|
|
// fetched from elsewhere next time we get an inv.
|
2013-10-01 00:53:21 +02:00
|
|
|
// TODO(oga) we could possibly here check which peers have these blocks
|
|
|
|
// and request them now to speed things up a little.
|
|
|
|
for k := range p.requestedBlocks {
|
|
|
|
delete(b.requestedBlocks, k)
|
2013-09-03 23:55:14 +02:00
|
|
|
}
|
2013-10-03 18:59:34 +02:00
|
|
|
|
|
|
|
// Attempt to find a new peer to sync from if the quitting peer is the
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
// sync peer. Also, reset the headers-first state if in headers-first
|
|
|
|
// mode so
|
2013-10-03 18:59:34 +02:00
|
|
|
if b.syncPeer != nil && b.syncPeer == p {
|
|
|
|
b.syncPeer = nil
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
if b.headersFirstMode {
|
|
|
|
// This really shouldn't fail. We have a fairly
|
|
|
|
// unrecoverable database issue if it does.
|
|
|
|
newestHash, height, err := b.server.db.NewestSha()
|
|
|
|
if err != nil {
|
|
|
|
bmgrLog.Warnf("Unable to obtain latest "+
|
|
|
|
"block information from the database: "+
|
|
|
|
"%v", err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
b.resetHeaderState(newestHash, height)
|
|
|
|
}
|
2013-10-03 18:59:34 +02:00
|
|
|
b.startSync(peers)
|
|
|
|
}
|
2013-09-03 23:55:14 +02:00
|
|
|
}
|
|
|
|
|
2013-08-06 23:55:22 +02:00
|
|
|
// logBlockHeight logs a new block height as an information message to show
|
|
|
|
// progress to the user. In order to prevent spam, it limits logging to one
|
|
|
|
// message every 10 seconds with duration and totals included.
|
2014-01-30 19:31:13 +01:00
|
|
|
func (b *blockManager) logBlockHeight(block *btcutil.Block) {
|
2013-08-06 23:55:22 +02:00
|
|
|
b.receivedLogBlocks++
|
2014-01-30 19:31:13 +01:00
|
|
|
b.receivedLogTx += int64(len(block.MsgBlock().Transactions))
|
2013-08-06 23:55:22 +02:00
|
|
|
|
|
|
|
now := time.Now()
|
|
|
|
duration := now.Sub(b.lastBlockLogTime)
|
2013-08-29 21:44:43 +02:00
|
|
|
if duration < time.Second*10 {
|
2013-08-06 23:55:22 +02:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
// Truncate the duration to 10s of milliseconds.
|
2013-10-10 21:13:54 +02:00
|
|
|
durationMillis := int64(duration / time.Millisecond)
|
|
|
|
tDuration := 10 * time.Millisecond * time.Duration(durationMillis/10)
|
|
|
|
|
2013-08-06 23:55:22 +02:00
|
|
|
// Log information about new block height.
|
|
|
|
blockStr := "blocks"
|
|
|
|
if b.receivedLogBlocks == 1 {
|
|
|
|
blockStr = "block"
|
|
|
|
}
|
|
|
|
txStr := "transactions"
|
|
|
|
if b.receivedLogTx == 1 {
|
|
|
|
txStr = "transaction"
|
|
|
|
}
|
2014-01-30 19:31:13 +01:00
|
|
|
bmgrLog.Infof("Processed %d %s in the last %s (%d %s, height %d, %s)",
|
2013-10-11 01:24:23 +02:00
|
|
|
b.receivedLogBlocks, blockStr, tDuration, b.receivedLogTx,
|
2014-01-30 19:31:13 +01:00
|
|
|
txStr, block.Height(), block.MsgBlock().Header.Timestamp)
|
2013-08-06 23:55:22 +02:00
|
|
|
|
|
|
|
b.receivedLogBlocks = 0
|
|
|
|
b.receivedLogTx = 0
|
|
|
|
b.lastBlockLogTime = now
|
|
|
|
}
|
|
|
|
|
2013-10-08 17:47:00 +02:00
|
|
|
// handleTxMsg handles transaction messages from all peers.
|
|
|
|
func (b *blockManager) handleTxMsg(tmsg *txMsg) {
|
|
|
|
// Keep track of which peer the tx was sent from.
|
2013-10-28 21:44:38 +01:00
|
|
|
txHash := tmsg.tx.Sha()
|
2013-10-08 17:47:00 +02:00
|
|
|
|
2013-10-30 20:13:29 +01:00
|
|
|
// If we didn't ask for this transaction then the peer is misbehaving.
|
2013-10-28 21:44:38 +01:00
|
|
|
if _, ok := tmsg.peer.requestedTxns[*txHash]; !ok {
|
2013-11-21 19:03:56 +01:00
|
|
|
bmgrLog.Warnf("Got unrequested transaction %v from %s -- "+
|
2013-10-28 21:44:38 +01:00
|
|
|
"disconnecting", txHash, tmsg.peer.addr)
|
2013-10-08 17:47:00 +02:00
|
|
|
tmsg.peer.Disconnect()
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// Process the transaction to include validation, insertion in the
|
|
|
|
// memory pool, orphan handling, etc.
|
|
|
|
err := tmsg.peer.server.txMemPool.ProcessTransaction(tmsg.tx)
|
|
|
|
|
|
|
|
// Remove transaction from request maps. Either the mempool/chain
|
|
|
|
// already knows about it and as such we shouldn't have any more
|
|
|
|
// instances of trying to fetch it, or we failed to insert and thus
|
|
|
|
// we'll retry next time we get an inv.
|
2013-10-28 21:44:38 +01:00
|
|
|
delete(tmsg.peer.requestedTxns, *txHash)
|
|
|
|
delete(b.requestedTxns, *txHash)
|
2013-10-08 17:47:00 +02:00
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
// When the error is a rule error, it means the transaction was
|
|
|
|
// simply rejected as opposed to something actually going wrong,
|
|
|
|
// so log it as such. Otherwise, something really did go wrong,
|
|
|
|
// so log it as an actual error.
|
|
|
|
if _, ok := err.(TxRuleError); ok {
|
2013-11-21 19:03:56 +01:00
|
|
|
bmgrLog.Debugf("Rejected transaction %v: %v", txHash, err)
|
2013-10-08 17:47:00 +02:00
|
|
|
} else {
|
2013-11-21 19:03:56 +01:00
|
|
|
bmgrLog.Errorf("Failed to process transaction %v: %v", txHash, err)
|
2013-10-08 17:47:00 +02:00
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-10-03 21:48:10 +02:00
|
|
|
// current returns true if we believe we are synced with our peers, false if we
|
|
|
|
// still have blocks to check
|
|
|
|
func (b *blockManager) current() bool {
|
|
|
|
if !b.blockChain.IsCurrent() {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
// if blockChain thinks we are current and we have no syncPeer it
|
|
|
|
// is probably right.
|
|
|
|
if b.syncPeer == nil {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
_, height, err := b.server.db.NewestSha()
|
|
|
|
// No matter what chain thinks, if we are below the block we are
|
|
|
|
// syncing to we are not current.
|
|
|
|
// TODO(oga) we can get chain to return the height of each block when we
|
|
|
|
// parse an orphan, which would allow us to update the height of peers
|
|
|
|
// from what it was at initial handshake.
|
|
|
|
if err != nil || height < int64(b.syncPeer.lastBlock) {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2013-08-29 21:44:43 +02:00
|
|
|
// handleBlockMsg handles block messages from all peers.
|
|
|
|
func (b *blockManager) handleBlockMsg(bmsg *blockMsg) {
|
2013-10-03 18:21:41 +02:00
|
|
|
// If we didn't ask for this block then the peer is misbehaving.
|
2014-01-29 01:53:25 +01:00
|
|
|
blockSha, _ := bmsg.block.Sha()
|
2013-10-01 22:32:50 +02:00
|
|
|
if _, ok := bmsg.peer.requestedBlocks[*blockSha]; !ok {
|
2013-10-03 18:21:41 +02:00
|
|
|
// The regression test intentionally sends some blocks twice
|
|
|
|
// to test duplicate block insertion fails. Don't disconnect
|
|
|
|
// the peer or ignore the block when we're in regression test
|
|
|
|
// mode in this case so the chain code is actually fed the
|
|
|
|
// duplicate blocks.
|
|
|
|
if !cfg.RegressionTest {
|
2013-11-21 19:03:56 +01:00
|
|
|
bmgrLog.Warnf("Got unrequested block %v from %s -- "+
|
2013-10-03 18:21:41 +02:00
|
|
|
"disconnecting", blockSha, bmsg.peer.addr)
|
|
|
|
bmsg.peer.Disconnect()
|
|
|
|
return
|
|
|
|
}
|
2013-10-01 22:32:50 +02:00
|
|
|
}
|
2014-01-29 01:53:25 +01:00
|
|
|
|
|
|
|
// Keep track of which peer the block was sent from so the notification
|
|
|
|
// handler can request the parent blocks from the appropriate peer.
|
2013-08-29 21:44:43 +02:00
|
|
|
b.blockPeer[*blockSha] = bmsg.peer
|
2013-08-06 23:55:22 +02:00
|
|
|
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
// When in headers-first mode, if the block matches the hash of the
|
|
|
|
// first header in the list of headers that are being fetched, it's
|
|
|
|
// eligible for less validation since the headers have already been
|
|
|
|
// verified to link together and are valid up to the next checkpoint.
|
|
|
|
// Also, remove the list entry for all blocks except the checkpoint
|
|
|
|
// since it is needed to verify the next round of headers links
|
|
|
|
// properly.
|
|
|
|
isCheckpointBlock := false
|
2013-11-16 00:16:51 +01:00
|
|
|
fastAdd := false
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
if b.headersFirstMode {
|
|
|
|
firstNodeEl := b.headerList.Front()
|
|
|
|
if firstNodeEl != nil {
|
|
|
|
firstNode := firstNodeEl.Value.(*headerNode)
|
|
|
|
if blockSha.IsEqual(firstNode.sha) {
|
2013-11-16 00:16:51 +01:00
|
|
|
fastAdd = true
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
if firstNode.sha.IsEqual(b.nextCheckpoint.Hash) {
|
|
|
|
isCheckpointBlock = true
|
|
|
|
} else {
|
|
|
|
b.headerList.Remove(firstNodeEl)
|
|
|
|
}
|
2013-11-16 00:16:51 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2014-01-29 01:53:25 +01:00
|
|
|
|
|
|
|
// Remove block from request maps. Either chain will know about it and
|
|
|
|
// so we shouldn't have any more instances of trying to fetch it, or we
|
|
|
|
// will fail the insert and thus we'll retry next time we get an inv.
|
|
|
|
delete(bmsg.peer.requestedBlocks, *blockSha)
|
|
|
|
delete(b.requestedBlocks, *blockSha)
|
2013-11-16 00:16:51 +01:00
|
|
|
|
2014-01-29 01:53:25 +01:00
|
|
|
// Process the block to include validation, best chain selection, orphan
|
|
|
|
// handling, etc.
|
|
|
|
err := b.blockChain.ProcessBlock(bmsg.block, fastAdd)
|
2013-08-06 23:55:22 +02:00
|
|
|
if err != nil {
|
2013-08-29 21:44:43 +02:00
|
|
|
delete(b.blockPeer, *blockSha)
|
2013-10-04 20:13:16 +02:00
|
|
|
|
|
|
|
// When the error is a rule error, it means the block was simply
|
|
|
|
// rejected as opposed to something actually going wrong, so log
|
|
|
|
// it as such. Otherwise, something really did go wrong, so log
|
|
|
|
// it as an actual error.
|
|
|
|
if _, ok := err.(btcchain.RuleError); ok {
|
2013-11-21 19:03:56 +01:00
|
|
|
bmgrLog.Infof("Rejected block %v: %v", blockSha, err)
|
2013-10-04 20:13:16 +02:00
|
|
|
} else {
|
2013-11-21 19:03:56 +01:00
|
|
|
bmgrLog.Errorf("Failed to process block %v: %v", blockSha, err)
|
2013-10-04 20:13:16 +02:00
|
|
|
}
|
2013-08-06 23:55:22 +02:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2014-01-30 19:31:13 +01:00
|
|
|
// When the block is not an orphan, don't keep track of the peer that
|
|
|
|
// sent it any longer and log information about it.
|
2013-08-29 21:44:43 +02:00
|
|
|
if !b.blockChain.IsKnownOrphan(blockSha) {
|
|
|
|
delete(b.blockPeer, *blockSha)
|
2014-01-30 19:31:13 +01:00
|
|
|
b.logBlockHeight(bmsg.block)
|
2013-08-29 21:44:43 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Sync the db to disk.
|
|
|
|
b.server.db.Sync()
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
|
|
|
|
// Nothing more to do if we aren't in headers-first mode.
|
|
|
|
if !b.headersFirstMode {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// This is headers-first mode, so if the block is not a checkpoint
|
|
|
|
// request more blocks using the header list when the request queue is
|
|
|
|
// getting short.
|
|
|
|
if !isCheckpointBlock {
|
|
|
|
if b.startHeader != nil &&
|
|
|
|
len(bmsg.peer.requestedBlocks) < minInFlightBlocks {
|
|
|
|
b.fetchHeaderBlocks()
|
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// This is headers-first mode and the block is a checkpoint. When
|
|
|
|
// there is a next checkpoint, get the next round of headers by asking
|
|
|
|
// for headers starting from the block after this one up to the next
|
|
|
|
// checkpoint.
|
|
|
|
prevHeight := b.nextCheckpoint.Height
|
|
|
|
prevHash := b.nextCheckpoint.Hash
|
|
|
|
b.nextCheckpoint = b.findNextHeaderCheckpoint(prevHeight)
|
|
|
|
if b.nextCheckpoint != nil {
|
2014-02-04 22:34:28 +01:00
|
|
|
locator := btcchain.BlockLocator([]*btcwire.ShaHash{prevHash})
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
err := bmsg.peer.PushGetHeadersMsg(locator, b.nextCheckpoint.Hash)
|
|
|
|
if err != nil {
|
|
|
|
bmgrLog.Warnf("Failed to send getheaders message to "+
|
|
|
|
"peer %s: %v", bmsg.peer.addr, err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
bmgrLog.Infof("Downloading headers for blocks %d to %d from "+
|
|
|
|
"peer %s", prevHeight+1, b.nextCheckpoint.Height,
|
|
|
|
b.syncPeer.addr)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// This is headers-first mode, the block is a checkpoint, and there are
|
|
|
|
// no more checkpoints, so switch to normal mode by requesting blocks
|
|
|
|
// from the block after this one up to the end of the chain (zero hash).
|
|
|
|
b.headersFirstMode = false
|
|
|
|
b.headerList.Init()
|
|
|
|
bmgrLog.Infof("Reached the final checkpoint -- switching to normal mode")
|
2014-02-04 22:34:28 +01:00
|
|
|
locator := btcchain.BlockLocator([]*btcwire.ShaHash{blockSha})
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
err = bmsg.peer.PushGetBlocksMsg(locator, &zeroHash)
|
|
|
|
if err != nil {
|
|
|
|
bmgrLog.Warnf("Failed to send getblocks message to peer %s: %v",
|
|
|
|
bmsg.peer.addr, err)
|
|
|
|
return
|
|
|
|
}
|
2013-08-06 23:55:22 +02:00
|
|
|
}
|
|
|
|
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
// fetchHeaderBlocks creates and sends a request to the syncPeer for the next
|
|
|
|
// list of blocks to be downloaded based on the current list of headers.
|
2014-01-29 01:53:25 +01:00
|
|
|
func (b *blockManager) fetchHeaderBlocks() {
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
// Nothing to do if there is not start header.
|
|
|
|
if b.startHeader == nil {
|
|
|
|
bmgrLog.Warnf("fetchHeaderBlocks called with no start header")
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// Build up a getdata request for the list of blocks the headers
|
|
|
|
// describe. The size hint will be limited to btcwire.MaxInvPerMsg by
|
|
|
|
// the function, so no need to double check it here.
|
|
|
|
gdmsg := btcwire.NewMsgGetDataSizeHint(uint(b.headerList.Len()))
|
2014-01-29 01:53:25 +01:00
|
|
|
numRequested := 0
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
for e := b.startHeader; e != nil; e = e.Next() {
|
|
|
|
node, ok := e.Value.(*headerNode)
|
2014-01-29 01:53:25 +01:00
|
|
|
if !ok {
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
bmgrLog.Warn("Header list node type is not a headerNode")
|
|
|
|
continue
|
2014-01-29 01:53:25 +01:00
|
|
|
}
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
|
|
|
|
iv := btcwire.NewInvVect(btcwire.InvTypeBlock, node.sha)
|
2014-01-29 01:53:25 +01:00
|
|
|
if !b.haveInventory(iv) {
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
b.requestedBlocks[*node.sha] = true
|
|
|
|
b.syncPeer.requestedBlocks[*node.sha] = true
|
2014-01-29 01:53:25 +01:00
|
|
|
gdmsg.AddInvVect(iv)
|
|
|
|
numRequested++
|
|
|
|
}
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
b.startHeader = e.Next()
|
2014-01-29 01:53:25 +01:00
|
|
|
if numRequested >= btcwire.MaxInvPerMsg {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if len(gdmsg.InvList) > 0 {
|
|
|
|
b.syncPeer.QueueMessage(gdmsg, nil)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// handleHeadersMsghandles headers messages from all peers.
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
func (b *blockManager) handleHeadersMsg(hmsg *headersMsg) {
|
|
|
|
// The remote peer is misbehaving if we didn't request headers.
|
|
|
|
msg := hmsg.headers
|
|
|
|
numHeaders := len(msg.Headers)
|
|
|
|
if !b.headersFirstMode {
|
|
|
|
bmgrLog.Warnf("Got %d unrequested headers from %s -- "+
|
|
|
|
"disconnecting", numHeaders, hmsg.peer.addr)
|
|
|
|
hmsg.peer.Disconnect()
|
2014-01-29 01:53:25 +01:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
// Nothing to do for an empty headers message.
|
|
|
|
if numHeaders == 0 {
|
|
|
|
return
|
2014-01-29 01:53:25 +01:00
|
|
|
}
|
|
|
|
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
// Process all of the received headers ensuring each one connects to the
|
|
|
|
// previous and that checkpoints match.
|
|
|
|
receivedCheckpoint := false
|
|
|
|
var finalHash *btcwire.ShaHash
|
|
|
|
for _, blockHeader := range msg.Headers {
|
|
|
|
blockHash, err := blockHeader.BlockSha()
|
|
|
|
if err != nil {
|
|
|
|
bmgrLog.Warnf("Failed to compute hash of header "+
|
|
|
|
"received from peer %s -- disconnecting",
|
|
|
|
hmsg.peer.addr)
|
|
|
|
hmsg.peer.Disconnect()
|
|
|
|
return
|
|
|
|
}
|
|
|
|
finalHash = &blockHash
|
|
|
|
|
|
|
|
// Ensure there is a previous header to compare against.
|
|
|
|
prevNodeEl := b.headerList.Back()
|
|
|
|
if prevNodeEl == nil {
|
|
|
|
bmgrLog.Warnf("Header list does not contain a previous" +
|
|
|
|
"element as expected -- disconnecting peer")
|
|
|
|
hmsg.peer.Disconnect()
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// Ensure the header properly connects to the previous one and
|
|
|
|
// add it to the list of headers.
|
|
|
|
node := headerNode{sha: &blockHash}
|
|
|
|
prevNode := prevNodeEl.Value.(*headerNode)
|
|
|
|
if prevNode.sha.IsEqual(&blockHeader.PrevBlock) {
|
|
|
|
node.height = prevNode.height + 1
|
|
|
|
e := b.headerList.PushBack(&node)
|
|
|
|
if b.startHeader == nil {
|
|
|
|
b.startHeader = e
|
2014-01-29 01:53:25 +01:00
|
|
|
}
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
} else {
|
|
|
|
bmgrLog.Warnf("Received block header that does not"+
|
|
|
|
"properly connect to the chain from peer %s "+
|
|
|
|
"-- disconnecting", hmsg.peer.addr)
|
|
|
|
hmsg.peer.Disconnect()
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// Verify the header at the next checkpoint height matches.
|
|
|
|
if node.height == b.nextCheckpoint.Height {
|
|
|
|
if node.sha.IsEqual(b.nextCheckpoint.Hash) {
|
|
|
|
receivedCheckpoint = true
|
|
|
|
bmgrLog.Infof("Verified downloaded block "+
|
|
|
|
"header against checkpoint at height "+
|
|
|
|
"%d/hash %s", node.height, node.sha)
|
2014-01-29 01:53:25 +01:00
|
|
|
} else {
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
bmgrLog.Warnf("Block header at height %d/hash "+
|
|
|
|
"%s from peer %s does NOT match "+
|
|
|
|
"expected checkpoint hash of %s -- "+
|
|
|
|
"disconnecting", node.height,
|
|
|
|
node.sha, hmsg.peer.addr,
|
|
|
|
b.nextCheckpoint.Hash)
|
|
|
|
hmsg.peer.Disconnect()
|
|
|
|
return
|
2014-01-29 01:53:25 +01:00
|
|
|
}
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
break
|
2014-01-29 01:53:25 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
// When this header is a checkpoint, switch to fetching the blocks for
|
|
|
|
// all of the headers since the last checkpoint.
|
|
|
|
if receivedCheckpoint {
|
|
|
|
// Since the first entry of the list is always the final block
|
|
|
|
// that is already in the database and is only used to ensure
|
|
|
|
// the next header links properly, it must be removed before
|
|
|
|
// fetching the blocks.
|
|
|
|
b.headerList.Remove(b.headerList.Front())
|
|
|
|
bmgrLog.Infof("Received %v block headers: Fetching blocks",
|
|
|
|
b.headerList.Len())
|
|
|
|
b.lastBlockLogTime = time.Now()
|
|
|
|
b.fetchHeaderBlocks()
|
2014-01-29 01:53:25 +01:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
// This header is not a checkpoint, so request the next batch of
|
|
|
|
// headers starting from the latest known header and ending with the
|
|
|
|
// next checkpoint.
|
2014-02-04 22:34:28 +01:00
|
|
|
locator := btcchain.BlockLocator([]*btcwire.ShaHash{finalHash})
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
err := hmsg.peer.PushGetHeadersMsg(locator, b.nextCheckpoint.Hash)
|
|
|
|
if err != nil {
|
|
|
|
bmgrLog.Warnf("Failed to send getheaders message to "+
|
|
|
|
"peer %s: %v", hmsg.peer.addr, err)
|
|
|
|
return
|
|
|
|
}
|
2014-01-29 01:53:25 +01:00
|
|
|
}
|
|
|
|
|
2013-10-08 17:47:00 +02:00
|
|
|
// haveInventory returns whether or not the inventory represented by the passed
|
|
|
|
// inventory vector is known. This includes checking all of the various places
|
|
|
|
// inventory can be when it is in different states such as blocks that are part
|
|
|
|
// of the main chain, on a side chain, in the orphan pool, and transactions that
|
2014-01-11 05:32:05 +01:00
|
|
|
// are in the memory pool (either the main pool or orphan pool).
|
2013-10-08 17:47:00 +02:00
|
|
|
func (b *blockManager) haveInventory(invVect *btcwire.InvVect) bool {
|
|
|
|
switch invVect.Type {
|
2014-01-19 04:10:36 +01:00
|
|
|
case btcwire.InvTypeBlock:
|
2013-10-08 17:47:00 +02:00
|
|
|
// Ask chain if the block is known to it in any form (main
|
|
|
|
// chain, side chain, or orphan).
|
|
|
|
return b.blockChain.HaveBlock(&invVect.Hash)
|
|
|
|
|
2014-01-19 04:10:36 +01:00
|
|
|
case btcwire.InvTypeTx:
|
2013-10-08 17:47:00 +02:00
|
|
|
// Ask the transaction memory pool if the transaction is known
|
|
|
|
// to it in any form (main pool or orphan).
|
2013-10-11 21:12:40 +02:00
|
|
|
if b.server.txMemPool.HaveTransaction(&invVect.Hash) {
|
2013-10-08 17:47:00 +02:00
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check if the transaction exists from the point of view of the
|
|
|
|
// end of the main chain.
|
|
|
|
return b.server.db.ExistsTxSha(&invVect.Hash)
|
|
|
|
}
|
|
|
|
|
|
|
|
// The requested inventory is is an unsupported type, so just claim
|
|
|
|
// it is known to avoid requesting it.
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2013-09-27 02:41:02 +02:00
|
|
|
// handleInvMsg handles inv messages from all peers.
|
|
|
|
// We examine the inventory advertised by the remote peer and act accordingly.
|
|
|
|
func (b *blockManager) handleInvMsg(imsg *invMsg) {
|
2013-10-03 17:25:13 +02:00
|
|
|
// Ignore invs from peers that aren't the sync if we are not current.
|
|
|
|
// Helps prevent fetching a mass of orphans.
|
2013-10-03 21:48:10 +02:00
|
|
|
if imsg.peer != b.syncPeer && !b.current() {
|
2013-10-03 17:25:13 +02:00
|
|
|
return
|
|
|
|
}
|
2013-10-04 05:31:00 +02:00
|
|
|
|
2013-09-27 02:41:02 +02:00
|
|
|
// Attempt to find the final block in the inventory list. There may
|
|
|
|
// not be one.
|
|
|
|
lastBlock := -1
|
|
|
|
invVects := imsg.inv.InvList
|
|
|
|
for i := len(invVects) - 1; i >= 0; i-- {
|
2013-10-08 22:55:07 +02:00
|
|
|
if invVects[i].Type == btcwire.InvTypeBlock {
|
2013-09-27 02:41:02 +02:00
|
|
|
lastBlock = i
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Request the advertised inventory if we don't already have it. Also,
|
|
|
|
// request parent blocks of orphans if we receive one we already have.
|
|
|
|
// Finally, attempt to detect potential stalls due to long side chains
|
|
|
|
// we already have and request more blocks to prevent them.
|
2013-09-20 19:55:27 +02:00
|
|
|
chain := b.blockChain
|
2013-09-27 02:41:02 +02:00
|
|
|
for i, iv := range invVects {
|
2013-09-20 19:55:27 +02:00
|
|
|
// Ignore unsupported inventory types.
|
2013-10-08 22:55:07 +02:00
|
|
|
if iv.Type != btcwire.InvTypeBlock && iv.Type != btcwire.InvTypeTx {
|
2013-09-20 19:55:27 +02:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
// Add the inventory to the cache of known inventory
|
|
|
|
// for the peer.
|
2013-12-24 17:32:20 +01:00
|
|
|
imsg.peer.AddKnownInventory(iv)
|
2013-09-20 19:55:27 +02:00
|
|
|
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
// Ignore inventory when we're in headers-first mode.
|
|
|
|
if b.headersFirstMode {
|
|
|
|
continue
|
2013-11-16 00:16:51 +01:00
|
|
|
}
|
|
|
|
|
2013-09-20 19:55:27 +02:00
|
|
|
// Request the inventory if we don't already have it.
|
2013-10-08 17:47:00 +02:00
|
|
|
if !b.haveInventory(iv) {
|
2013-09-20 19:55:27 +02:00
|
|
|
// Add it to the request queue.
|
|
|
|
imsg.peer.requestQueue.PushBack(iv)
|
|
|
|
continue
|
|
|
|
}
|
2013-09-27 02:41:02 +02:00
|
|
|
|
2013-10-08 22:55:07 +02:00
|
|
|
if iv.Type == btcwire.InvTypeBlock {
|
2013-09-27 02:41:02 +02:00
|
|
|
// The block is an orphan block that we already have.
|
|
|
|
// When the existing orphan was processed, it requested
|
|
|
|
// the missing parent blocks. When this scenario
|
|
|
|
// happens, it means there were more blocks missing
|
|
|
|
// than are allowed into a single inventory message. As
|
|
|
|
// a result, once this peer requested the final
|
|
|
|
// advertised block, the remote peer noticed and is now
|
|
|
|
// resending the orphan block as an available block
|
|
|
|
// to signal there are more missing blocks that need to
|
|
|
|
// be requested.
|
2013-09-20 19:55:27 +02:00
|
|
|
if chain.IsKnownOrphan(&iv.Hash) {
|
2013-09-27 02:41:02 +02:00
|
|
|
// Request blocks starting at the latest known
|
|
|
|
// up to the root of the orphan that just came
|
|
|
|
// in.
|
2013-09-20 19:55:27 +02:00
|
|
|
orphanRoot := chain.GetOrphanRoot(&iv.Hash)
|
|
|
|
locator, err := chain.LatestBlockLocator()
|
2013-09-27 02:41:02 +02:00
|
|
|
if err != nil {
|
2013-11-21 19:03:56 +01:00
|
|
|
bmgrLog.Errorf("PEER: Failed to get block "+
|
2013-09-27 02:41:02 +02:00
|
|
|
"locator for the latest block: "+
|
|
|
|
"%v", err)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
imsg.peer.PushGetBlocksMsg(locator, orphanRoot)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
// We already have the final block advertised by this
|
|
|
|
// inventory message, so force a request for more. This
|
2013-09-27 04:06:01 +02:00
|
|
|
// should only happen if we're on a really long side
|
|
|
|
// chain.
|
2013-09-27 02:41:02 +02:00
|
|
|
if i == lastBlock {
|
|
|
|
// Request blocks after this one up to the
|
|
|
|
// final one the remote peer knows about (zero
|
|
|
|
// stop hash).
|
2013-09-20 19:55:27 +02:00
|
|
|
locator := chain.BlockLocatorFromHash(&iv.Hash)
|
2013-09-27 02:41:02 +02:00
|
|
|
imsg.peer.PushGetBlocksMsg(locator, &zeroHash)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Request as much as possible at once. Anything that won't fit into
|
|
|
|
// the request will be requested on the next inv message.
|
|
|
|
numRequested := 0
|
|
|
|
gdmsg := btcwire.NewMsgGetData()
|
2013-09-20 19:55:27 +02:00
|
|
|
requestQueue := imsg.peer.requestQueue
|
|
|
|
for e := requestQueue.Front(); e != nil; e = requestQueue.Front() {
|
2013-09-27 02:41:02 +02:00
|
|
|
iv := e.Value.(*btcwire.InvVect)
|
|
|
|
imsg.peer.requestQueue.Remove(e)
|
2013-10-08 17:47:00 +02:00
|
|
|
|
|
|
|
switch iv.Type {
|
2014-01-19 04:10:36 +01:00
|
|
|
case btcwire.InvTypeBlock:
|
2013-10-08 17:47:00 +02:00
|
|
|
// Request the block if there is not already a pending
|
|
|
|
// request.
|
|
|
|
if _, exists := b.requestedBlocks[iv.Hash]; !exists {
|
|
|
|
b.requestedBlocks[iv.Hash] = true
|
|
|
|
imsg.peer.requestedBlocks[iv.Hash] = true
|
|
|
|
gdmsg.AddInvVect(iv)
|
|
|
|
numRequested++
|
|
|
|
}
|
|
|
|
|
2014-01-19 04:10:36 +01:00
|
|
|
case btcwire.InvTypeTx:
|
2013-10-08 17:47:00 +02:00
|
|
|
// Request the transaction if there is not already a
|
|
|
|
// pending request.
|
|
|
|
if _, exists := b.requestedTxns[iv.Hash]; !exists {
|
|
|
|
b.requestedTxns[iv.Hash] = true
|
|
|
|
imsg.peer.requestedTxns[iv.Hash] = true
|
|
|
|
gdmsg.AddInvVect(iv)
|
|
|
|
numRequested++
|
|
|
|
}
|
2013-10-01 00:53:21 +02:00
|
|
|
}
|
2013-09-27 02:41:02 +02:00
|
|
|
|
|
|
|
if numRequested >= btcwire.MaxInvPerMsg {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if len(gdmsg.InvList) > 0 {
|
2013-10-16 16:49:09 +02:00
|
|
|
imsg.peer.QueueMessage(gdmsg, nil)
|
2013-09-27 02:41:02 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-08-29 21:44:43 +02:00
|
|
|
// blockHandler is the main handler for the block manager. It must be run
|
|
|
|
// as a goroutine. It processes block and inv messages in a separate goroutine
|
2013-10-04 07:34:24 +02:00
|
|
|
// from the peer handlers so the block (MsgBlock) messages are handled by a
|
|
|
|
// single thread without needing to lock memory data structures. This is
|
|
|
|
// important because the block manager controls which blocks are needed and how
|
|
|
|
// the fetching should proceed.
|
2013-08-06 23:55:22 +02:00
|
|
|
func (b *blockManager) blockHandler() {
|
2013-10-01 00:53:21 +02:00
|
|
|
candidatePeers := list.New()
|
2013-08-06 23:55:22 +02:00
|
|
|
out:
|
2013-10-03 01:33:42 +02:00
|
|
|
for {
|
2013-08-06 23:55:22 +02:00
|
|
|
select {
|
2013-10-01 00:53:21 +02:00
|
|
|
case m := <-b.msgChan:
|
|
|
|
switch msg := m.(type) {
|
|
|
|
case *newPeerMsg:
|
|
|
|
b.handleNewPeerMsg(candidatePeers, msg.peer)
|
|
|
|
|
2013-10-08 17:47:00 +02:00
|
|
|
case *txMsg:
|
|
|
|
b.handleTxMsg(msg)
|
|
|
|
msg.peer.txProcessed <- true
|
|
|
|
|
2013-10-01 00:53:21 +02:00
|
|
|
case *blockMsg:
|
|
|
|
b.handleBlockMsg(msg)
|
|
|
|
msg.peer.blockProcessed <- true
|
2013-09-27 04:06:01 +02:00
|
|
|
|
2013-10-01 00:53:21 +02:00
|
|
|
case *invMsg:
|
|
|
|
b.handleInvMsg(msg)
|
2013-08-06 23:55:22 +02:00
|
|
|
|
2013-11-16 00:16:51 +01:00
|
|
|
case *headersMsg:
|
|
|
|
b.handleHeadersMsg(msg)
|
|
|
|
|
2013-10-01 00:53:21 +02:00
|
|
|
case *donePeerMsg:
|
|
|
|
b.handleDonePeerMsg(candidatePeers, msg.peer)
|
|
|
|
|
|
|
|
default:
|
2014-02-04 16:44:01 +01:00
|
|
|
bmgrLog.Warnf("Invalid message type in block "+
|
|
|
|
"handler: %T", msg)
|
2013-10-01 00:53:21 +02:00
|
|
|
}
|
2013-10-03 01:50:36 +02:00
|
|
|
|
2014-02-04 16:39:00 +01:00
|
|
|
// Queries used for atomically retrieving internal state.
|
|
|
|
case qmsg := <-b.query:
|
|
|
|
switch msg := qmsg.(type) {
|
|
|
|
case getSyncPeerMsg:
|
|
|
|
msg.reply <- b.syncPeer
|
|
|
|
|
|
|
|
default:
|
|
|
|
bmgrLog.Warnf("Invalid query type in block "+
|
|
|
|
"handler query: %T", msg)
|
|
|
|
}
|
2014-02-04 07:24:40 +01:00
|
|
|
|
2013-08-06 23:55:22 +02:00
|
|
|
case <-b.quit:
|
|
|
|
break out
|
|
|
|
}
|
|
|
|
}
|
|
|
|
b.wg.Done()
|
2013-11-21 19:03:56 +01:00
|
|
|
bmgrLog.Trace("Block handler done")
|
2013-08-06 23:55:22 +02:00
|
|
|
}
|
|
|
|
|
2013-09-29 22:26:03 +02:00
|
|
|
// handleNotifyMsg handles notifications from btcchain. It does things such
|
|
|
|
// as request orphan block parents and relay accepted blocks to connected peers.
|
2013-08-06 23:55:22 +02:00
|
|
|
func (b *blockManager) handleNotifyMsg(notification *btcchain.Notification) {
|
|
|
|
switch notification.Type {
|
2013-09-29 22:26:03 +02:00
|
|
|
// An orphan block has been accepted by the block chain. Request
|
|
|
|
// its parents from the peer that sent it.
|
2013-08-06 23:55:22 +02:00
|
|
|
case btcchain.NTOrphanBlock:
|
2013-09-10 02:58:38 +02:00
|
|
|
orphanHash := notification.Data.(*btcwire.ShaHash)
|
|
|
|
if peer, exists := b.blockPeer[*orphanHash]; exists {
|
|
|
|
orphanRoot := b.blockChain.GetOrphanRoot(orphanHash)
|
2013-08-29 21:44:43 +02:00
|
|
|
locator, err := b.blockChain.LatestBlockLocator()
|
|
|
|
if err != nil {
|
2013-11-21 19:03:56 +01:00
|
|
|
bmgrLog.Errorf("Failed to get block locator "+
|
2013-08-29 21:44:43 +02:00
|
|
|
"for the latest block: %v", err)
|
|
|
|
break
|
|
|
|
}
|
2013-09-27 02:41:02 +02:00
|
|
|
peer.PushGetBlocksMsg(locator, orphanRoot)
|
2013-08-29 21:44:43 +02:00
|
|
|
delete(b.blockPeer, *orphanRoot)
|
2013-09-12 21:19:10 +02:00
|
|
|
} else {
|
2013-11-21 19:03:56 +01:00
|
|
|
bmgrLog.Warnf("Notification for orphan %v with no peer",
|
2013-09-12 21:19:10 +02:00
|
|
|
orphanHash)
|
2013-08-29 21:44:43 +02:00
|
|
|
}
|
2013-08-06 23:55:22 +02:00
|
|
|
|
2013-09-29 22:26:03 +02:00
|
|
|
// A block has been accepted into the block chain. Relay it to other
|
|
|
|
// peers.
|
2013-08-06 23:55:22 +02:00
|
|
|
case btcchain.NTBlockAccepted:
|
2013-10-03 17:25:13 +02:00
|
|
|
// Don't relay if we are not current. Other peers that are
|
|
|
|
// current should already know about it.
|
2013-10-03 21:48:10 +02:00
|
|
|
|
|
|
|
if !b.current() {
|
2013-10-03 17:25:13 +02:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2013-09-09 17:58:56 +02:00
|
|
|
block, ok := notification.Data.(*btcutil.Block)
|
|
|
|
if !ok {
|
2013-11-21 19:03:56 +01:00
|
|
|
bmgrLog.Warnf("Chain accepted notification is not a block.")
|
2013-09-09 17:58:56 +02:00
|
|
|
break
|
|
|
|
}
|
|
|
|
|
|
|
|
// It's ok to ignore the error here since the notification is
|
|
|
|
// coming from the chain code which has already cached the hash.
|
|
|
|
hash, _ := block.Sha()
|
|
|
|
|
|
|
|
// Generate the inventory vector and relay it.
|
2013-10-08 22:55:07 +02:00
|
|
|
iv := btcwire.NewInvVect(btcwire.InvTypeBlock, hash)
|
2013-09-09 17:58:56 +02:00
|
|
|
b.server.RelayInventory(iv)
|
2013-09-20 19:55:27 +02:00
|
|
|
|
|
|
|
// A block has been connected to the main block chain.
|
|
|
|
case btcchain.NTBlockConnected:
|
|
|
|
block, ok := notification.Data.(*btcutil.Block)
|
|
|
|
if !ok {
|
2013-11-21 19:03:56 +01:00
|
|
|
bmgrLog.Warnf("Chain connected notification is not a block.")
|
2013-09-20 19:55:27 +02:00
|
|
|
break
|
|
|
|
}
|
|
|
|
|
|
|
|
// Remove all of the transactions (except the coinbase) in the
|
2013-11-15 23:12:23 +01:00
|
|
|
// connected block from the transaction pool. Also, remove any
|
|
|
|
// transactions which are now double spends as a result of these
|
|
|
|
// new transactions. Note that removing a transaction from
|
|
|
|
// pool also removes any transactions which depend on it,
|
|
|
|
// recursively.
|
2013-10-28 21:44:38 +01:00
|
|
|
for _, tx := range block.Transactions()[1:] {
|
2013-11-15 23:12:23 +01:00
|
|
|
b.server.txMemPool.RemoveTransaction(tx)
|
|
|
|
b.server.txMemPool.RemoveDoubleSpends(tx)
|
2013-09-20 19:55:27 +02:00
|
|
|
}
|
|
|
|
|
2014-03-04 17:15:25 +01:00
|
|
|
// Notify registered websocket clients
|
2013-08-14 22:55:31 +02:00
|
|
|
if r := b.server.rpcServer; r != nil {
|
2014-03-04 17:15:25 +01:00
|
|
|
r.ntfnMgr.NotifyBlockConnected(block)
|
2013-08-14 22:55:31 +02:00
|
|
|
}
|
|
|
|
|
2013-09-20 19:55:27 +02:00
|
|
|
// A block has been disconnected from the main block chain.
|
|
|
|
case btcchain.NTBlockDisconnected:
|
|
|
|
block, ok := notification.Data.(*btcutil.Block)
|
|
|
|
if !ok {
|
2013-11-21 19:03:56 +01:00
|
|
|
bmgrLog.Warnf("Chain disconnected notification is not a block.")
|
2013-09-20 19:55:27 +02:00
|
|
|
break
|
|
|
|
}
|
|
|
|
|
|
|
|
// Reinsert all of the transactions (except the coinbase) into
|
|
|
|
// the transaction pool.
|
2013-10-28 21:44:38 +01:00
|
|
|
for _, tx := range block.Transactions()[1:] {
|
2014-02-08 23:15:17 +01:00
|
|
|
err := b.server.txMemPool.MaybeAcceptTransaction(tx, nil, false)
|
2013-09-20 19:55:27 +02:00
|
|
|
if err != nil {
|
|
|
|
// Remove the transaction and all transactions
|
|
|
|
// that depend on it if it wasn't accepted into
|
|
|
|
// the transaction pool.
|
2013-11-15 23:12:23 +01:00
|
|
|
b.server.txMemPool.RemoveTransaction(tx)
|
2013-09-20 19:55:27 +02:00
|
|
|
}
|
|
|
|
}
|
2013-08-14 22:55:31 +02:00
|
|
|
|
2014-03-04 17:15:25 +01:00
|
|
|
// Notify registered websocket clients
|
2013-08-14 22:55:31 +02:00
|
|
|
if r := b.server.rpcServer; r != nil {
|
2014-03-04 17:15:25 +01:00
|
|
|
r.ntfnMgr.NotifyBlockDisconnected(block)
|
2013-08-14 22:55:31 +02:00
|
|
|
}
|
2013-08-06 23:55:22 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-10-08 00:27:59 +02:00
|
|
|
// NewPeer informs the block manager of a newly active peer.
|
2013-10-01 00:53:21 +02:00
|
|
|
func (b *blockManager) NewPeer(p *peer) {
|
|
|
|
// Ignore if we are shutting down.
|
2013-10-03 01:33:42 +02:00
|
|
|
if atomic.LoadInt32(&b.shutdown) != 0 {
|
2013-10-01 00:53:21 +02:00
|
|
|
return
|
|
|
|
}
|
2013-10-08 00:27:59 +02:00
|
|
|
|
2013-10-01 00:53:21 +02:00
|
|
|
b.msgChan <- &newPeerMsg{peer: p}
|
|
|
|
}
|
|
|
|
|
2013-10-08 17:47:00 +02:00
|
|
|
// QueueTx adds the passed transaction message and peer to the block handling
|
|
|
|
// queue.
|
2013-10-28 21:44:38 +01:00
|
|
|
func (b *blockManager) QueueTx(tx *btcutil.Tx, p *peer) {
|
2013-10-08 17:47:00 +02:00
|
|
|
// Don't accept more transactions if we're shutting down.
|
|
|
|
if atomic.LoadInt32(&b.shutdown) != 0 {
|
|
|
|
p.txProcessed <- false
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
b.msgChan <- &txMsg{tx: tx, peer: p}
|
|
|
|
}
|
|
|
|
|
2013-08-06 23:55:22 +02:00
|
|
|
// QueueBlock adds the passed block message and peer to the block handling queue.
|
2013-08-16 20:35:38 +02:00
|
|
|
func (b *blockManager) QueueBlock(block *btcutil.Block, p *peer) {
|
2013-08-06 23:55:22 +02:00
|
|
|
// Don't accept more blocks if we're shutting down.
|
2013-10-03 01:33:42 +02:00
|
|
|
if atomic.LoadInt32(&b.shutdown) != 0 {
|
2013-08-16 20:35:38 +02:00
|
|
|
p.blockProcessed <- false
|
2013-08-06 23:55:22 +02:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2013-10-08 00:27:59 +02:00
|
|
|
b.msgChan <- &blockMsg{block: block, peer: p}
|
2013-08-06 23:55:22 +02:00
|
|
|
}
|
|
|
|
|
2013-09-27 02:41:02 +02:00
|
|
|
// QueueInv adds the passed inv message and peer to the block handling queue.
|
|
|
|
func (b *blockManager) QueueInv(inv *btcwire.MsgInv, p *peer) {
|
2013-09-27 04:06:01 +02:00
|
|
|
// No channel handling here because peers do not need to block on inv
|
2013-09-27 02:41:02 +02:00
|
|
|
// messages.
|
2013-10-03 01:33:42 +02:00
|
|
|
if atomic.LoadInt32(&b.shutdown) != 0 {
|
2013-09-27 02:41:02 +02:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2013-10-08 00:27:59 +02:00
|
|
|
b.msgChan <- &invMsg{inv: inv, peer: p}
|
2013-10-01 00:53:21 +02:00
|
|
|
}
|
|
|
|
|
2014-01-29 01:53:25 +01:00
|
|
|
// QueueHeaders adds the passed headers message and peer to the block handling
|
|
|
|
// queue.
|
2013-11-16 00:16:51 +01:00
|
|
|
func (b *blockManager) QueueHeaders(headers *btcwire.MsgHeaders, p *peer) {
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
// No channel handling here because peers do not need to block on
|
|
|
|
// headers messages.
|
2013-11-16 00:16:51 +01:00
|
|
|
if atomic.LoadInt32(&b.shutdown) != 0 {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
b.msgChan <- &headersMsg{headers: headers, peer: p}
|
|
|
|
}
|
|
|
|
|
2013-10-01 00:53:21 +02:00
|
|
|
// DonePeer informs the blockmanager that a peer has disconnected.
|
|
|
|
func (b *blockManager) DonePeer(p *peer) {
|
|
|
|
// Ignore if we are shutting down.
|
2013-10-03 01:33:42 +02:00
|
|
|
if atomic.LoadInt32(&b.shutdown) != 0 {
|
2013-10-01 00:53:21 +02:00
|
|
|
return
|
|
|
|
}
|
2013-10-08 00:27:59 +02:00
|
|
|
|
2013-10-01 00:53:21 +02:00
|
|
|
b.msgChan <- &donePeerMsg{peer: p}
|
2013-09-27 02:41:02 +02:00
|
|
|
}
|
|
|
|
|
2013-08-06 23:55:22 +02:00
|
|
|
// Start begins the core block handler which processes block and inv messages.
|
|
|
|
func (b *blockManager) Start() {
|
|
|
|
// Already started?
|
2013-10-03 01:33:42 +02:00
|
|
|
if atomic.AddInt32(&b.started, 1) != 1 {
|
2013-08-06 23:55:22 +02:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2013-11-21 19:03:56 +01:00
|
|
|
bmgrLog.Trace("Starting block manager")
|
2013-10-03 04:21:52 +02:00
|
|
|
b.wg.Add(1)
|
2013-08-06 23:55:22 +02:00
|
|
|
go b.blockHandler()
|
|
|
|
}
|
|
|
|
|
|
|
|
// Stop gracefully shuts down the block manager by stopping all asynchronous
|
|
|
|
// handlers and waiting for them to finish.
|
|
|
|
func (b *blockManager) Stop() error {
|
2013-10-03 01:33:42 +02:00
|
|
|
if atomic.AddInt32(&b.shutdown, 1) != 1 {
|
2013-11-21 19:03:56 +01:00
|
|
|
bmgrLog.Warnf("Block manager is already in the process of " +
|
2013-08-06 23:55:22 +02:00
|
|
|
"shutting down")
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2013-11-21 19:03:56 +01:00
|
|
|
bmgrLog.Infof("Block manager shutting down")
|
2013-08-06 23:55:22 +02:00
|
|
|
close(b.quit)
|
|
|
|
b.wg.Wait()
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2014-02-04 07:24:40 +01:00
|
|
|
// SyncPeer returns the current sync peer.
|
|
|
|
func (b *blockManager) SyncPeer() *peer {
|
2014-02-04 16:39:00 +01:00
|
|
|
reply := make(chan *peer)
|
|
|
|
b.query <- getSyncPeerMsg{reply: reply}
|
|
|
|
return <-reply
|
2014-02-04 07:24:40 +01:00
|
|
|
}
|
|
|
|
|
2013-08-06 23:55:22 +02:00
|
|
|
// newBlockManager returns a new bitcoin block manager.
|
|
|
|
// Use Start to begin processing asynchronous block and inv updates.
|
2013-10-01 00:43:14 +02:00
|
|
|
func newBlockManager(s *server) (*blockManager, error) {
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
newestHash, height, err := s.db.NewestSha()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2013-08-06 23:55:22 +02:00
|
|
|
bm := blockManager{
|
|
|
|
server: s,
|
2013-08-29 21:44:43 +02:00
|
|
|
blockPeer: make(map[btcwire.ShaHash]*peer),
|
2013-10-08 17:47:00 +02:00
|
|
|
requestedTxns: make(map[btcwire.ShaHash]bool),
|
2013-10-01 00:53:21 +02:00
|
|
|
requestedBlocks: make(map[btcwire.ShaHash]bool),
|
2013-08-06 23:55:22 +02:00
|
|
|
lastBlockLogTime: time.Now(),
|
2013-10-01 00:53:21 +02:00
|
|
|
msgChan: make(chan interface{}, cfg.MaxPeers*3),
|
2014-02-04 16:39:00 +01:00
|
|
|
query: make(chan interface{}),
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
headerList: list.New(),
|
2013-08-06 23:55:22 +02:00
|
|
|
quit: make(chan bool),
|
|
|
|
}
|
2013-10-03 04:21:52 +02:00
|
|
|
bm.blockChain = btcchain.New(s.db, s.btcnet, bm.handleNotifyMsg)
|
2013-10-10 02:34:02 +02:00
|
|
|
bm.blockChain.DisableCheckpoints(cfg.DisableCheckpoints)
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
if !cfg.DisableCheckpoints {
|
|
|
|
// Initialize the next checkpoint based on the current height.
|
|
|
|
bm.nextCheckpoint = bm.findNextHeaderCheckpoint(height)
|
|
|
|
if bm.nextCheckpoint != nil {
|
|
|
|
bm.resetHeaderState(newestHash, height)
|
|
|
|
}
|
|
|
|
} else {
|
2013-11-21 19:03:56 +01:00
|
|
|
bmgrLog.Info("Checkpoints are disabled")
|
2013-10-10 02:34:02 +02:00
|
|
|
}
|
2013-10-01 00:43:14 +02:00
|
|
|
|
2013-11-21 19:03:56 +01:00
|
|
|
bmgrLog.Infof("Generating initial block node index. This may " +
|
2013-10-01 00:43:14 +02:00
|
|
|
"take a while...")
|
Rework and improve headers-first mode.
This commit improves how the headers-first mode works in several ways.
The previous headers-first code was an initial implementation that did not
have all of the bells and whistles and a few less than ideal
characteristics. This commit improves the heaers-first code to resolve
the issues discussed next.
- The previous code only used headers-first mode when starting out from
block height 0 rather than allowing it to work starting at any height
before the final checkpoint. This means if you stopped the chain
download at any point before the final checkpoint and restarted, it
would not resume and you therefore would not have the benefit of the
faster processing offered by headers-first mode.
- Previously all headers (even those after the final checkpoint) were
downloaded and only the final checkpoint was verified. This resulted in
the following issues:
- As the block chain grew, increasingly larger numbers of headers were
downloaded and kept in memory
- If the node the node serving up the headers was serving an invalid
chain, it wouldn't be detected until downloading a large number of
headers
- When an invalid checkpoint was detected, no action was taken to
recover which meant the chain download would essentially be stalled
- The headers were kept in memory even though they didn't need to be as
merely keeping track of the hashes and heights is enough to provde they
properly link together and checkpoints match
- There was no logging when headers were being downloaded so it could
appear like nothing was happening
- Duplicate requests for the same headers weren't being filtered which
meant is was possible to inadvertently download the same headers twice
only to throw them away.
This commit resolves these issues with the following changes:
- The current height is now examined at startup and prior each sync peer
selection to allow it to resume headers-first mode starting from the
known height to the next checkpoint
- All checkpoints are now verified and the headers are only downloaded
from the current known block height up to the next checkpoint. This has
several desirable properties:
- The amount of memory required is bounded by the maximum distance
between to checkpoints rather than the entire length of the chain
- A node serving up an invalid chain is detected very quickly and with
little work
- When an invalid checkpoint is detected, the headers are simply
discarded and the peer is disconnected for serving an invalid chain
- When the sync peer disconnets, all current headers are thrown away
and, due to the new aforementioned resume code, when a new sync peer
is selected, headers-first mode will continue from the last known good
block
- In addition to reduced memory usage from only keeping information about
headers between two checkpoints, the only information now kept in memory
about the headers is the hash and height rather than the entire header
- There is now logging information about what is happening with headers
- Duplicate header requests are now filtered
2014-01-30 18:29:02 +01:00
|
|
|
err = bm.blockChain.GenerateInitialIndex()
|
2013-10-01 00:43:14 +02:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2013-11-21 19:03:56 +01:00
|
|
|
bmgrLog.Infof("Block index generation complete")
|
2013-10-01 00:43:14 +02:00
|
|
|
|
|
|
|
return &bm, nil
|
2013-08-06 23:55:22 +02:00
|
|
|
}
|
|
|
|
|
2013-09-15 23:58:02 +02:00
|
|
|
// removeRegressionDB removes the existing regression test database if running
|
|
|
|
// in regression test mode and it already exists.
|
|
|
|
func removeRegressionDB(dbPath string) error {
|
|
|
|
// Dont do anything if not in regression test mode.
|
|
|
|
if !cfg.RegressionTest {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Remove the old regression test database if it already exists.
|
|
|
|
fi, err := os.Stat(dbPath)
|
|
|
|
if err == nil {
|
2013-11-21 19:03:56 +01:00
|
|
|
btcdLog.Infof("Removing regression test database from '%s'", dbPath)
|
2013-09-15 23:58:02 +02:00
|
|
|
if fi.IsDir() {
|
|
|
|
err := os.RemoveAll(dbPath)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
err := os.Remove(dbPath)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2013-10-08 22:00:34 +02:00
|
|
|
// dbPath returns the path to the block database given a database type.
|
|
|
|
func blockDbPath(dbType string) string {
|
2013-09-15 19:08:42 +02:00
|
|
|
// The database name is based on the database type.
|
2013-10-08 22:00:34 +02:00
|
|
|
dbName := blockDbNamePrefix + "_" + dbType
|
|
|
|
if dbType == "sqlite" {
|
2013-09-15 19:08:42 +02:00
|
|
|
dbName = dbName + ".db"
|
|
|
|
}
|
|
|
|
dbPath := filepath.Join(cfg.DataDir, dbName)
|
2013-10-08 22:00:34 +02:00
|
|
|
return dbPath
|
|
|
|
}
|
|
|
|
|
|
|
|
// warnMultipeDBs shows a warning if multiple block database types are detected.
|
|
|
|
// This is not a situation most users want. It is handy for development however
|
|
|
|
// to support multiple side-by-side databases.
|
|
|
|
func warnMultipeDBs() {
|
|
|
|
// This is intentionally not using the known db types which depend
|
|
|
|
// on the database types compiled into the binary since we want to
|
|
|
|
// detect legacy db types as well.
|
|
|
|
dbTypes := []string{"leveldb", "sqlite"}
|
|
|
|
duplicateDbPaths := make([]string, 0, len(dbTypes)-1)
|
|
|
|
for _, dbType := range dbTypes {
|
|
|
|
if dbType == cfg.DbType {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
// Store db path as a duplicate db if it exists.
|
|
|
|
dbPath := blockDbPath(dbType)
|
|
|
|
if fileExists(dbPath) {
|
|
|
|
duplicateDbPaths = append(duplicateDbPaths, dbPath)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Warn if there are extra databases.
|
|
|
|
if len(duplicateDbPaths) > 0 {
|
|
|
|
selectedDbPath := blockDbPath(cfg.DbType)
|
2013-11-21 19:03:56 +01:00
|
|
|
btcdLog.Warnf("WARNING: There are multiple block chain databases "+
|
2013-10-08 22:00:34 +02:00
|
|
|
"using different database types.\nYou probably don't "+
|
|
|
|
"want to waste disk space by having more than one.\n"+
|
|
|
|
"Your current database is located at [%v].\nThe "+
|
|
|
|
"additional database is located at %v", selectedDbPath,
|
|
|
|
duplicateDbPaths)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-10-27 05:10:27 +01:00
|
|
|
// setupBlockDB loads (or creates when needed) the block database taking into
|
|
|
|
// account the selected database backend. It also contains additional logic
|
|
|
|
// such warning the user if there are multiple databases which consume space on
|
|
|
|
// the file system and ensuring the regression test database is clean when in
|
|
|
|
// regression test mode.
|
|
|
|
func setupBlockDB() (btcdb.Db, error) {
|
|
|
|
// The memdb backend does not have a file path associated with it, so
|
|
|
|
// handle it uniquely. We also don't want to worry about the multiple
|
|
|
|
// database type warnings when running with the memory database.
|
|
|
|
if cfg.DbType == "memdb" {
|
|
|
|
btcdLog.Infof("Creating block database in memory.")
|
|
|
|
db, err := btcdb.CreateDB(cfg.DbType)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return db, nil
|
|
|
|
}
|
|
|
|
|
2013-10-08 22:00:34 +02:00
|
|
|
warnMultipeDBs()
|
|
|
|
|
|
|
|
// The database name is based on the database type.
|
|
|
|
dbPath := blockDbPath(cfg.DbType)
|
2013-09-15 19:08:42 +02:00
|
|
|
|
2013-09-15 23:58:02 +02:00
|
|
|
// The regression test is special in that it needs a clean database for
|
|
|
|
// each run, so remove it now if it already exists.
|
|
|
|
removeRegressionDB(dbPath)
|
|
|
|
|
2013-11-21 19:03:56 +01:00
|
|
|
btcdLog.Infof("Loading block database from '%s'", dbPath)
|
2013-09-05 20:42:12 +02:00
|
|
|
db, err := btcdb.OpenDB(cfg.DbType, dbPath)
|
2013-08-06 23:55:22 +02:00
|
|
|
if err != nil {
|
2013-10-27 05:10:27 +01:00
|
|
|
// Return the error if it's not because the database
|
|
|
|
// doesn't exist.
|
2013-08-06 23:55:22 +02:00
|
|
|
if err != btcdb.DbDoesNotExist {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Create the db if it does not exist.
|
2013-09-14 01:02:10 +02:00
|
|
|
err = os.MkdirAll(cfg.DataDir, 0700)
|
2013-08-06 23:55:22 +02:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2013-09-05 20:42:12 +02:00
|
|
|
db, err = btcdb.CreateDB(cfg.DbType, dbPath)
|
2013-08-06 23:55:22 +02:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2013-08-08 07:47:48 +02:00
|
|
|
}
|
|
|
|
|
2013-10-27 05:10:27 +01:00
|
|
|
return db, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// loadBlockDB opens the block database and returns a handle to it.
|
|
|
|
func loadBlockDB() (btcdb.Db, error) {
|
|
|
|
db, err := setupBlockDB()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2013-08-08 07:47:48 +02:00
|
|
|
// Get the latest block height from the database.
|
|
|
|
_, height, err := db.NewestSha()
|
|
|
|
if err != nil {
|
|
|
|
db.Close()
|
|
|
|
return nil, err
|
|
|
|
}
|
2013-08-06 23:55:22 +02:00
|
|
|
|
2013-08-08 07:47:48 +02:00
|
|
|
// Insert the appropriate genesis block for the bitcoin network being
|
|
|
|
// connected to if needed.
|
|
|
|
if height == -1 {
|
2013-08-06 23:55:22 +02:00
|
|
|
genesis := btcutil.NewBlock(activeNetParams.genesisBlock)
|
|
|
|
_, err := db.InsertBlock(genesis)
|
|
|
|
if err != nil {
|
|
|
|
db.Close()
|
|
|
|
return nil, err
|
|
|
|
}
|
2013-11-21 19:03:56 +01:00
|
|
|
btcdLog.Infof("Inserted genesis block %v",
|
2013-08-06 23:55:22 +02:00
|
|
|
activeNetParams.genesisHash)
|
2013-08-08 07:47:48 +02:00
|
|
|
height = 0
|
2013-08-06 23:55:22 +02:00
|
|
|
}
|
|
|
|
|
2013-11-21 19:03:56 +01:00
|
|
|
btcdLog.Infof("Block database loaded with block height %d", height)
|
2013-08-06 23:55:22 +02:00
|
|
|
return db, nil
|
|
|
|
}
|