Handle out-of-order notifications from btcd.

Notifications ariving from btcd were being reordered (each handled by
its own goroutine, rather then being always sent in the order they
originated).  This was breaking the new transaction store by inserting
transaction records in an 'impossible' manner, that is, inserting txs
without block info after the store already held records of the same tx
with block info, without first performing a rollback.

This is handled by the transaction store insert methods by checking
for identical transactions (double spends with the same tx sha), but
where the block heights mismatch and the new record does not have a
block set.  The error is returned all the way up to the goroutine
running each rpc request/notification handler, and if hit, the btcd
connection is closed and all accounts are reopened from disk.  This is
not optimal, but it allows us to use the connect logic to correctly
catch us up to the best chain with the last good state of all accounts
while only rescanning a few blocks.

Fixes #72.

rpcserver.go

@@ -207,8 +207,31 @@ func WalletRequestProcessor() {
 		case n := <-handleNtfn:
 			if f, ok := notificationHandlers[n.Method()]; ok {
 				AcctMgr.Grab()
-				f(n)
+				err := f(n)
 				AcctMgr.Release()
+				switch err {
+				case nil:
+					// ignore
+
+				case tx.ErrInconsistantStore:
+					// Likely due to a mis-ordered btcd notification.
+					// To recover, close server connection and reopen
+					// all accounts from their last good state saved
+					// to disk.  This will trigger the handshake on
+					// next connect, and a rescan of one or two blocks
+					// to catch up rather than throwing away all tx
+					// history and rescanning everything.
+					s := CurrentServerConn()
+					if btcd, ok := s.(*BtcdRPCConn); ok {
+						AcctMgr.Grab()
+						btcd.Close()
+						AcctMgr.OpenAccounts()
+						AcctMgr.Release()
+					}
+
+				default: // other non-nil
+					log.Warn(err)
+				}
 			}
 		}
 	}
@@ -1341,7 +1364,11 @@ func SendBeforeReceiveHistorySync(add, done, remove chan btcwire.ShaHash,
 
 func handleSendRawTxReply(icmd btcjson.Cmd, txIDStr string, a *Account, txInfo *CreatedTx) (interface{}, *btcjson.Error) {
 	// Add to transaction store.
-	stx := a.TxStore.InsertSignedTx(txInfo.tx, nil)
+	stx, err := a.TxStore.InsertSignedTx(txInfo.tx, nil)
+	if err != nil {
+		log.Warnf("Error adding sent tx history: %v", err)
+		return nil, &btcjson.ErrInternal
+	}
 	AcctMgr.ds.ScheduleTxStoreWrite(a)
 
 	// Notify frontends of new SendTx.