Correct reconnect handling for persistent peers.

This commit correctly replaces persistent peers that are being retried in
the list of persistent peers so it will continue to be retried as
intended.

Also, limit the maximum retry interval for persistent peers to 5 minutes.

Fixes #463.

peer.go

@@ -53,6 +53,17 @@ const (
 	// pingTimeoutMinutes is the number of minutes since we last sent a
 	// message requiring a reply before we will ping a host.
 	pingTimeoutMinutes = 2
+
+	// connectionRetryInterval is the base amount of time to wait in between
+	// retries when connecting to persistent peers.  It is adjusted by the
+	// number of retries such that there is a retry backoff.
+	connectionRetryInterval = time.Second * 10
+
+	// maxConnectionRetryInterval is the max amount of time retrying of a
+	// persistent peer is allowed to grow to.  This is necessary since the
+	// retry logic uses a backoff mechanism which increases the interval
+	// base done the number of retries that have been done.
+	maxConnectionRetryInterval = time.Minute * 5
 )
 
 var (
@@ -2015,6 +2026,9 @@ func newOutboundPeer(s *server, addr string, persistent bool, retryCount int64)
 		if p.retryCount > 0 {
 			scaledInterval := connectionRetryInterval.Nanoseconds() * p.retryCount / 2
 			scaledDuration := time.Duration(scaledInterval)
+			if scaledDuration > maxConnectionRetryInterval {
+				scaledDuration = maxConnectionRetryInterval
+			}
 			srvrLog.Debugf("Retrying connection to %s in %s", addr, scaledDuration)
 			time.Sleep(scaledDuration)
 		}

server.go

@@ -39,10 +39,6 @@ const (
 	// server.
 	supportedServices = wire.SFNodeNetwork
 
-	// connectionRetryInterval is the amount of time to wait in between
-	// retries when connecting to persistent peers.
-	connectionRetryInterval = time.Second * 10
-
 	// defaultMaxOutbound is the default number of max outbound peers.
 	defaultMaxOutbound = 8
 )
@@ -307,7 +303,9 @@ func (s *server) handleDonePeerMsg(state *peerState, p *peer) {
 			// Issue an asynchronous reconnect if the peer was a
 			// persistent outbound connection.
 			if !p.inbound && p.persistent && atomic.LoadInt32(&s.shutdown) == 0 {
+				delete(list, e)
 				e = newOutboundPeer(s, p.addr, true, p.retryCount+1)
+				list[e] = struct{}{}
 				return
 			}
 			if !p.inbound {