diff --git a/server.go b/server.go
index 334ac5e9..840ed8b6 100644
--- a/server.go
+++ b/server.go
@@ -349,7 +349,7 @@ out:
 		}
 
 		// Only try connect to more peers if we actually need more
-		if outboundPeers >= maxOutbound ||
+		if outboundPeers >= maxOutbound || len(cfg.ConnectPeers) > 0 ||
 			atomic.LoadInt32(&s.shutdown) != 0 {
 			continue
 		}