diff --git a/addrmanager.go b/addrmanager.go
index ba96a482..cbdca7a2 100644
--- a/addrmanager.go
+++ b/addrmanager.go
@@ -720,17 +720,20 @@ func (a *AddrManager) AddressCache() []*btcwire.NetAddress {
 		allAddr[i] = v.na
 		i++
 	}
-	// Fisher-Yates shuffle the array
-	for i := range allAddr {
-		j := rand.Intn(i + 1)
-		allAddr[i], allAddr[j] = allAddr[j], allAddr[i]
-	}
 
 	numAddresses := len(allAddr) * getAddrPercent / 100
 	if numAddresses > getAddrMax {
 		numAddresses = getAddrMax
 	}
 
+	// Fisher-Yates shuffle the array. We only need to do the first
+	// `numAddresses' since we are throwing the rest.
+	for i := 0; i < numAddresses; i++ {
+		// pick a number between current index and the end
+		j := rand.Intn(len(allAddr)-i) + i
+		allAddr[i], allAddr[j] = allAddr[j], allAddr[i]
+	}
+
 	// slice off the limit we are willing to share.
 	return allAddr[:numAddresses]
 }