From fc9b05b8c67dcda9c7fe632f7b6545e01fbdf3f4 Mon Sep 17 00:00:00 2001 From: Alex Grintsvayg Date: Thu, 14 Jun 2018 11:48:02 -0400 Subject: [PATCH] move bitmap into separate package --- dht/{ => bits}/bitmap.go | 96 +++++++++++++++++------------------ dht/{ => bits}/bitmap_test.go | 65 ++++++++++++++---------- dht/bootstrap.go | 7 +-- dht/bootstrap_test.go | 4 +- dht/dht.go | 33 +++++++----- dht/dht_test.go | 18 ++++--- dht/message.go | 31 +++++------ dht/message_test.go | 13 ++--- dht/node.go | 9 ++-- dht/node_finder.go | 11 ++-- dht/node_test.go | 51 ++++++++++--------- dht/routing_table.go | 39 +++++++------- dht/routing_table_test.go | 43 ++++++++-------- dht/store.go | 20 +++++--- dht/testing.go | 16 ++---- dht/token_manager.go | 7 +-- 16 files changed, 243 insertions(+), 220 deletions(-) rename dht/{ => bits}/bitmap.go (76%) rename dht/{ => bits}/bitmap_test.go (89%) diff --git a/dht/bitmap.go b/dht/bits/bitmap.go similarity index 76% rename from dht/bitmap.go rename to dht/bits/bitmap.go index 8b9e7e2..164df90 100644 --- a/dht/bitmap.go +++ b/dht/bits/bitmap.go @@ -1,38 +1,38 @@ -package dht +package bits import ( - "bytes" "crypto/rand" "encoding/hex" + "strconv" "strings" - "strconv" - "github.com/lbryio/lbry.go/errors" + "github.com/lyoshenka/bencode" ) // TODO: http://roaringbitmap.org/ +const ( + NumBytes = 48 // bytes + NumBits = NumBytes * 8 +) + // Bitmap is a generalized representation of an identifier or data that can be sorted, compared fast. Used by the DHT // package as a way to handle the unique identifiers of a DHT node. -type Bitmap [nodeIDLength]byte +type Bitmap [NumBytes]byte -func (b Bitmap) rawString() string { +func (b Bitmap) String() string { return string(b[:]) } // BString returns the bitmap as a string of 0s and 1s func (b Bitmap) BString() string { - var buf bytes.Buffer - for i := 0; i < nodeIDBits; i++ { - if b.Get(i) { - buf.WriteString("1") - } else { - buf.WriteString("0") - } + var s string + for _, byte := range b { + s += strconv.FormatInt(int64(byte), 2) } - return buf.String() + return s } // Hex returns a hexadecimal representation of the bitmap. @@ -147,7 +147,7 @@ func (b Bitmap) Not() Bitmap { func (b Bitmap) add(other Bitmap) (Bitmap, bool) { var ret Bitmap carry := false - for i := nodeIDBits - 1; i >= 0; i-- { + for i := NumBits - 1; i >= 0; i-- { bBit := getBit(b[:], i) oBit := getBit(other[:], i) setBit(ret[:], i, bBit != oBit != carry) @@ -161,7 +161,7 @@ func (b Bitmap) add(other Bitmap) (Bitmap, bool) { func (b Bitmap) Add(other Bitmap) Bitmap { ret, carry := b.add(other) if carry { - panic("overflow in bitmap addition. limited to " + strconv.Itoa(nodeIDBits) + " bits.") + panic("overflow in bitmap addition. limited to " + strconv.Itoa(NumBits) + " bits.") } return ret } @@ -173,7 +173,7 @@ func (b Bitmap) Sub(other Bitmap) Bitmap { // ToDo: Why is this not supported? Should it say not implemented? BitMap might have a generic use case outside of dht. panic("negative bitmaps not supported") } - complement, _ := other.Not().add(BitmapFromShortHexP("1")) + complement, _ := other.Not().add(FromShortHexP("1")) ret, _ := b.add(complement) return ret } @@ -199,7 +199,7 @@ func (b Bitmap) PrefixLen() int { } } } - return nodeIDBits + return NumBits } // Prefix returns a copy of b with the first n bits set to 1 (if `one` is true) or 0 (if `one` is false) @@ -233,7 +233,7 @@ func (b Bitmap) Suffix(n int, one bool) Bitmap { Outer: for i := len(ret) - 1; i >= 0; i-- { for j := 7; j >= 0; j-- { - if i*8+j >= nodeIDBits-n { + if i*8+j >= NumBits-n { if one { ret[i] |= 1 << uint(7-j) } else { @@ -261,15 +261,15 @@ func (b *Bitmap) UnmarshalBencode(encoded []byte) error { if err != nil { return err } - if len(str) != nodeIDLength { + if len(str) != NumBytes { return errors.Err("invalid bitmap length") } copy(b[:], str) return nil } -// BitmapFromBytes returns a bitmap as long as the byte array is of a specific length specified in the parameters. -func BitmapFromBytes(data []byte) (Bitmap, error) { +// FromBytes returns a bitmap as long as the byte array is of a specific length specified in the parameters. +func FromBytes(data []byte) (Bitmap, error) { var bmp Bitmap if len(data) != len(bmp) { @@ -280,71 +280,71 @@ func BitmapFromBytes(data []byte) (Bitmap, error) { return bmp, nil } -// BitmapFromBytesP returns a bitmap as long as the byte array is of a specific length specified in the parameters +// FromBytesP returns a bitmap as long as the byte array is of a specific length specified in the parameters // otherwise it wil panic. -func BitmapFromBytesP(data []byte) Bitmap { - bmp, err := BitmapFromBytes(data) +func FromBytesP(data []byte) Bitmap { + bmp, err := FromBytes(data) if err != nil { panic(err) } return bmp } -//BitmapFromString returns a bitmap by converting the string to bytes and creating from bytes as long as the byte array +//FromString returns a bitmap by converting the string to bytes and creating from bytes as long as the byte array // is of a specific length specified in the parameters -func BitmapFromString(data string) (Bitmap, error) { - return BitmapFromBytes([]byte(data)) +func FromString(data string) (Bitmap, error) { + return FromBytes([]byte(data)) } -//BitmapFromStringP returns a bitmap by converting the string to bytes and creating from bytes as long as the byte array +//FromStringP returns a bitmap by converting the string to bytes and creating from bytes as long as the byte array // is of a specific length specified in the parameters otherwise it wil panic. -func BitmapFromStringP(data string) Bitmap { - bmp, err := BitmapFromString(data) +func FromStringP(data string) Bitmap { + bmp, err := FromString(data) if err != nil { panic(err) } return bmp } -//BitmapFromHex returns a bitmap by converting the hex string to bytes and creating from bytes as long as the byte array +//FromHex returns a bitmap by converting the hex string to bytes and creating from bytes as long as the byte array // is of a specific length specified in the parameters -func BitmapFromHex(hexStr string) (Bitmap, error) { +func FromHex(hexStr string) (Bitmap, error) { decoded, err := hex.DecodeString(hexStr) if err != nil { return Bitmap{}, errors.Err(err) } - return BitmapFromBytes(decoded) + return FromBytes(decoded) } -//BitmapFromHexP returns a bitmap by converting the hex string to bytes and creating from bytes as long as the byte array +//FromHexP returns a bitmap by converting the hex string to bytes and creating from bytes as long as the byte array // is of a specific length specified in the parameters otherwise it wil panic. -func BitmapFromHexP(hexStr string) Bitmap { - bmp, err := BitmapFromHex(hexStr) +func FromHexP(hexStr string) Bitmap { + bmp, err := FromHex(hexStr) if err != nil { panic(err) } return bmp } -//BitmapFromShortHex returns a bitmap by converting the hex string to bytes, adding the leading zeros prefix to the +//FromShortHex returns a bitmap by converting the hex string to bytes, adding the leading zeros prefix to the // hex string and creating from bytes as long as the byte array is of a specific length specified in the parameters -func BitmapFromShortHex(hexStr string) (Bitmap, error) { - return BitmapFromHex(strings.Repeat("0", nodeIDLength*2-len(hexStr)) + hexStr) +func FromShortHex(hexStr string) (Bitmap, error) { + return FromHex(strings.Repeat("0", NumBytes*2-len(hexStr)) + hexStr) } -//BitmapFromShortHexP returns a bitmap by converting the hex string to bytes, adding the leading zeros prefix to the +//FromShortHexP returns a bitmap by converting the hex string to bytes, adding the leading zeros prefix to the // hex string and creating from bytes as long as the byte array is of a specific length specified in the parameters // otherwise it wil panic. -func BitmapFromShortHexP(hexStr string) Bitmap { - bmp, err := BitmapFromShortHex(hexStr) +func FromShortHexP(hexStr string) Bitmap { + bmp, err := FromShortHex(hexStr) if err != nil { panic(err) } return bmp } -// RandomBitmapP generates a cryptographically random bitmap with the confines of the parameters specified. -func RandomBitmapP() Bitmap { +// Rand generates a cryptographically random bitmap with the confines of the parameters specified. +func Rand() Bitmap { var id Bitmap _, err := rand.Read(id[:]) if err != nil { @@ -353,11 +353,11 @@ func RandomBitmapP() Bitmap { return id } -// RandomBitmapInRangeP generates a cryptographically random bitmap and while it is greater than the high threshold +// RandInRangeP generates a cryptographically random bitmap and while it is greater than the high threshold // bitmap will subtract the diff between high and low until it is no longer greater that the high. -func RandomBitmapInRangeP(low, high Bitmap) Bitmap { +func RandInRangeP(low, high Bitmap) Bitmap { diff := high.Sub(low) - r := RandomBitmapP() + r := Rand() for r.Greater(diff) { r = r.Sub(diff) } diff --git a/dht/bitmap_test.go b/dht/bits/bitmap_test.go similarity index 89% rename from dht/bitmap_test.go rename to dht/bits/bitmap_test.go index 3b541a2..482d0d5 100644 --- a/dht/bitmap_test.go +++ b/dht/bits/bitmap_test.go @@ -1,4 +1,4 @@ -package dht +package bits import ( "fmt" @@ -47,8 +47,8 @@ func TestBitmap(t *testing.T) { } id := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" - if BitmapFromHexP(id).Hex() != id { - t.Error(BitmapFromHexP(id).Hex()) + if FromHexP(id).Hex() != id { + t.Error(FromHexP(id).Hex()) } } @@ -64,7 +64,7 @@ func TestBitmap_GetBit(t *testing.T) { {bit: 380, expected: true, panic: false}, } - b := BitmapFromShortHexP("a") + b := FromShortHexP("a") for _, test := range tt { actual := getBit(b[:], test.bit) @@ -90,8 +90,8 @@ func TestBitmap_SetBit(t *testing.T) { } for _, test := range tt { - expected := BitmapFromShortHexP(test.expected) - actual := BitmapFromShortHexP(test.hex) + expected := FromShortHexP(test.expected) + actual := FromShortHexP(test.hex) if test.panic { assertPanic(t, fmt.Sprintf("setting bit %d to %t", test.bit, test.one), func() { setBit(actual[:], test.bit, test.one) }) } else { @@ -119,8 +119,8 @@ func TestBitmap_FromHexShort(t *testing.T) { } for _, test := range tt { - short := BitmapFromShortHexP(test.short) - long := BitmapFromHexP(test.long) + short := FromShortHexP(test.short) + long := FromHexP(test.long) if !short.Equals(long) { t.Errorf("short hex %s: expected %s, got %s", test.short, long.Hex(), short.Hex()) } @@ -128,7 +128,7 @@ func TestBitmap_FromHexShort(t *testing.T) { } func TestBitmapMarshal(t *testing.T) { - b := BitmapFromStringP("123456789012345678901234567890123456789012345678") + b := FromStringP("123456789012345678901234567890123456789012345678") encoded, err := bencode.EncodeBytes(b) if err != nil { t.Error(err) @@ -146,7 +146,7 @@ func TestBitmapMarshalEmbedded(t *testing.T) { C int }{ A: "1", - B: BitmapFromStringP("222222222222222222222222222222222222222222222222"), + B: FromStringP("222222222222222222222222222222222222222222222222"), C: 3, } @@ -162,7 +162,7 @@ func TestBitmapMarshalEmbedded(t *testing.T) { func TestBitmapMarshalEmbedded2(t *testing.T) { encoded, err := bencode.EncodeBytes([]interface{}{ - BitmapFromStringP("333333333333333333333333333333333333333333333333"), + FromStringP("333333333333333333333333333333333333333333333333"), }) if err != nil { t.Error(err) @@ -189,7 +189,7 @@ func TestBitmap_PrefixLen(t *testing.T) { } for _, test := range tt { - len := BitmapFromHexP(test.hex).PrefixLen() + len := FromHexP(test.hex).PrefixLen() if len != test.len { t.Errorf("got prefix len %d; expected %d for %s", len, test.len, test.hex) } @@ -197,7 +197,7 @@ func TestBitmap_PrefixLen(t *testing.T) { } func TestBitmap_Prefix(t *testing.T) { - allOne := BitmapFromHexP("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + allOne := FromHexP("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") zerosTT := []struct { zeros int @@ -213,21 +213,21 @@ func TestBitmap_Prefix(t *testing.T) { } for _, test := range zerosTT { - expected := BitmapFromHexP(test.expected) + expected := FromHexP(test.expected) actual := allOne.Prefix(test.zeros, false) if !actual.Equals(expected) { t.Errorf("%d zeros: got %s; expected %s", test.zeros, actual.Hex(), expected.Hex()) } } - for i := 0; i < nodeIDLength*8; i++ { + for i := 0; i < NumBits; i++ { b := allOne.Prefix(i, false) if b.PrefixLen() != i { t.Errorf("got prefix len %d; expected %d for %s", b.PrefixLen(), i, b.Hex()) } } - allZero := BitmapFromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000") + allZero := FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000") onesTT := []struct { ones int @@ -243,7 +243,7 @@ func TestBitmap_Prefix(t *testing.T) { } for _, test := range onesTT { - expected := BitmapFromHexP(test.expected) + expected := FromHexP(test.expected) actual := allZero.Prefix(test.ones, true) if !actual.Equals(expected) { t.Errorf("%d ones: got %s; expected %s", test.ones, actual.Hex(), expected.Hex()) @@ -252,7 +252,7 @@ func TestBitmap_Prefix(t *testing.T) { } func TestBitmap_Suffix(t *testing.T) { - allOne := BitmapFromHexP("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + allOne := FromHexP("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") zerosTT := []struct { zeros int @@ -268,21 +268,21 @@ func TestBitmap_Suffix(t *testing.T) { } for _, test := range zerosTT { - expected := BitmapFromHexP(test.expected) + expected := FromHexP(test.expected) actual := allOne.Suffix(test.zeros, false) if !actual.Equals(expected) { t.Errorf("%d zeros: got %s; expected %s", test.zeros, actual.Hex(), expected.Hex()) } } - for i := 0; i < nodeIDLength*8; i++ { + for i := 0; i < NumBits; i++ { b := allOne.Prefix(i, false) if b.PrefixLen() != i { t.Errorf("got prefix len %d; expected %d for %s", b.PrefixLen(), i, b.Hex()) } } - allZero := BitmapFromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000") + allZero := FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000") onesTT := []struct { ones int @@ -298,7 +298,7 @@ func TestBitmap_Suffix(t *testing.T) { } for _, test := range onesTT { - expected := BitmapFromHexP(test.expected) + expected := FromHexP(test.expected) actual := allZero.Suffix(test.ones, true) if !actual.Equals(expected) { t.Errorf("%d ones: got %s; expected %s", test.ones, actual.Hex(), expected.Hex()) @@ -324,9 +324,9 @@ func TestBitmap_Add(t *testing.T) { } for _, test := range tt { - a := BitmapFromShortHexP(test.a) - b := BitmapFromShortHexP(test.b) - expected := BitmapFromShortHexP(test.sum) + a := FromShortHexP(test.a) + b := FromShortHexP(test.b) + expected := FromShortHexP(test.sum) if test.panic { assertPanic(t, fmt.Sprintf("adding %s and %s", test.a, test.b), func() { a.Add(b) }) } else { @@ -358,9 +358,9 @@ func TestBitmap_Sub(t *testing.T) { } for _, test := range tt { - a := BitmapFromShortHexP(test.a) - b := BitmapFromShortHexP(test.b) - expected := BitmapFromShortHexP(test.sum) + a := FromShortHexP(test.a) + b := FromShortHexP(test.b) + expected := FromShortHexP(test.sum) if test.panic { assertPanic(t, fmt.Sprintf("subtracting %s - %s", test.a, test.b), func() { a.Sub(b) }) } else { @@ -371,3 +371,12 @@ func TestBitmap_Sub(t *testing.T) { } } } + +func assertPanic(t *testing.T, text string, f func()) { + defer func() { + if r := recover(); r == nil { + t.Errorf("%s: did not panic as expected", text) + } + }() + f() +} diff --git a/dht/bootstrap.go b/dht/bootstrap.go index 30077c2..797ac22 100644 --- a/dht/bootstrap.go +++ b/dht/bootstrap.go @@ -6,6 +6,7 @@ import ( "sync" "time" + "github.com/lbryio/reflector.go/dht/bits" log "github.com/sirupsen/logrus" ) @@ -22,11 +23,11 @@ type BootstrapNode struct { nlock *sync.RWMutex nodes []peer - nodeKeys map[Bitmap]int + nodeKeys map[bits.Bitmap]int } // NewBootstrapNode returns a BootstrapNode pointer. -func NewBootstrapNode(id Bitmap, initialPingInterval, rePingInterval time.Duration) *BootstrapNode { +func NewBootstrapNode(id bits.Bitmap, initialPingInterval, rePingInterval time.Duration) *BootstrapNode { b := &BootstrapNode{ Node: *NewNode(id), @@ -35,7 +36,7 @@ func NewBootstrapNode(id Bitmap, initialPingInterval, rePingInterval time.Durati nlock: &sync.RWMutex{}, nodes: make([]peer, 0), - nodeKeys: make(map[Bitmap]int), + nodeKeys: make(map[bits.Bitmap]int), } b.requestHandler = b.handleRequest diff --git a/dht/bootstrap_test.go b/dht/bootstrap_test.go index e57fc83..1b89466 100644 --- a/dht/bootstrap_test.go +++ b/dht/bootstrap_test.go @@ -3,10 +3,12 @@ package dht import ( "net" "testing" + + "github.com/lbryio/reflector.go/dht/bits" ) func TestBootstrapPing(t *testing.T) { - b := NewBootstrapNode(RandomBitmapP(), 10, bootstrapDefaultRefreshDuration) + b := NewBootstrapNode(bits.Rand(), 10, bootstrapDefaultRefreshDuration) listener, err := net.ListenPacket(network, "127.0.0.1:54320") if err != nil { diff --git a/dht/dht.go b/dht/dht.go index bc069de..2cbaae9 100644 --- a/dht/dht.go +++ b/dht/dht.go @@ -9,6 +9,7 @@ import ( "github.com/lbryio/lbry.go/errors" "github.com/lbryio/lbry.go/stopOnce" + "github.com/lbryio/reflector.go/dht/bits" "github.com/spf13/cast" log "github.com/sirupsen/logrus" @@ -24,11 +25,11 @@ const ( // TODO: all these constants should be defaults, and should be used to set values in the standard Config. then the code should use values in the config // TODO: alternatively, have a global Config for constants. at least that way tests can modify the values - alpha = 3 // this is the constant alpha in the spec - bucketSize = 8 // this is the constant k in the spec - nodeIDLength = 48 // bytes. this is the constant B in the spec - nodeIDBits = nodeIDLength * 8 // number of bits in node ID - messageIDLength = 20 // bytes. + alpha = 3 // this is the constant alpha in the spec + bucketSize = 8 // this is the constant k in the spec + nodeIDLength = bits.NumBytes // bytes. this is the constant B in the spec + nodeIDBits = bits.NumBits // number of bits in node ID + messageIDLength = 20 // bytes. udpRetry = 3 udpTimeout = 5 * time.Second @@ -85,7 +86,7 @@ type DHT struct { // lock for announced list lock *sync.RWMutex // list of bitmaps that need to be reannounced periodically - announced map[Bitmap]bool + announced map[bits.Bitmap]bool } // New returns a DHT pointer. If config is nil, then config will be set to the default config. @@ -106,7 +107,7 @@ func New(config *Config) (*DHT, error) { stop: stopOnce.New(), joined: make(chan struct{}), lock: &sync.RWMutex{}, - announced: make(map[Bitmap]bool), + announced: make(map[bits.Bitmap]bool), } return d, nil } @@ -188,7 +189,7 @@ func (dht *DHT) Ping(addr string) error { return err } - tmpNode := Contact{ID: RandomBitmapP(), IP: raddr.IP, Port: raddr.Port} + tmpNode := Contact{ID: bits.Rand(), IP: raddr.IP, Port: raddr.Port} res := dht.node.Send(tmpNode, Request{Method: pingMethod}) if res == nil { return errors.Err("no response from node %s", addr) @@ -198,7 +199,7 @@ func (dht *DHT) Ping(addr string) error { } // Get returns the list of nodes that have the blob for the given hash -func (dht *DHT) Get(hash Bitmap) ([]Contact, error) { +func (dht *DHT) Get(hash bits.Bitmap) ([]Contact, error) { contacts, found, err := FindContacts(dht.node, hash, true, dht.stop.Ch()) if err != nil { return nil, err @@ -211,7 +212,7 @@ func (dht *DHT) Get(hash Bitmap) ([]Contact, error) { } // Announce announces to the DHT that this node has the blob for the given hash -func (dht *DHT) Announce(hash Bitmap) error { +func (dht *DHT) Announce(hash bits.Bitmap) error { contacts, _, err := FindContacts(dht.node, hash, false, dht.stop.Ch()) if err != nil { return err @@ -251,7 +252,7 @@ func (dht *DHT) startReannouncer() { dht.lock.RLock() for h := range dht.announced { dht.stop.Add(1) - go func(bm Bitmap) { + go func(bm bits.Bitmap) { defer dht.stop.Done() err := dht.Announce(bm) if err != nil { @@ -264,7 +265,7 @@ func (dht *DHT) startReannouncer() { } } -func (dht *DHT) storeOnNode(hash Bitmap, c Contact) { +func (dht *DHT) storeOnNode(hash bits.Bitmap, c Contact) { // self-store if dht.contact.Equals(c) { dht.node.Store(hash, c) @@ -322,12 +323,16 @@ func (dht *DHT) PrintState() { } } +func (dht DHT) ID() bits.Bitmap { + return dht.contact.ID +} + func getContact(nodeID, addr string) (Contact, error) { var c Contact if nodeID == "" { - c.ID = RandomBitmapP() + c.ID = bits.Rand() } else { - c.ID = BitmapFromHexP(nodeID) + c.ID = bits.FromHexP(nodeID) } ip, port, err := net.SplitHostPort(addr) diff --git a/dht/dht_test.go b/dht/dht_test.go index 15ab99b..1de7e31 100644 --- a/dht/dht_test.go +++ b/dht/dht_test.go @@ -5,6 +5,8 @@ import ( "sync" "testing" "time" + + "github.com/lbryio/reflector.go/dht/bits" ) func TestNodeFinder_FindNodes(t *testing.T) { @@ -16,7 +18,7 @@ func TestNodeFinder_FindNodes(t *testing.T) { bs.Shutdown() }() - contacts, found, err := FindContacts(dhts[2].node, RandomBitmapP(), false, nil) + contacts, found, err := FindContacts(dhts[2].node, bits.Rand(), false, nil) if err != nil { t.Fatal(err) } @@ -64,7 +66,7 @@ func TestNodeFinder_FindNodes_NoBootstrap(t *testing.T) { } }() - _, _, err := FindContacts(dhts[2].node, RandomBitmapP(), false, nil) + _, _, err := FindContacts(dhts[2].node, bits.Rand(), false, nil) if err == nil { t.Fatal("contact finder should have errored saying that there are no contacts in the routing table") } @@ -79,8 +81,8 @@ func TestNodeFinder_FindValue(t *testing.T) { bs.Shutdown() }() - blobHashToFind := RandomBitmapP() - nodeToFind := Contact{ID: RandomBitmapP(), IP: net.IPv4(1, 2, 3, 4), Port: 5678} + blobHashToFind := bits.Rand() + nodeToFind := Contact{ID: bits.Rand(), IP: net.IPv4(1, 2, 3, 4), Port: 5678} dhts[0].node.store.Upsert(blobHashToFind, nodeToFind) contacts, found, err := FindContacts(dhts[2].node, blobHashToFind, true, nil) @@ -113,9 +115,9 @@ func TestDHT_LargeDHT(t *testing.T) { }() wg := &sync.WaitGroup{} - ids := make([]Bitmap, nodes) + ids := make([]bits.Bitmap, nodes) for i := range ids { - ids[i] = RandomBitmapP() + ids[i] = bits.Rand() wg.Add(1) go func(index int) { defer wg.Done() @@ -127,7 +129,7 @@ func TestDHT_LargeDHT(t *testing.T) { wg.Wait() // check that each node is in at learst 1 other routing table - rtCounts := make(map[Bitmap]int) + rtCounts := make(map[bits.Bitmap]int) for _, d := range dhts { for _, d2 := range dhts { if d.node.id.Equals(d2.node.id) { @@ -149,7 +151,7 @@ func TestDHT_LargeDHT(t *testing.T) { } // check that each ID is stored by at least 3 nodes - storeCounts := make(map[Bitmap]int) + storeCounts := make(map[bits.Bitmap]int) for _, d := range dhts { for _, id := range ids { if len(d.node.store.Get(id)) > 0 { diff --git a/dht/message.go b/dht/message.go index e67f42c..e78ddd6 100644 --- a/dht/message.go +++ b/dht/message.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/lbryio/lbry.go/errors" + "github.com/lbryio/reflector.go/dht/bits" "github.com/lyoshenka/bencode" "github.com/spf13/cast" @@ -83,9 +84,9 @@ func newMessageID() messageID { // Request represents the structured request from one node to another. type Request struct { ID messageID - NodeID Bitmap + NodeID bits.Bitmap Method string - Arg *Bitmap + Arg *bits.Bitmap StoreArgs *storeArgs } @@ -95,7 +96,7 @@ func (r Request) MarshalBencode() ([]byte, error) { if r.StoreArgs != nil { args = r.StoreArgs } else if r.Arg != nil { - args = []Bitmap{*r.Arg} + args = []bits.Bitmap{*r.Arg} } else { args = []string{} // request must always have keys 0-4, so we use an empty list for PING } @@ -112,7 +113,7 @@ func (r Request) MarshalBencode() ([]byte, error) { func (r *Request) UnmarshalBencode(b []byte) error { var raw struct { ID messageID `bencode:"1"` - NodeID Bitmap `bencode:"2"` + NodeID bits.Bitmap `bencode:"2"` Method string `bencode:"3"` Args bencode.RawMessage `bencode:"4"` } @@ -132,7 +133,7 @@ func (r *Request) UnmarshalBencode(b []byte) error { return errors.Prefix("request unmarshal", err) } } else if len(raw.Args) > 2 { // 2 because an empty list is `le` - tmp := []Bitmap{} + tmp := []bits.Bitmap{} err = bencode.DecodeBytes(raw.Args, &tmp) if err != nil { return errors.Prefix("request unmarshal", err) @@ -153,16 +154,16 @@ func (r Request) argsDebug() string { } type storeArgsValue struct { - Token string `bencode:"token"` - LbryID Bitmap `bencode:"lbryid"` - Port int `bencode:"port"` + Token string `bencode:"token"` + LbryID bits.Bitmap `bencode:"lbryid"` + Port int `bencode:"port"` } type storeArgs struct { - BlobHash Bitmap + BlobHash bits.Bitmap Value storeArgsValue - NodeID Bitmap // original publisher id? I think this is getting fixed in the new dht stuff - SelfStore bool // this is an int on the wire + NodeID bits.Bitmap // original publisher id? I think this is getting fixed in the new dht stuff + SelfStore bool // this is an int on the wire } // MarshalBencode returns the serialized byte slice representation of the storage arguments. @@ -231,7 +232,7 @@ func (s *storeArgs) UnmarshalBencode(b []byte) error { // Response represents the structured response one node returns to another. type Response struct { ID messageID - NodeID Bitmap + NodeID bits.Bitmap Data string Contacts []Contact FindValueKey string @@ -308,7 +309,7 @@ func (r Response) MarshalBencode() ([]byte, error) { func (r *Response) UnmarshalBencode(b []byte) error { var raw struct { ID messageID `bencode:"1"` - NodeID Bitmap `bencode:"2"` + NodeID bits.Bitmap `bencode:"2"` Data bencode.RawMessage `bencode:"3"` } err := bencode.DecodeBytes(b, &raw) @@ -377,7 +378,7 @@ func (r *Response) UnmarshalBencode(b []byte) error { // Error represents an error message that is returned from one node to another in communication. type Error struct { ID messageID - NodeID Bitmap + NodeID bits.Bitmap ExceptionType string Response []string } @@ -397,7 +398,7 @@ func (e Error) MarshalBencode() ([]byte, error) { func (e *Error) UnmarshalBencode(b []byte) error { var raw struct { ID messageID `bencode:"1"` - NodeID Bitmap `bencode:"2"` + NodeID bits.Bitmap `bencode:"2"` ExceptionType string `bencode:"3"` Args interface{} `bencode:"4"` } diff --git a/dht/message_test.go b/dht/message_test.go index 1cac9e3..71e99de 100644 --- a/dht/message_test.go +++ b/dht/message_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/davecgh/go-spew/spew" + "github.com/lbryio/reflector.go/dht/bits" "github.com/lyoshenka/bencode" ) @@ -76,10 +77,10 @@ func TestBencodeDecodeStoreArgs(t *testing.T) { func TestBencodeFindNodesResponse(t *testing.T) { res := Response{ ID: newMessageID(), - NodeID: RandomBitmapP(), + NodeID: bits.Rand(), Contacts: []Contact{ - {ID: RandomBitmapP(), IP: net.IPv4(1, 2, 3, 4).To4(), Port: 5678}, - {ID: RandomBitmapP(), IP: net.IPv4(4, 3, 2, 1).To4(), Port: 8765}, + {ID: bits.Rand(), IP: net.IPv4(1, 2, 3, 4).To4(), Port: 5678}, + {ID: bits.Rand(), IP: net.IPv4(4, 3, 2, 1).To4(), Port: 8765}, }, } @@ -100,11 +101,11 @@ func TestBencodeFindNodesResponse(t *testing.T) { func TestBencodeFindValueResponse(t *testing.T) { res := Response{ ID: newMessageID(), - NodeID: RandomBitmapP(), - FindValueKey: RandomBitmapP().rawString(), + NodeID: bits.Rand(), + FindValueKey: bits.Rand().String(), Token: "arst", Contacts: []Contact{ - {ID: RandomBitmapP(), IP: net.IPv4(1, 2, 3, 4).To4(), Port: 5678}, + {ID: bits.Rand(), IP: net.IPv4(1, 2, 3, 4).To4(), Port: 5678}, }, } diff --git a/dht/node.go b/dht/node.go index d2e5ebc..d35de51 100644 --- a/dht/node.go +++ b/dht/node.go @@ -11,6 +11,7 @@ import ( "github.com/lbryio/errors.go" "github.com/lbryio/lbry.go/stopOnce" "github.com/lbryio/lbry.go/util" + "github.com/lbryio/reflector.go/dht/bits" "github.com/davecgh/go-spew/spew" "github.com/lyoshenka/bencode" @@ -39,7 +40,7 @@ type RequestHandlerFunc func(addr *net.UDPAddr, request Request) // Node is a type representation of a node on the network. type Node struct { // the node's id - id Bitmap + id bits.Bitmap // UDP connection for sending and receiving data conn UDPConn // true if we've closed the connection on purpose @@ -64,7 +65,7 @@ type Node struct { } // NewNode returns an initialized Node's pointer. -func NewNode(id Bitmap) *Node { +func NewNode(id bits.Bitmap) *Node { return &Node{ id: id, rt: newRoutingTable(id), @@ -270,7 +271,7 @@ func (n *Node) handleRequest(addr *net.UDPAddr, request Request) { } if contacts := n.store.Get(*request.Arg); len(contacts) > 0 { - res.FindValueKey = request.Arg.rawString() + res.FindValueKey = request.Arg.String() res.Contacts = contacts } else { res.Contacts = n.rt.GetClosest(*request.Arg, bucketSize) @@ -446,6 +447,6 @@ func (n *Node) startRoutingTableGrooming() { } // Store stores a node contact in the node's contact store. -func (n *Node) Store(hash Bitmap, c Contact) { +func (n *Node) Store(hash bits.Bitmap, c Contact) { n.store.Upsert(hash, c) } diff --git a/dht/node_finder.go b/dht/node_finder.go index 878c4ee..72c1b1a 100644 --- a/dht/node_finder.go +++ b/dht/node_finder.go @@ -7,6 +7,7 @@ import ( "github.com/lbryio/lbry.go/errors" "github.com/lbryio/lbry.go/stopOnce" + "github.com/lbryio/reflector.go/dht/bits" log "github.com/sirupsen/logrus" ) @@ -16,7 +17,7 @@ import ( type contactFinder struct { findValue bool // true if we're using findValue - target Bitmap + target bits.Bitmap node *Node stop *stopOnce.Stopper @@ -29,13 +30,13 @@ type contactFinder struct { shortlistMutex *sync.Mutex shortlist []Contact - shortlistAdded map[Bitmap]bool + shortlistAdded map[bits.Bitmap]bool outstandingRequestsMutex *sync.RWMutex outstandingRequests uint } -func FindContacts(node *Node, target Bitmap, findValue bool, upstreamStop stopOnce.Chan) ([]Contact, bool, error) { +func FindContacts(node *Node, target bits.Bitmap, findValue bool, upstreamStop stopOnce.Chan) ([]Contact, bool, error) { cf := &contactFinder{ node: node, target: target, @@ -43,7 +44,7 @@ func FindContacts(node *Node, target Bitmap, findValue bool, upstreamStop stopOn findValueMutex: &sync.Mutex{}, activeContactsMutex: &sync.Mutex{}, shortlistMutex: &sync.Mutex{}, - shortlistAdded: make(map[Bitmap]bool), + shortlistAdded: make(map[bits.Bitmap]bool), stop: stopOnce.New(), outstandingRequestsMutex: &sync.RWMutex{}, } @@ -259,7 +260,7 @@ func (cf *contactFinder) areRequestsOutstanding() bool { return cf.outstandingRequests > 0 } -func sortInPlace(contacts []Contact, target Bitmap) { +func sortInPlace(contacts []Contact, target bits.Bitmap) { toSort := make([]sortedContact, len(contacts)) for i, n := range contacts { diff --git a/dht/node_test.go b/dht/node_test.go index f9fbdd5..b9537f7 100644 --- a/dht/node_test.go +++ b/dht/node_test.go @@ -5,12 +5,13 @@ import ( "testing" "time" + "github.com/lbryio/reflector.go/dht/bits" "github.com/lyoshenka/bencode" ) func TestPing(t *testing.T) { - dhtNodeID := RandomBitmapP() - testNodeID := RandomBitmapP() + dhtNodeID := bits.Rand() + testNodeID := bits.Rand() conn := newTestUDPConn("127.0.0.1:21217") @@ -30,7 +31,7 @@ func TestPing(t *testing.T) { data, err := bencode.EncodeBytes(map[string]interface{}{ headerTypeField: requestType, headerMessageIDField: messageID, - headerNodeIDField: testNodeID.rawString(), + headerNodeIDField: testNodeID.String(), headerPayloadField: "ping", headerArgsField: []string{}, }) @@ -86,7 +87,7 @@ func TestPing(t *testing.T) { rNodeID, ok := response[headerNodeIDField].(string) if !ok { t.Error("node ID is not a string") - } else if rNodeID != dhtNodeID.rawString() { + } else if rNodeID != dhtNodeID.String() { t.Error("unexpected node ID") } } @@ -106,8 +107,8 @@ func TestPing(t *testing.T) { } func TestStore(t *testing.T) { - dhtNodeID := RandomBitmapP() - testNodeID := RandomBitmapP() + dhtNodeID := bits.Rand() + testNodeID := bits.Rand() conn := newTestUDPConn("127.0.0.1:21217") @@ -123,7 +124,7 @@ func TestStore(t *testing.T) { defer dht.Shutdown() messageID := newMessageID() - blobHashToStore := RandomBitmapP() + blobHashToStore := bits.Rand() storeRequest := Request{ ID: messageID, @@ -176,7 +177,7 @@ func TestStore(t *testing.T) { } } - verifyResponse(t, response, messageID, dhtNodeID.rawString()) + verifyResponse(t, response, messageID, dhtNodeID.String()) _, ok := response[headerPayloadField] if !ok { @@ -204,8 +205,8 @@ func TestStore(t *testing.T) { } func TestFindNode(t *testing.T) { - dhtNodeID := RandomBitmapP() - testNodeID := RandomBitmapP() + dhtNodeID := bits.Rand() + testNodeID := bits.Rand() conn := newTestUDPConn("127.0.0.1:21217") @@ -223,13 +224,13 @@ func TestFindNode(t *testing.T) { nodesToInsert := 3 var nodes []Contact for i := 0; i < nodesToInsert; i++ { - n := Contact{ID: RandomBitmapP(), IP: net.ParseIP("127.0.0.1"), Port: 10000 + i} + n := Contact{ID: bits.Rand(), IP: net.ParseIP("127.0.0.1"), Port: 10000 + i} nodes = append(nodes, n) dht.node.rt.Update(n) } messageID := newMessageID() - blobHashToFind := RandomBitmapP() + blobHashToFind := bits.Rand() request := Request{ ID: messageID, @@ -257,7 +258,7 @@ func TestFindNode(t *testing.T) { } } - verifyResponse(t, response, messageID, dhtNodeID.rawString()) + verifyResponse(t, response, messageID, dhtNodeID.String()) _, ok := response[headerPayloadField] if !ok { @@ -273,8 +274,8 @@ func TestFindNode(t *testing.T) { } func TestFindValueExisting(t *testing.T) { - dhtNodeID := RandomBitmapP() - testNodeID := RandomBitmapP() + dhtNodeID := bits.Rand() + testNodeID := bits.Rand() conn := newTestUDPConn("127.0.0.1:21217") @@ -291,16 +292,16 @@ func TestFindValueExisting(t *testing.T) { nodesToInsert := 3 for i := 0; i < nodesToInsert; i++ { - n := Contact{ID: RandomBitmapP(), IP: net.ParseIP("127.0.0.1"), Port: 10000 + i} + n := Contact{ID: bits.Rand(), IP: net.ParseIP("127.0.0.1"), Port: 10000 + i} dht.node.rt.Update(n) } //data, _ := hex.DecodeString("64313a30693065313a3132303a7de8e57d34e316abbb5a8a8da50dcd1ad4c80e0f313a3234383a7ce1b831dec8689e44f80f547d2dea171f6a625e1a4ff6c6165e645f953103dabeb068a622203f859c6c64658fd3aa3b313a33393a66696e6456616c7565313a346c34383aa47624b8e7ee1e54df0c45e2eb858feb0b705bd2a78d8b739be31ba188f4bd6f56b371c51fecc5280d5fd26ba4168e966565") messageID := newMessageID() - valueToFind := RandomBitmapP() + valueToFind := bits.Rand() - nodeToFind := Contact{ID: RandomBitmapP(), IP: net.ParseIP("1.2.3.4"), Port: 1286} + nodeToFind := Contact{ID: bits.Rand(), IP: net.ParseIP("1.2.3.4"), Port: 1286} dht.node.store.Upsert(valueToFind, nodeToFind) dht.node.store.Upsert(valueToFind, nodeToFind) dht.node.store.Upsert(valueToFind, nodeToFind) @@ -331,7 +332,7 @@ func TestFindValueExisting(t *testing.T) { } } - verifyResponse(t, response, messageID, dhtNodeID.rawString()) + verifyResponse(t, response, messageID, dhtNodeID.String()) _, ok := response[headerPayloadField] if !ok { @@ -343,7 +344,7 @@ func TestFindValueExisting(t *testing.T) { t.Fatal("payload is not a dictionary") } - compactContacts, ok := payload[valueToFind.rawString()] + compactContacts, ok := payload[valueToFind.String()] if !ok { t.Fatal("payload is missing key for search value") } @@ -357,8 +358,8 @@ func TestFindValueExisting(t *testing.T) { } func TestFindValueFallbackToFindNode(t *testing.T) { - dhtNodeID := RandomBitmapP() - testNodeID := RandomBitmapP() + dhtNodeID := bits.Rand() + testNodeID := bits.Rand() conn := newTestUDPConn("127.0.0.1:21217") @@ -376,13 +377,13 @@ func TestFindValueFallbackToFindNode(t *testing.T) { nodesToInsert := 3 var nodes []Contact for i := 0; i < nodesToInsert; i++ { - n := Contact{ID: RandomBitmapP(), IP: net.ParseIP("127.0.0.1"), Port: 10000 + i} + n := Contact{ID: bits.Rand(), IP: net.ParseIP("127.0.0.1"), Port: 10000 + i} nodes = append(nodes, n) dht.node.rt.Update(n) } messageID := newMessageID() - valueToFind := RandomBitmapP() + valueToFind := bits.Rand() request := Request{ ID: messageID, @@ -410,7 +411,7 @@ func TestFindValueFallbackToFindNode(t *testing.T) { } } - verifyResponse(t, response, messageID, dhtNodeID.rawString()) + verifyResponse(t, response, messageID, dhtNodeID.String()) _, ok := response[headerPayloadField] if !ok { diff --git a/dht/routing_table.go b/dht/routing_table.go index 29ec855..7ac80d8 100644 --- a/dht/routing_table.go +++ b/dht/routing_table.go @@ -13,6 +13,7 @@ import ( "github.com/lbryio/lbry.go/errors" "github.com/lbryio/lbry.go/stopOnce" + "github.com/lbryio/reflector.go/dht/bits" "github.com/lyoshenka/bencode" log "github.com/sirupsen/logrus" @@ -25,7 +26,7 @@ import ( // Contact is a type representation of another node that a specific node is in communication with. type Contact struct { - ID Bitmap + ID bits.Bitmap IP net.IP Port int } @@ -74,7 +75,7 @@ func (c *Contact) UnmarshalCompact(b []byte) error { } c.IP = net.IPv4(b[0], b[1], b[2], b[3]).To4() c.Port = int(uint16(b[5]) | uint16(b[4])<<8) - c.ID = BitmapFromBytesP(b[6:]) + c.ID = bits.FromBytesP(b[6:]) return nil } @@ -120,7 +121,7 @@ func (c *Contact) UnmarshalBencode(b []byte) error { type sortedContact struct { contact Contact - xorDistanceToTarget Bitmap + xorDistanceToTarget bits.Bitmap } type byXorDistance []sortedContact @@ -222,7 +223,7 @@ func (b *bucket) UpdateContact(c Contact, insertIfNew bool) { } // FailContact marks a contact as having failed, and removes it if it failed too many times -func (b *bucket) FailContact(id Bitmap) { +func (b *bucket) FailContact(id bits.Bitmap) { b.lock.Lock() defer b.lock.Unlock() i := find(id, b.peers) @@ -233,7 +234,7 @@ func (b *bucket) FailContact(id Bitmap) { } // find returns the contact in the bucket, or nil if the bucket does not contain the contact -func find(id Bitmap, peers []peer) int { +func find(id bits.Bitmap, peers []peer) int { for i := range peers { if peers[i].Contact.ID.Equals(id) { return i @@ -250,11 +251,11 @@ func (b *bucket) NeedsRefresh(refreshInterval time.Duration) bool { } type routingTable struct { - id Bitmap + id bits.Bitmap buckets [nodeIDBits]bucket } -func newRoutingTable(id Bitmap) *routingTable { +func newRoutingTable(id bits.Bitmap) *routingTable { var rt routingTable rt.id = id for i := range rt.buckets { @@ -301,7 +302,7 @@ func (rt *routingTable) Fail(c Contact) { // GetClosest returns the closest `limit` contacts from the routing table // It marks each bucket it accesses as having been accessed -func (rt *routingTable) GetClosest(target Bitmap, limit int) []Contact { +func (rt *routingTable) GetClosest(target bits.Bitmap, limit int) []Contact { var toSort []sortedContact var bucketNum int @@ -335,7 +336,7 @@ func (rt *routingTable) GetClosest(target Bitmap, limit int) []Contact { return contacts } -func appendContacts(contacts []sortedContact, b bucket, target Bitmap) []sortedContact { +func appendContacts(contacts []sortedContact, b bucket, target bits.Bitmap) []sortedContact { for _, contact := range b.Contacts() { contacts = append(contacts, sortedContact{contact, contact.ID.Xor(target)}) } @@ -353,8 +354,8 @@ func (rt *routingTable) Count() int { // Range is a structure that holds a min and max bitmaps. The range is used in bucket sizing. type Range struct { - start Bitmap - end Bitmap + start bits.Bitmap + end bits.Bitmap } // BucketRanges returns a slice of ranges, where the `start` of each range is the smallest id that can @@ -370,22 +371,22 @@ func (rt *routingTable) BucketRanges() []Range { return ranges } -func (rt *routingTable) bucketNumFor(target Bitmap) int { +func (rt *routingTable) bucketNumFor(target bits.Bitmap) int { if rt.id.Equals(target) { panic("routing table does not have a bucket for its own id") } return nodeIDBits - 1 - target.Xor(rt.id).PrefixLen() } -func (rt *routingTable) bucketFor(target Bitmap) *bucket { +func (rt *routingTable) bucketFor(target bits.Bitmap) *bucket { return &rt.buckets[rt.bucketNumFor(target)] } -func (rt *routingTable) GetIDsForRefresh(refreshInterval time.Duration) []Bitmap { - var bitmaps []Bitmap +func (rt *routingTable) GetIDsForRefresh(refreshInterval time.Duration) []bits.Bitmap { + var bitmaps []bits.Bitmap for i, bucket := range rt.buckets { if bucket.NeedsRefresh(refreshInterval) { - bitmaps = append(bitmaps, RandomBitmapP().Prefix(i, false)) + bitmaps = append(bitmaps, bits.Rand().Prefix(i, false)) } } return bitmaps @@ -416,7 +417,7 @@ func (rt *routingTable) UnmarshalJSON(b []byte) error { return err } - rt.id, err = BitmapFromHex(data.ID) + rt.id, err = bits.FromHex(data.ID) if err != nil { return errors.Prefix("decoding ID", err) } @@ -427,7 +428,7 @@ func (rt *routingTable) UnmarshalJSON(b []byte) error { return errors.Err("decoding contact %s: wrong number of parts", s) } var c Contact - c.ID, err = BitmapFromHex(parts[0]) + c.ID, err = bits.FromHex(parts[0]) if err != nil { return errors.Err("decoding contact %s: invalid ID: %s", s, err) } @@ -451,7 +452,7 @@ func RoutingTableRefresh(n *Node, refreshInterval time.Duration, upstreamStop st for _, id := range n.rt.GetIDsForRefresh(refreshInterval) { done.Add(1) - go func(id Bitmap) { + go func(id bits.Bitmap) { defer done.Done() _, _, err := FindContacts(n, id, false, upstreamStop) if err != nil { diff --git a/dht/routing_table_test.go b/dht/routing_table_test.go index 95de453..a889a5a 100644 --- a/dht/routing_table_test.go +++ b/dht/routing_table_test.go @@ -8,24 +8,25 @@ import ( "strings" "testing" + "github.com/lbryio/reflector.go/dht/bits" "github.com/sebdah/goldie" ) func TestRoutingTable_bucketFor(t *testing.T) { - rt := newRoutingTable(BitmapFromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")) + rt := newRoutingTable(bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")) var tests = []struct { - id Bitmap + id bits.Bitmap expected int }{ - {BitmapFromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), 0}, - {BitmapFromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002"), 1}, - {BitmapFromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003"), 1}, - {BitmapFromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004"), 2}, - {BitmapFromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005"), 2}, - {BitmapFromHexP("00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f"), 3}, - {BitmapFromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010"), 4}, - {BitmapFromHexP("F00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 383}, - {BitmapFromHexP("F0000000000000000000000000000000F0000000000000000000000000F0000000000000000000000000000000000000"), 383}, + {bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), 0}, + {bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002"), 1}, + {bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003"), 1}, + {bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004"), 2}, + {bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005"), 2}, + {bits.FromHexP("00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f"), 3}, + {bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010"), 4}, + {bits.FromHexP("F00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 383}, + {bits.FromHexP("F0000000000000000000000000000000F0000000000000000000000000F0000000000000000000000000000000000000"), 383}, } for _, tt := range tests { @@ -37,14 +38,14 @@ func TestRoutingTable_bucketFor(t *testing.T) { } func TestRoutingTable_GetClosest(t *testing.T) { - n1 := BitmapFromHexP("FFFFFFFF0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000") - n2 := BitmapFromHexP("FFFFFFF00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000") - n3 := BitmapFromHexP("111111110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000") + n1 := bits.FromHexP("FFFFFFFF0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000") + n2 := bits.FromHexP("FFFFFFF00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000") + n3 := bits.FromHexP("111111110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000") rt := newRoutingTable(n1) rt.Update(Contact{n2, net.ParseIP("127.0.0.1"), 8001}) rt.Update(Contact{n3, net.ParseIP("127.0.0.1"), 8002}) - contacts := rt.GetClosest(BitmapFromHexP("222222220000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 1) + contacts := rt.GetClosest(bits.FromHexP("222222220000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 1) if len(contacts) != 1 { t.Fail() return @@ -68,7 +69,7 @@ func TestRoutingTable_GetClosest(t *testing.T) { func TestCompactEncoding(t *testing.T) { c := Contact{ - ID: BitmapFromHexP("1c8aff71b99462464d9eeac639595ab99664be3482cb91a29d87467515c7d9158fe72aa1f1582dab07d8f8b5db277f41"), + ID: bits.FromHexP("1c8aff71b99462464d9eeac639595ab99664be3482cb91a29d87467515c7d9158fe72aa1f1582dab07d8f8b5db277f41"), IP: net.ParseIP("1.2.3.4"), Port: int(55<<8 + 66), } @@ -144,13 +145,13 @@ func TestRoutingTable_MoveToBack(t *testing.T) { } func TestRoutingTable_BucketRanges(t *testing.T) { - id := BitmapFromHexP("1c8aff71b99462464d9eeac639595ab99664be3482cb91a29d87467515c7d9158fe72aa1f1582dab07d8f8b5db277f41") + id := bits.FromHexP("1c8aff71b99462464d9eeac639595ab99664be3482cb91a29d87467515c7d9158fe72aa1f1582dab07d8f8b5db277f41") ranges := newRoutingTable(id).BucketRanges() if !ranges[0].start.Equals(ranges[0].end) { t.Error("first bucket should only fit exactly one id") } for i := 0; i < 1000; i++ { - randID := RandomBitmapP() + randID := bits.Rand() found := -1 for i, r := range ranges { if r.start.LessOrEqual(randID) && r.end.GreaterOrEqual(randID) { @@ -168,17 +169,17 @@ func TestRoutingTable_BucketRanges(t *testing.T) { } func TestRoutingTable_Save(t *testing.T) { - id := BitmapFromHexP("1c8aff71b99462464d9eeac639595ab99664be3482cb91a29d87467515c7d9158fe72aa1f1582dab07d8f8b5db277f41") + id := bits.FromHexP("1c8aff71b99462464d9eeac639595ab99664be3482cb91a29d87467515c7d9158fe72aa1f1582dab07d8f8b5db277f41") rt := newRoutingTable(id) ranges := rt.BucketRanges() for i, r := range ranges { for j := 0; j < bucketSize; j++ { - toAdd := r.start.Add(BitmapFromShortHexP(strconv.Itoa(j))) + toAdd := r.start.Add(bits.FromShortHexP(strconv.Itoa(j))) if toAdd.LessOrEqual(r.end) { rt.Update(Contact{ - ID: r.start.Add(BitmapFromShortHexP(strconv.Itoa(j))), + ID: r.start.Add(bits.FromShortHexP(strconv.Itoa(j))), IP: net.ParseIP("1.2.3." + strconv.Itoa(j)), Port: 1 + i*bucketSize + j, }) diff --git a/dht/store.go b/dht/store.go index 25a85d8..bc77e53 100644 --- a/dht/store.go +++ b/dht/store.go @@ -1,36 +1,40 @@ package dht -import "sync" +import ( + "sync" + + "github.com/lbryio/reflector.go/dht/bits" +) // TODO: expire stored data after tExpire time type contactStore struct { // map of blob hashes to (map of node IDs to bools) - hashes map[Bitmap]map[Bitmap]bool + hashes map[bits.Bitmap]map[bits.Bitmap]bool // stores the peers themselves, so they can be updated in one place - contacts map[Bitmap]Contact + contacts map[bits.Bitmap]Contact lock sync.RWMutex } func newStore() *contactStore { return &contactStore{ - hashes: make(map[Bitmap]map[Bitmap]bool), - contacts: make(map[Bitmap]Contact), + hashes: make(map[bits.Bitmap]map[bits.Bitmap]bool), + contacts: make(map[bits.Bitmap]Contact), } } -func (s *contactStore) Upsert(blobHash Bitmap, contact Contact) { +func (s *contactStore) Upsert(blobHash bits.Bitmap, contact Contact) { s.lock.Lock() defer s.lock.Unlock() if _, ok := s.hashes[blobHash]; !ok { - s.hashes[blobHash] = make(map[Bitmap]bool) + s.hashes[blobHash] = make(map[bits.Bitmap]bool) } s.hashes[blobHash][contact.ID] = true s.contacts[contact.ID] = contact } -func (s *contactStore) Get(blobHash Bitmap) []Contact { +func (s *contactStore) Get(blobHash bits.Bitmap) []Contact { s.lock.RLock() defer s.lock.RUnlock() diff --git a/dht/testing.go b/dht/testing.go index b93d2a0..95efea7 100644 --- a/dht/testing.go +++ b/dht/testing.go @@ -8,6 +8,7 @@ import ( "time" "github.com/lbryio/lbry.go/errors" + "github.com/lbryio/reflector.go/dht/bits" ) var testingDHTIP = "127.0.0.1" @@ -21,7 +22,7 @@ func TestingCreateDHT(t *testing.T, numNodes int, bootstrap, concurrent bool) (* if bootstrap { bootstrapAddress := testingDHTIP + ":" + strconv.Itoa(testingDHTFirstPort) seeds = []string{bootstrapAddress} - bootstrapNode = NewBootstrapNode(RandomBitmapP(), 0, bootstrapDefaultRefreshDuration) + bootstrapNode = NewBootstrapNode(bits.Rand(), 0, bootstrapDefaultRefreshDuration) listener, err := net.ListenPacket(network, bootstrapAddress) if err != nil { panic(err) @@ -39,7 +40,7 @@ func TestingCreateDHT(t *testing.T, numNodes int, bootstrap, concurrent bool) (* dhts := make([]*DHT, numNodes) for i := 0; i < numNodes; i++ { - dht, err := New(&Config{Address: testingDHTIP + ":" + strconv.Itoa(firstPort+i), NodeID: RandomBitmapP().Hex(), SeedNodes: seeds}) + dht, err := New(&Config{Address: testingDHTIP + ":" + strconv.Itoa(firstPort+i), NodeID: bits.Rand().Hex(), SeedNodes: seeds}) if err != nil { panic(err) } @@ -225,7 +226,7 @@ func verifyContacts(t *testing.T, contacts []interface{}, nodes []Contact) { continue } for _, n := range nodes { - if n.ID.rawString() == id { + if n.ID.String() == id { currNode = n currNodeFound = true foundNodes[id] = true @@ -305,12 +306,3 @@ func verifyCompactContacts(t *testing.T, contacts []interface{}, nodes []Contact } } } - -func assertPanic(t *testing.T, text string, f func()) { - defer func() { - if r := recover(); r == nil { - t.Errorf("%s: did not panic as expected", text) - } - }() - f() -} diff --git a/dht/token_manager.go b/dht/token_manager.go index e4dfa6d..718cad0 100644 --- a/dht/token_manager.go +++ b/dht/token_manager.go @@ -10,6 +10,7 @@ import ( "time" "github.com/lbryio/lbry.go/stopOnce" + "github.com/lbryio/reflector.go/dht/bits" ) type tokenManager struct { @@ -46,15 +47,15 @@ func (tm *tokenManager) Stop() { tm.stop.StopAndWait() } -func (tm *tokenManager) Get(nodeID Bitmap, addr *net.UDPAddr) string { +func (tm *tokenManager) Get(nodeID bits.Bitmap, addr *net.UDPAddr) string { return genToken(tm.secret, nodeID, addr) } -func (tm *tokenManager) Verify(token string, nodeID Bitmap, addr *net.UDPAddr) bool { +func (tm *tokenManager) Verify(token string, nodeID bits.Bitmap, addr *net.UDPAddr) bool { return token == genToken(tm.secret, nodeID, addr) || token == genToken(tm.prevSecret, nodeID, addr) } -func genToken(secret []byte, nodeID Bitmap, addr *net.UDPAddr) string { +func genToken(secret []byte, nodeID bits.Bitmap, addr *net.UDPAddr) string { buf := bytes.Buffer{} buf.Write(nodeID[:]) buf.Write(addr.IP)