From a6041dc6544ddfba1f60185b0c011615871b6c49 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie <jimmyzelinskie@gmail.com> Date: Sat, 30 Nov 2013 23:00:20 -0500 Subject: [PATCH] move drivers to dedicated repos --- storage/backend/batter/batter.go | 56 -- storage/backend/batter/load.go | 25 - storage/backend/gazelle/flush.go | 62 -- storage/backend/gazelle/gazelle.go | 99 ---- storage/backend/gazelle/load.go | 25 - storage/tracker/redis/conn_test.go | 563 ------------------ storage/tracker/redis/redis.go | 689 ---------------------- storage/tracker/redis/redis_bench_test.go | 288 --------- storage/tracker/redis/redis_test.go | 184 ------ 9 files changed, 1991 deletions(-) delete mode 100644 storage/backend/batter/batter.go delete mode 100644 storage/backend/batter/load.go delete mode 100644 storage/backend/gazelle/flush.go delete mode 100644 storage/backend/gazelle/gazelle.go delete mode 100644 storage/backend/gazelle/load.go delete mode 100644 storage/tracker/redis/conn_test.go delete mode 100644 storage/tracker/redis/redis.go delete mode 100644 storage/tracker/redis/redis_bench_test.go delete mode 100644 storage/tracker/redis/redis_test.go diff --git a/storage/backend/batter/batter.go b/storage/backend/batter/batter.go deleted file mode 100644 index 6112bd9..0000000 --- a/storage/backend/batter/batter.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2013 The Chihaya Authors. All rights reserved. -// Use of this source code is governed by the BSD 2-Clause license, -// which can be found in the LICENSE file. - -// Package batter provides a driver for a BitTorrent tracker to interface -// with the postgres database used by batter (github.com/wafflesfm/batter). -package batter - -import ( - "database/sql" - "fmt" - - "github.com/chihaya/chihaya/config" - "github.com/chihaya/chihaya/storage/backend" - - _ "github.com/bmizerany/pq" -) - -type driver struct{} - -func (d *driver) New(conf *config.DataStore) backend.Conn { - dsn := fmt.Sprintf( - "host=%s user=%s password=%s dbname=%s", - conf.Host, - conf.Port, - conf.Username, - conf.Password, - conf.Schema, - ) - db, err := sql.Open("postgres", dsn) - if err != nil { - panic("batter: failed to open connection to postgres") - } - - if conf.MaxIdleConns != 0 { - db.SetMaxIdleConns(conf.MaxIdleConns) - } - - return &Conn{db} -} - -type Conn struct { - *sql.DB -} - -func (c *Conn) Start() error { - return nil -} - -func (c *Conn) RecordAnnounce(delta *backend.AnnounceDelta) error { - return nil -} - -func init() { - backend.Register("batter", &driver{}) -} diff --git a/storage/backend/batter/load.go b/storage/backend/batter/load.go deleted file mode 100644 index 287b51b..0000000 --- a/storage/backend/batter/load.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2013 The Chihaya Authors. All rights reserved. -// Use of this source code is governed by the BSD 2-Clause license, -// which can be found in the LICENSE file. - -package batter - -import ( - "github.com/chihaya/chihaya/storage" -) - -func (c *Conn) LoadTorrents(ids []uint64) ([]*storage.Torrent, error) { - return nil, nil -} - -func (c *Conn) LoadAllTorrents() ([]*storage.Torrent, error) { - return nil, nil -} - -func (c *Conn) LoadUsers(ids []uint64) ([]*storage.User, error) { - return nil, nil -} - -func (c *Conn) LoadAllUsers(ids []uint64) ([]*storage.User, error) { - return nil, nil -} diff --git a/storage/backend/gazelle/flush.go b/storage/backend/gazelle/flush.go deleted file mode 100644 index d847d2d..0000000 --- a/storage/backend/gazelle/flush.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2013 The Chihaya Authors. All rights reserved. -// Use of this source code is governed by the BSD 2-Clause license, -// which can be found in the LICENSE file. - -package gazelle - -import ( - "bytes" - "log" - "time" -) - -func (c *Conn) flushTorrents() { - var query bytes.Buffer - c.waitGroup.Add(1) - defer c.waitGroup.Done() - var count int - - for { - length := len(c.torrentChannel) - query.Reset() - - query.WriteString("INSERT INTO torrents (ID, Snatched, Seeders, Leechers, last_action) VALUES\n") - - for count = 0; count < length; count++ { - s := <-c.torrentChannel - if s == "" { - break - } - query.WriteString(s) - - if count != length-1 { - query.WriteRune(',') - } - } - - if !c.terminate { - log.Printf("[torrents] Flushing %d\n", count) - } - - if count > 0 { - query.WriteString("\nON DUPLICATE KEY UPDATE Snatched = Snatched + VALUES(Snatched), " + - "Seeders = VALUES(Seeders), Leechers = VALUES(Leechers), " + - "last_action = IF(last_action < VALUES(last_action), VALUES(last_action), last_action);") - - c.Exec(query.String()) - - if length < cap(c.torrentChannel)/2 { - time.Sleep(200 * time.Millisecond) - } - } else if c.terminate { - break - } else { - time.Sleep(time.Second) - } - } -} - -func (c *Conn) flushUsers() {} -func (c *Conn) flushTransferHistory() {} -func (c *Conn) flushTransferIps() {} -func (c *Conn) flushSnatches() {} diff --git a/storage/backend/gazelle/gazelle.go b/storage/backend/gazelle/gazelle.go deleted file mode 100644 index f439474..0000000 --- a/storage/backend/gazelle/gazelle.go +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2013 The Chihaya Authors. All rights reserved. -// Use of this source code is governed by the BSD 2-Clause license, -// which can be found in the LICENSE file. - -// Package gazelle provides a driver for a BitTorrent tracker to interface -// with the MySQL database used by Gazelle (github.com/WhatCD/Gazelle). -package gazelle - -import ( - "database/sql" - "fmt" - "sync" - - "github.com/chihaya/chihaya/config" - "github.com/chihaya/chihaya/storage/backend" - - _ "github.com/go-sql-driver/mysql" -) - -type driver struct{} - -func (d *driver) New(conf *config.DataStore) backend.Conn { - dsn := fmt.Sprintf( - "%s:%s@%s:%s/%s?charset=utf8mb4,utf8", - conf.Username, - conf.Password, - conf.Host, - conf.Port, - conf.Schema, - ) - db, err := sql.Open("mysql", dsn) - if err != nil { - panic("gazelle: failed to open connection to MySQL") - } - - if conf.MaxIdleConns != 0 { - db.SetMaxIdleConns(conf.MaxIdleConns) - } - - conn := &Conn{DB: db} - - // TODO Buffer sizes - conn.torrentChannel = make(chan string, 1000) - conn.userChannel = make(chan string, 1000) - conn.transferHistoryChannel = make(chan string, 1000) - conn.transferIpsChannel = make(chan string, 1000) - conn.snatchChannel = make(chan string, 100) - - return conn -} - -type Conn struct { - waitGroup sync.WaitGroup - terminate bool - - torrentChannel chan string - userChannel chan string - transferHistoryChannel chan string - transferIpsChannel chan string - snatchChannel chan string - - *sql.DB -} - -func (c *Conn) Start() error { - go c.flushTorrents() - go c.flushUsers() - go c.flushTransferHistory() - go c.flushTransferIps() - go c.flushSnatches() - return nil -} - -func (c *Conn) Close() error { - c.terminate = true - c.waitGroup.Wait() - return c.DB.Close() -} - -func (c *Conn) RecordAnnounce(delta *backend.AnnounceDelta) error { - snatchCount := 0 - if delta.Snatched { - snatchCount = 1 - } - - c.torrentChannel <- fmt.Sprintf( - "('%d','%d','%d','%d','%d')", - delta.Torrent.ID, - snatchCount, - len(delta.Torrent.Seeders), - len(delta.Torrent.Leechers), - delta.Torrent.LastAction, - ) - return nil -} - -func init() { - backend.Register("gazelle", &driver{}) -} diff --git a/storage/backend/gazelle/load.go b/storage/backend/gazelle/load.go deleted file mode 100644 index 6da82d4..0000000 --- a/storage/backend/gazelle/load.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2013 The Chihaya Authors. All rights reserved. -// Use of this source code is governed by the BSD 2-Clause license, -// which can be found in the LICENSE file. - -package gazelle - -import ( - "github.com/chihaya/chihaya/storage" -) - -func (c *Conn) LoadTorrents(ids []uint64) ([]*storage.Torrent, error) { - return nil, nil -} - -func (c *Conn) LoadAllTorrents() ([]*storage.Torrent, error) { - return nil, nil -} - -func (c *Conn) LoadUsers(ids []uint64) ([]*storage.User, error) { - return nil, nil -} - -func (c *Conn) LoadAllUsers(ids []uint64) ([]*storage.User, error) { - return nil, nil -} diff --git a/storage/tracker/redis/conn_test.go b/storage/tracker/redis/conn_test.go deleted file mode 100644 index a9a7570..0000000 --- a/storage/tracker/redis/conn_test.go +++ /dev/null @@ -1,563 +0,0 @@ -// Copyright 2013 The Chihaya Authors. All rights reserved. -// Use of this source code is governed by the BSD 2-Clause license, -// which can be found in the LICENSE file. - -package redis - -import ( - "math/rand" - "os" - "reflect" - "testing" - "time" - - "github.com/chihaya/chihaya/config" - "github.com/chihaya/chihaya/storage" - "github.com/chihaya/chihaya/storage/tracker" -) - -func createTestConn() tracker.Conn { - testConfig, err := config.Open(os.Getenv("TESTCONFIGPATH")) - panicOnErr(err) - conf := &testConfig.Cache - - testPool, err := tracker.Open(conf) - panicOnErr(err) - - newConn, err := testPool.Get() - panicOnErr(err) - - return newConn -} - -func TestFindUserSuccess(t *testing.T) { - conn := createTestConn() - testUser := createTestUser() - - panicOnErr(conn.AddUser(testUser)) - foundUser, found, err := conn.FindUser(testUser.Passkey) - panicOnErr(err) - if !found { - t.Error("user not found", testUser) - } - if *foundUser != *testUser { - t.Error("found user mismatch", *foundUser, testUser) - } - // Cleanup - panicOnErr(conn.RemoveUser(testUser)) -} - -func TestFindUserFail(t *testing.T) { - conn := createTestConn() - testUser := createTestUser() - - foundUser, found, err := conn.FindUser(testUser.Passkey) - panicOnErr(err) - if found { - t.Error("user found", foundUser) - } -} - -func TestRemoveUser(t *testing.T) { - conn := createTestConn() - testUser := createTestUser() - - panicOnErr(conn.AddUser(testUser)) - err := conn.RemoveUser(testUser) - panicOnErr(err) - foundUser, found, err := conn.FindUser(testUser.Passkey) - panicOnErr(err) - if found { - t.Error("removed user found", foundUser) - } -} - -func TestFindTorrentSuccess(t *testing.T) { - conn := createTestConn() - testTorrent := createTestTorrent() - panicOnErr(conn.AddTorrent(testTorrent)) - - foundTorrent, found, err := conn.FindTorrent(testTorrent.Infohash) - panicOnErr(err) - if !found { - t.Error("torrent not found", testTorrent) - } - if !reflect.DeepEqual(foundTorrent, testTorrent) { - t.Error("found torrent mismatch", foundTorrent, testTorrent) - } - // Cleanup - panicOnErr(conn.RemoveTorrent(testTorrent)) -} - -func TestFindTorrentFail(t *testing.T) { - conn := createTestConn() - testTorrent := createTestTorrent() - - foundTorrent, found, err := conn.FindTorrent(testTorrent.Infohash) - panicOnErr(err) - if found { - t.Error("torrent found", foundTorrent) - } -} - -func TestRemoveTorrent(t *testing.T) { - conn := createTestConn() - testTorrent := createTestTorrent() - panicOnErr(conn.AddTorrent(testTorrent)) - - panicOnErr(conn.RemoveTorrent(testTorrent)) - foundTorrent, found, err := conn.FindTorrent(testTorrent.Infohash) - panicOnErr(err) - if found { - t.Error("removed torrent found", foundTorrent) - } - // Cleanup - panicOnErr(conn.RemoveTorrent(testTorrent)) -} - -func TestClientWhitelistSuccess(t *testing.T) { - conn := createTestConn() - testPeerID := "-lt0D30-" - - panicOnErr(conn.WhitelistClient(testPeerID)) - found, err := conn.ClientWhitelisted(testPeerID) - panicOnErr(err) - if !found { - t.Error("peerID not found", testPeerID) - } - // Cleanup - panicOnErr(conn.UnWhitelistClient(testPeerID)) -} - -func TestClientWhitelistFail(t *testing.T) { - conn := createTestConn() - testPeerID2 := "TIX0192" - - found, err := conn.ClientWhitelisted(testPeerID2) - panicOnErr(err) - if found { - t.Error("peerID found", testPeerID2) - } -} - -func TestRecordSnatch(t *testing.T) { - conn := createTestConn() - testTorrent := createTestTorrent() - testUser := createTestUser() - panicOnErr(conn.AddTorrent(testTorrent)) - panicOnErr(conn.AddUser(testUser)) - - userSnatches := testUser.Snatches - torrentSnatches := testTorrent.Snatches - - panicOnErr(conn.RecordSnatch(testUser, testTorrent)) - - foundTorrent, _, err := conn.FindTorrent(testTorrent.Infohash) - panicOnErr(err) - foundUser, _, err := conn.FindUser(testUser.Passkey) - panicOnErr(err) - - if testUser.Snatches != userSnatches+1 { - t.Error("snatch not recorded to local user", testUser.Snatches, userSnatches+1) - } - if testTorrent.Snatches != torrentSnatches+1 { - t.Error("snatch not recorded to local torrent") - } - if foundUser.Snatches != userSnatches+1 { - t.Error("snatch not recorded to cached user", foundUser.Snatches, userSnatches+1) - } - if foundTorrent.Snatches != torrentSnatches+1 { - t.Error("snatch not recorded to cached torrent") - } - // Cleanup - panicOnErr(conn.RemoveTorrent(testTorrent)) - panicOnErr(conn.RemoveUser(testUser)) -} - -func TestMarkActive(t *testing.T) { - conn := createTestConn() - testTorrent := createTestTorrent() - testTorrent.Active = false - panicOnErr(conn.AddTorrent(testTorrent)) - - panicOnErr(conn.MarkActive(testTorrent)) - foundTorrent, _, err := conn.FindTorrent(testTorrent.Infohash) - panicOnErr(err) - - if foundTorrent.Active != true { - t.Error("cached torrent not activated") - } - if testTorrent.Active != true { - t.Error("cached torrent not activated") - } - // Cleanup - panicOnErr(conn.RemoveTorrent(testTorrent)) -} - -func TestClientWhitelistRemove(t *testing.T) { - conn := createTestConn() - testPeerID := "-lt0D30-" - panicOnErr(conn.WhitelistClient(testPeerID)) - panicOnErr(conn.UnWhitelistClient(testPeerID)) - - found, err := conn.ClientWhitelisted(testPeerID) - panicOnErr(err) - if found { - t.Error("removed peerID found", testPeerID) - } -} - -func TestAddSeeder(t *testing.T) { - conn := createTestConn() - testTorrent := createTestTorrent() - panicOnErr(conn.AddTorrent(testTorrent)) - testSeeder := createTestPeer(createTestUserID(), testTorrent.ID) - - panicOnErr(conn.AddSeeder(testTorrent, testSeeder)) - foundTorrent, found, err := conn.FindTorrent(testTorrent.Infohash) - panicOnErr(err) - foundSeeder, found := foundTorrent.Seeders[storage.PeerMapKey(testSeeder)] - if found && foundSeeder != *testSeeder { - t.Error("seeder not added to cache", testSeeder) - } - foundSeeder, found = testTorrent.Seeders[storage.PeerMapKey(testSeeder)] - if found && foundSeeder != *testSeeder { - t.Error("seeder not added to local", testSeeder) - } - // Cleanup - panicOnErr(conn.RemoveTorrent(testTorrent)) -} - -func TestAddLeecher(t *testing.T) { - conn := createTestConn() - testTorrent := createTestTorrent() - panicOnErr(conn.AddTorrent(testTorrent)) - testLeecher := createTestPeer(createTestUserID(), testTorrent.ID) - - panicOnErr(conn.AddLeecher(testTorrent, testLeecher)) - foundTorrent, found, err := conn.FindTorrent(testTorrent.Infohash) - panicOnErr(err) - foundLeecher, found := foundTorrent.Leechers[storage.PeerMapKey(testLeecher)] - if found && foundLeecher != *testLeecher { - t.Error("leecher not added to cache", testLeecher) - } - foundLeecher, found = testTorrent.Leechers[storage.PeerMapKey(testLeecher)] - if found && foundLeecher != *testLeecher { - t.Error("leecher not added to local", testLeecher) - } - // Cleanup - panicOnErr(conn.RemoveTorrent(testTorrent)) -} - -func TestRemoveSeeder(t *testing.T) { - conn := createTestConn() - testTorrent := createTestTorrent() - panicOnErr(conn.AddTorrent(testTorrent)) - testSeeder := createTestPeer(createTestUserID(), testTorrent.ID) - panicOnErr(conn.AddSeeder(testTorrent, testSeeder)) - - panicOnErr(conn.RemoveSeeder(testTorrent, testSeeder)) - foundSeeder, found := testTorrent.Seeders[storage.PeerMapKey(testSeeder)] - if found || foundSeeder == *testSeeder { - t.Error("seeder not removed from local", foundSeeder) - } - - foundTorrent, found, err := conn.FindTorrent(testTorrent.Infohash) - panicOnErr(err) - foundSeeder, found = foundTorrent.Seeders[storage.PeerMapKey(testSeeder)] - if found || foundSeeder == *testSeeder { - t.Error("seeder not removed from cache", foundSeeder, *testSeeder) - } - // Cleanup - panicOnErr(conn.RemoveTorrent(testTorrent)) -} - -func TestRemoveLeecher(t *testing.T) { - conn := createTestConn() - testTorrent := createTestTorrent() - panicOnErr(conn.AddTorrent(testTorrent)) - testLeecher := createTestPeer(createTestUserID(), testTorrent.ID) - panicOnErr(conn.AddLeecher(testTorrent, testLeecher)) - - panicOnErr(conn.RemoveLeecher(testTorrent, testLeecher)) - foundTorrent, found, err := conn.FindTorrent(testTorrent.Infohash) - panicOnErr(err) - foundLeecher, found := foundTorrent.Leechers[storage.PeerMapKey(testLeecher)] - if found || foundLeecher == *testLeecher { - t.Error("leecher not removed from cache", foundLeecher, *testLeecher) - } - foundLeecher, found = testTorrent.Leechers[storage.PeerMapKey(testLeecher)] - if found || foundLeecher == *testLeecher { - t.Error("leecher not removed from local", foundLeecher, *testLeecher) - } - // Cleanup - panicOnErr(conn.RemoveTorrent(testTorrent)) -} - -func TestSetSeeder(t *testing.T) { - conn := createTestConn() - testTorrent := createTestTorrent() - panicOnErr(conn.AddTorrent(testTorrent)) - testSeeder := createTestPeer(createTestUserID(), testTorrent.ID) - panicOnErr(conn.AddSeeder(testTorrent, testSeeder)) - - r := rand.New(rand.NewSource(time.Now().UnixNano())) - testSeeder.Uploaded += uint64(r.Int63()) - - panicOnErr(conn.SetSeeder(testTorrent, testSeeder)) - - foundTorrent, _, err := conn.FindTorrent(testTorrent.Infohash) - panicOnErr(err) - foundSeeder, _ := foundTorrent.Seeders[storage.PeerMapKey(testSeeder)] - if foundSeeder != *testSeeder { - t.Error("seeder not updated in cache", foundSeeder, *testSeeder) - } - foundSeeder, _ = testTorrent.Seeders[storage.PeerMapKey(testSeeder)] - if foundSeeder != *testSeeder { - t.Error("seeder not updated in local", foundSeeder, *testSeeder) - } - // Cleanup - panicOnErr(conn.RemoveTorrent(testTorrent)) -} - -func TestSetLeecher(t *testing.T) { - conn := createTestConn() - testTorrent := createTestTorrent() - panicOnErr(conn.AddTorrent(testTorrent)) - testLeecher := createTestPeer(createTestUserID(), testTorrent.ID) - panicOnErr(conn.AddLeecher(testTorrent, testLeecher)) - - r := rand.New(rand.NewSource(time.Now().UnixNano())) - testLeecher.Uploaded += uint64(r.Int63()) - - panicOnErr(conn.SetLeecher(testTorrent, testLeecher)) - foundTorrent, _, err := conn.FindTorrent(testTorrent.Infohash) - panicOnErr(err) - foundLeecher, _ := foundTorrent.Leechers[storage.PeerMapKey(testLeecher)] - if foundLeecher != *testLeecher { - t.Error("leecher not updated in cache", testLeecher) - } - foundLeecher, _ = testTorrent.Leechers[storage.PeerMapKey(testLeecher)] - if foundLeecher != *testLeecher { - t.Error("leecher not updated in local", testLeecher) - } - // Cleanup - panicOnErr(conn.RemoveTorrent(testTorrent)) -} - -func TestIncrementSlots(t *testing.T) { - conn := createTestConn() - testUser := createTestUser() - panicOnErr(conn.AddUser(testUser)) - numSlots := testUser.Slots - - panicOnErr(conn.IncrementSlots(testUser)) - foundUser, _, err := conn.FindUser(testUser.Passkey) - panicOnErr(err) - - if foundUser.Slots != numSlots+1 { - t.Error("cached slots not incremented") - } - if testUser.Slots != numSlots+1 { - t.Error("local slots not incremented") - } - // Cleanup - panicOnErr(conn.RemoveUser(testUser)) -} - -func TestDecrementSlots(t *testing.T) { - conn := createTestConn() - testUser := createTestUser() - panicOnErr(conn.AddUser(testUser)) - numSlots := testUser.Slots - - panicOnErr(conn.DecrementSlots(testUser)) - foundUser, _, err := conn.FindUser(testUser.Passkey) - panicOnErr(err) - - if foundUser.Slots != numSlots-1 { - t.Error("cached slots not incremented") - } - if testUser.Slots != numSlots-1 { - t.Error("local slots not incremented") - } - // Cleanup - panicOnErr(conn.RemoveUser(testUser)) -} - -func TestLeecherFinished(t *testing.T) { - conn := createTestConn() - testTorrent := createTestTorrent() - panicOnErr(conn.AddTorrent(testTorrent)) - testLeecher := createTestPeer(createTestUserID(), testTorrent.ID) - panicOnErr(conn.AddLeecher(testTorrent, testLeecher)) - testLeecher.Left = 0 - - panicOnErr(conn.LeecherFinished(testTorrent, testLeecher)) - - foundTorrent, _, err := conn.FindTorrent(testTorrent.Infohash) - panicOnErr(err) - foundSeeder, _ := foundTorrent.Seeders[storage.PeerMapKey(testLeecher)] - if foundSeeder != *testLeecher { - t.Error("seeder not added to cache", foundSeeder, *testLeecher) - } - foundSeeder, _ = foundTorrent.Leechers[storage.PeerMapKey(testLeecher)] - if foundSeeder == *testLeecher { - t.Error("leecher not removed from cache", testLeecher) - } - foundSeeder, _ = testTorrent.Seeders[storage.PeerMapKey(testLeecher)] - if foundSeeder != *testLeecher { - t.Error("seeder not added to local", testLeecher) - } - foundSeeder, _ = testTorrent.Leechers[storage.PeerMapKey(testLeecher)] - if foundSeeder == *testLeecher { - t.Error("leecher not removed from local", testLeecher) - } - // Cleanup - panicOnErr(conn.RemoveTorrent(testTorrent)) -} - -// Add, update, verify remove -func TestUpdatePeer(t *testing.T) { - conn := createTestConn() - testTorrent := createTestTorrent() - testSeeder := createTestPeer(createTestUserID(), testTorrent.ID) - panicOnErr(conn.AddTorrent(testTorrent)) - panicOnErr(conn.AddSeeder(testTorrent, testSeeder)) - // Update a seeder, set it, then check to make sure it updated - r := rand.New(rand.NewSource(time.Now().UnixNano())) - testSeeder.Uploaded += uint64(r.Int63()) - - panicOnErr(conn.SetSeeder(testTorrent, testSeeder)) - - panicOnErr(conn.RemoveSeeder(testTorrent, testSeeder)) - foundTorrent, _, err := conn.FindTorrent(testTorrent.Infohash) - panicOnErr(err) - if seeder, exists := foundTorrent.Seeders[storage.PeerMapKey(testSeeder)]; exists { - t.Error("seeder not removed from cache", seeder) - } - if seeder, exists := testTorrent.Seeders[storage.PeerMapKey(testSeeder)]; exists { - t.Error("seeder not removed from local", seeder) - } - // Cleanup - panicOnErr(conn.RemoveTorrent(testTorrent)) -} - -func TestParallelFindUser(t *testing.T) { - t.Parallel() - if testing.Short() { - t.Skip() - } - conn := createTestConn() - testUserSuccess := createTestUser() - testUserFail := createTestUser() - panicOnErr(conn.AddUser(testUserSuccess)) - - for i := 0; i < 10; i++ { - foundUser, found, err := conn.FindUser(testUserFail.Passkey) - panicOnErr(err) - if found { - t.Error("user found", foundUser) - } - foundUser, found, err = conn.FindUser(testUserSuccess.Passkey) - panicOnErr(err) - if !found { - t.Error("user not found", testUserSuccess) - } - if *foundUser != *testUserSuccess { - t.Error("found user mismatch", *foundUser, testUserSuccess) - } - } - // Cleanup - panicOnErr(conn.RemoveUser(testUserSuccess)) -} - -func TestParallelFindTorrent(t *testing.T) { - t.Parallel() - if testing.Short() { - t.Skip() - } - conn := createTestConn() - testTorrentSuccess := createTestTorrent() - testTorrentFail := createTestTorrent() - panicOnErr(conn.AddTorrent(testTorrentSuccess)) - - for i := 0; i < 10; i++ { - foundTorrent, found, err := conn.FindTorrent(testTorrentSuccess.Infohash) - panicOnErr(err) - if !found { - t.Error("torrent not found", testTorrentSuccess) - } - if !reflect.DeepEqual(foundTorrent, testTorrentSuccess) { - t.Error("found torrent mismatch", foundTorrent, testTorrentSuccess) - } - foundTorrent, found, err = conn.FindTorrent(testTorrentFail.Infohash) - panicOnErr(err) - if found { - t.Error("torrent found", foundTorrent) - } - } - // Cleanup - panicOnErr(conn.RemoveTorrent(testTorrentSuccess)) -} - -func TestParallelSetSeeder(t *testing.T) { - t.Parallel() - if testing.Short() { - t.Skip() - } - conn := createTestConn() - testTorrent := createTestTorrent() - panicOnErr(conn.AddTorrent(testTorrent)) - testSeeder := createTestPeer(createTestUserID(), testTorrent.ID) - panicOnErr(conn.AddSeeder(testTorrent, testSeeder)) - r := rand.New(rand.NewSource(time.Now().UnixNano())) - - for i := 0; i < 10; i++ { - testSeeder.Uploaded += uint64(r.Int63()) - - panicOnErr(conn.SetSeeder(testTorrent, testSeeder)) - - foundTorrent, _, err := conn.FindTorrent(testTorrent.Infohash) - panicOnErr(err) - foundSeeder, _ := foundTorrent.Seeders[storage.PeerMapKey(testSeeder)] - if foundSeeder != *testSeeder { - t.Error("seeder not updated in cache", foundSeeder, *testSeeder) - } - foundSeeder, _ = testTorrent.Seeders[storage.PeerMapKey(testSeeder)] - if foundSeeder != *testSeeder { - t.Error("seeder not updated in local", foundSeeder, *testSeeder) - } - } - // Cleanup - panicOnErr(conn.RemoveTorrent(testTorrent)) -} - -func TestParallelAddLeecher(t *testing.T) { - t.Parallel() - if testing.Short() { - t.Skip() - } - conn := createTestConn() - testTorrent := createTestTorrent() - panicOnErr(conn.AddTorrent(testTorrent)) - - for i := 0; i < 10; i++ { - testLeecher := createTestPeer(createTestUserID(), testTorrent.ID) - - panicOnErr(conn.AddLeecher(testTorrent, testLeecher)) - - foundTorrent, found, err := conn.FindTorrent(testTorrent.Infohash) - panicOnErr(err) - foundLeecher, found := foundTorrent.Leechers[storage.PeerMapKey(testLeecher)] - if found && foundLeecher != *testLeecher { - t.Error("leecher not added to cache", testLeecher) - } - foundLeecher, found = testTorrent.Leechers[storage.PeerMapKey(testLeecher)] - if found && foundLeecher != *testLeecher { - t.Error("leecher not added to local", testLeecher) - } - } - // Cleanup - panicOnErr(conn.RemoveTorrent(testTorrent)) -} diff --git a/storage/tracker/redis/redis.go b/storage/tracker/redis/redis.go deleted file mode 100644 index 04f7754..0000000 --- a/storage/tracker/redis/redis.go +++ /dev/null @@ -1,689 +0,0 @@ -// Copyright 2013 The Chihaya Authors. All rights reserved. -// Use of this source code is governed by the BSD 2-Clause license, -// which can be found in the LICENSE file. - -// Package redis implements the storage interface for a BitTorrent tracker. -// -// This interface is configured by a config.DataStore. -// To get a handle to this interface, call New on the initialized driver and -// then Get() on returned the tracker.Pool. -// -// Torrents, Users, and Peers are all stored in Redis hash types. All Redis -// keys can have an optional prefix specified during configuration. -// The relationship between Torrents and Peers is a Redis set that holds -// the peers' keys. There are two sets per torrent, one for seeders and -// one for leechers. The Redis sets are keyed by type and the torrent's ID. -// -// The whitelist is a Redis set with the key "whitelist" that holds client IDs. -// Operations on the whitelist do not parse the client ID from a peer ID. -// -// Some functions in this interface are not atomic. The data being modified may -// change while the function is executing. This will not cause the function to -// return an error; instead the function will complete and return valid, stale -// data. -package redis - -import ( - "errors" - "strconv" - "time" - - "github.com/garyburd/redigo/redis" - - "github.com/chihaya/chihaya/config" - "github.com/chihaya/chihaya/storage" - "github.com/chihaya/chihaya/storage/tracker" -) - -var ( - ErrCreateUser = errors.New("redis: Incorrect reply length for user") - ErrCreateTorrent = errors.New("redis: Incorrect reply length for torrent") - ErrCreatePeer = errors.New("redis: Incorrect reply length for peer") - ErrMarkActive = errors.New("redis: Torrent doesn't exist") - - SeedersPrefix = "seeders:" - LeechersPrefix = "leechers:" - TorrentPrefix = "torrent:" - UserPrefix = "user:" - PeerPrefix = "peer:" -) - -type driver struct{} - -// New creates and returns a tracker.Pool. -func (d *driver) New(conf *config.DataStore) tracker.Pool { - return &Pool{ - conf: conf, - pool: redis.Pool{ - MaxIdle: conf.MaxIdleConns, - IdleTimeout: conf.IdleTimeout.Duration, - Dial: makeDialFunc(conf), - TestOnBorrow: testOnBorrow, - }, - } -} - -// makeDialFunc configures and returns a new redis.Dial struct using the specified configuration. -func makeDialFunc(conf *config.DataStore) func() (redis.Conn, error) { - return func() (conn redis.Conn, err error) { - conn, err = redis.Dial(conf.Network, conf.Host+":"+conf.Port) - if err != nil { - return nil, err - } - return conn, nil - } -} - -// testOnBorrow pings the Redis instance -func testOnBorrow(c redis.Conn, t time.Time) error { - _, err := c.Do("PING") - return err -} - -type Pool struct { - conf *config.DataStore - pool redis.Pool -} - -func (p *Pool) Close() error { - return p.pool.Close() -} - -func (p *Pool) Get() (tracker.Conn, error) { - newConn := &Conn{ - conf: p.conf, - done: false, - Conn: p.pool.Get(), - } - return newConn, nil -} - -type Conn struct { - conf *config.DataStore - done bool - redis.Conn -} - -func (conn *Conn) close() { - if conn.done { - panic("redis: connection closed twice") - } - conn.done = true - conn.Conn.Close() -} - -// createUser takes a string slice of length 14 and returns a pointer to a new -// storage.User or an error. -// This function is used to create a user from a Redis hash response(HGETALL). -// The order of strings the in the slice must follow the pattern: -// [<field name>, <field value>, <field name>, <field value>, ...] -// If the field value string cannot be converted to the correct type, -// createUser will return a nil user and the conversion error. -func createUser(userVals []string) (*storage.User, error) { - if len(userVals) != 14 { - return nil, ErrCreateUser - } - var user storage.User - var err error - for index, userString := range userVals { - switch userString { - case "id": - user.ID, err = strconv.ParseUint(userVals[index+1], 10, 64) - case "passkey": - user.Passkey = userVals[index+1] - case "up_multiplier": - user.UpMultiplier, err = strconv.ParseFloat(userVals[index+1], 64) - case "down_multiplier": - user.DownMultiplier, err = strconv.ParseFloat(userVals[index+1], 64) - case "slots": - user.Slots, err = strconv.ParseInt(userVals[index+1], 10, 64) - case "slots_used": - user.SlotsUsed, err = strconv.ParseInt(userVals[index+1], 10, 64) - case "snatches": - user.Snatches, err = strconv.ParseUint(userVals[index+1], 10, 64) - } - if err != nil { - return nil, err - } - } - return &user, nil -} - -// createTorrent takes a string slice of length 14 and returns a pointer to a new storage.Torrent -// or an error. -// This function can be used to create a torrent from a Redis hash response(HGETALL). -// The order of strings the in the slice must follow the pattern: -// [<field name>, <field value>, <field name>, <field value>, ...] -// This function calls multiple redis commands, it's not internally atomic. -// If the field values cannot be converted to the correct type, -// createTorrent will return a nil user and the conversion error. -// After converting the torrent fields, the seeders and leechers are populated by redis.getPeers -func (conn *Conn) createTorrent(torrentVals []string) (*storage.Torrent, error) { - if len(torrentVals) != 14 { - return nil, ErrCreateTorrent - } - var torrent storage.Torrent - var err error - for index, torrentString := range torrentVals { - switch torrentString { - case "id": - torrent.ID, err = strconv.ParseUint(torrentVals[index+1], 10, 64) - case "infohash": - torrent.Infohash = torrentVals[index+1] - case "active": - torrent.Active, err = strconv.ParseBool(torrentVals[index+1]) - case "snatches": - torrent.Snatches, err = strconv.ParseUint(torrentVals[index+1], 10, 32) - case "up_multiplier": - torrent.UpMultiplier, err = strconv.ParseFloat(torrentVals[index+1], 64) - case "down_multiplier": - torrent.DownMultiplier, err = strconv.ParseFloat(torrentVals[index+1], 64) - case "last_action": - torrent.LastAction, err = strconv.ParseInt(torrentVals[index+1], 10, 64) - } - if err != nil { - return nil, err - } - } - torrent.Seeders, err = conn.getPeers(torrent.ID, SeedersPrefix) - if err != nil { - return nil, err - } - torrent.Leechers, err = conn.getPeers(torrent.ID, LeechersPrefix) - if err != nil { - return nil, err - } - return &torrent, nil -} - -// setPeer writes or overwrites peer information, stored as a Redis hash. -// The hash fields names are the same as the JSON tags on the storage.Peer struct. -func (conn *Conn) setPeer(peer *storage.Peer) error { - hashKey := conn.conf.Prefix + getPeerHashKey(peer) - _, err := conn.Do("HMSET", hashKey, - "id", peer.ID, - "user_id", peer.UserID, - "torrent_id", peer.TorrentID, - "ip", peer.IP, - "port", peer.Port, - "uploaded", peer.Uploaded, - "downloaded", peer.Downloaded, - "left", peer.Left, - "last_announce", peer.LastAnnounce) - - return err -} - -// removePeer removes the given peer from the specified peer set (seeder or leecher), -// and removes the peer information. -// This function calls multiple redis commands, it's not internally atomic. -// This function will not return an error if the peer to remove doesn't exist. -func (conn *Conn) removePeer(peer *storage.Peer, peerTypePrefix string) error { - setKey := conn.conf.Prefix + getPeerSetKey(peerTypePrefix, peer) - _, err := conn.Do("SREM", setKey, getPeerHashKey(peer)) - if err != nil { - return err - } - hashKey := conn.conf.Prefix + getPeerHashKey(peer) - _, err = conn.Do("DEL", hashKey) - return nil -} - -// removePeers removes all peers from specified peer set (seeders or leechers), -// removes the peer information, and then removes the associated peer from the given map. -// This function will not return an error if the peer to remove doesn't exist. -// This function will only delete the peer set if all the individual peer deletions were successful -// This function calls multiple redis commands, it's not internally atomic. -func (conn *Conn) removePeers(torrentID uint64, peers map[string]storage.Peer, peerTypePrefix string) error { - for _, peer := range peers { - hashKey := conn.conf.Prefix + getPeerHashKey(&peer) - _, err := conn.Do("DEL", hashKey) - if err != nil { - return err - } - delete(peers, storage.PeerMapKey(&peer)) - } - setKey := conn.conf.Prefix + peerTypePrefix + strconv.FormatUint(torrentID, 36) - _, err := conn.Do("DEL", setKey) - if err != nil { - return err - } - return nil -} - -// getPeerHashKey returns a string with the peer.ID, encoded peer.UserID, and encoded peer.TorrentID, -// concatenated and delimited by colons -// This key corresponds to a Redis hash type with fields containing a peer's data. -// The peer hashkey relies on the combination of peerID, userID, and torrentID being unique. -func getPeerHashKey(peer *storage.Peer) string { - return peer.ID + ":" + strconv.FormatUint(peer.UserID, 36) + ":" + strconv.FormatUint(peer.TorrentID, 36) -} - -// getPeerSetKey returns a string that is the peer's encoded torrentID appended to the typePrefix -// This key corresponds to a torrent's pool of leechers or seeders -func getPeerSetKey(typePrefix string, peer *storage.Peer) string { - return typePrefix + strconv.FormatUint(peer.TorrentID, 36) -} - -// addPeers adds each peer's key to the specified peer set and saves the peer's information. -// This function will not return an error if the peer already exists in the set. -// This function calls multiple redis commands, it's not internally atomic. -func (conn *Conn) addPeers(peers map[string]storage.Peer, peerTypePrefix string) error { - for _, peer := range peers { - setKey := conn.conf.Prefix + getPeerSetKey(peerTypePrefix, &peer) - _, err := conn.Do("SADD", setKey, getPeerHashKey(&peer)) - if err != nil { - return err - } - conn.setPeer(&peer) - } - return nil -} - -// createPeer takes a slice of length 9 and returns a pointer to a new storage.Peer or an error. -// This function is used to create a peer from a Redis hash response(HGETALL). -// The order of strings the in the slice must follow the pattern: -// [<field name>, <field value>, <field name>, <field value>, ...] -// If the field value string cannot be converted to the correct type, -// the function will return a nil peer and the conversion error. -func createPeer(peerVals []string) (*storage.Peer, error) { - if len(peerVals) != 18 { - return nil, ErrCreatePeer - } - var peer storage.Peer - var err error - for index, peerString := range peerVals { - switch peerString { - case "id": - peer.ID = peerVals[index+1] - case "user_id": - peer.UserID, err = strconv.ParseUint(peerVals[index+1], 10, 64) - case "torrent_id": - peer.TorrentID, err = strconv.ParseUint(peerVals[index+1], 10, 64) - case "ip": - peer.IP = peerVals[index+1] - case "port": - peer.Port, err = strconv.ParseUint(peerVals[index+1], 10, 64) - case "uploaded": - peer.Uploaded, err = strconv.ParseUint(peerVals[index+1], 10, 64) - case "downloaded": - peer.Downloaded, err = strconv.ParseUint(peerVals[index+1], 10, 64) - case "left": - peer.Left, err = strconv.ParseUint(peerVals[index+1], 10, 64) - case "last_announce": - peer.LastAnnounce, err = strconv.ParseInt(peerVals[index+1], 10, 64) - } - if err != nil { - return nil, err - } - } - return &peer, nil -} - -// getPeers returns a map of peers from a specified torrent's peer set(seeders or leechers). -// This is a multiple action command, it's not internally atomic. -func (conn *Conn) getPeers(torrentID uint64, peerTypePrefix string) (peers map[string]storage.Peer, err error) { - peers = make(map[string]storage.Peer) - setKey := conn.conf.Prefix + peerTypePrefix + strconv.FormatUint(torrentID, 36) - peerStrings, err := redis.Strings(conn.Do("SMEMBERS", setKey)) - if err != nil { - return nil, err - } - // Keys map to peer objects stored in hashes - for _, peerHashKey := range peerStrings { - hashKey := conn.conf.Prefix + peerHashKey - peerVals, err := redis.Strings(conn.Do("HGETALL", hashKey)) - if err != nil { - return nil, err - } - if len(peerVals) == 0 { - continue - } - peer, err := createPeer(peerVals) - if err != nil { - return nil, err - } - peers[storage.PeerMapKey(peer)] = *peer - } - return -} - -// AddTorrent writes/overwrites torrent information and saves peers from both peer sets. -// The hash fields names are the same as the JSON tags on the storage.Torrent struct. -// This is a multiple action command, it's not internally atomic. -func (conn *Conn) AddTorrent(t *storage.Torrent) error { - hashkey := conn.conf.Prefix + TorrentPrefix + t.Infohash - _, err := conn.Do("HMSET", hashkey, - "id", t.ID, - "infohash", t.Infohash, - "active", t.Active, - "snatches", t.Snatches, - "up_multiplier", t.UpMultiplier, - "down_multiplier", t.DownMultiplier, - "last_action", t.LastAction) - if err != nil { - return err - } - - err = conn.addPeers(t.Seeders, SeedersPrefix) - if err != nil { - return err - } - err = conn.addPeers(t.Leechers, LeechersPrefix) - if err != nil { - return err - } - return nil -} - -// RemoveTorrent deletes the torrent's Redis hash and then deletes all peers. -// This function will not return an error if the torrent has already been removed. -// This is a multiple action command, it's not internally atomic. -func (conn *Conn) RemoveTorrent(t *storage.Torrent) error { - hashkey := conn.conf.Prefix + TorrentPrefix + t.Infohash - _, err := conn.Do("DEL", hashkey) - if err != nil { - return err - } - // Remove seeders and leechers as well - err = conn.removePeers(t.ID, t.Seeders, SeedersPrefix) - if err != nil { - return err - } - err = conn.removePeers(t.ID, t.Leechers, LeechersPrefix) - if err != nil { - return err - } - return nil -} - -// AddUser writes/overwrites user information to a Redis hash. -// The hash fields names are the same as the JSON tags on the storage.user struct. -func (conn *Conn) AddUser(u *storage.User) error { - hashkey := conn.conf.Prefix + UserPrefix + u.Passkey - _, err := conn.Do("HMSET", hashkey, - "id", u.ID, - "passkey", u.Passkey, - "up_multiplier", u.UpMultiplier, - "down_multiplier", u.DownMultiplier, - "slots", u.Slots, - "slots_used", u.SlotsUsed, - "snatches", u.Snatches) - if err != nil { - return err - } - return nil -} - -// RemoveUser removes the user's hash from Redis. -// This function does not return an error if the user doesn't exist. -func (conn *Conn) RemoveUser(u *storage.User) error { - hashkey := conn.conf.Prefix + UserPrefix + u.Passkey - _, err := conn.Do("DEL", hashkey) - if err != nil { - return err - } - return nil -} - -// FindUser returns a pointer to a new user struct and true if the user exists, -// or nil and false if the user doesn't exist. -// This function does not return an error if the torrent doesn't exist. -func (conn *Conn) FindUser(passkey string) (*storage.User, bool, error) { - hashkey := conn.conf.Prefix + UserPrefix + passkey - // Consider using HGETALL instead of HVALS here for robustness - userStrings, err := redis.Strings(conn.Do("HGETALL", hashkey)) - if err != nil { - return nil, false, err - } else if len(userStrings) == 0 { - return nil, false, nil - } - foundUser, err := createUser(userStrings) - if err != nil { - return nil, false, err - } - return foundUser, true, nil -} - -// FindTorrent returns a pointer to a new torrent struct and true if the torrent exists, -// or nil and false if the torrent doesn't exist. -// This is a multiple action command, it's not internally atomic. -func (conn *Conn) FindTorrent(infohash string) (*storage.Torrent, bool, error) { - hashkey := conn.conf.Prefix + TorrentPrefix + infohash - torrentStrings, err := redis.Strings(conn.Do("HGETALL", hashkey)) - if err != nil { - return nil, false, err - } else if len(torrentStrings) == 0 { - return nil, false, nil - } - - foundTorrent, err := conn.createTorrent(torrentStrings) - if err != nil { - return nil, false, err - } - return foundTorrent, true, nil -} - -// ClientWhitelisted returns true if the ClientID exists in the Client set. -// This function does not parse the client ID from the peer ID. -// The clientID must match exactly to a member of the set. -func (conn *Conn) ClientWhitelisted(peerID string) (exists bool, err error) { - key := conn.conf.Prefix + "whitelist" - return redis.Bool(conn.Do("SISMEMBER", key, peerID)) -} - -// WhitelistClient adds a client ID to the client whitelist set. -// This function does not return an error if the client ID is already in the set. -func (conn *Conn) WhitelistClient(peerID string) error { - key := conn.conf.Prefix + "whitelist" - _, err := conn.Do("SADD", key, peerID) - return err -} - -// UnWhitelistClient removes a client ID from the client whitelist set -// This function does not return an error if the client ID is not in the set. -func (conn *Conn) UnWhitelistClient(peerID string) error { - key := conn.conf.Prefix + "whitelist" - _, err := conn.Do("SREM", key, peerID) - return err -} - -// RecordSnatch increments the snatch counter on the torrent and user by one. -// This modifies the arguments as well as the hash field in Redis. -// This is a multiple action command, it's not internally atomic. -func (conn *Conn) RecordSnatch(user *storage.User, torrent *storage.Torrent) error { - - torrentKey := conn.conf.Prefix + TorrentPrefix + torrent.Infohash - snatchCount, err := redis.Int(conn.Do("HINCRBY", torrentKey, "snatches", 1)) - if err != nil { - return err - } - torrent.Snatches = uint64(snatchCount) - - userKey := conn.conf.Prefix + UserPrefix + user.Passkey - snatchCount, err = redis.Int(conn.Do("HINCRBY", userKey, "snatches", 1)) - if err != nil { - return err - } - user.Snatches = uint64(snatchCount) - return nil -} - -// MarkActive sets the active field of the torrent to true. -// This modifies the argument as well as the hash field in Redis. -// This function will return ErrMarkActive if the torrent does not exist. -func (conn *Conn) MarkActive(torrent *storage.Torrent) error { - hashkey := conn.conf.Prefix + TorrentPrefix + torrent.Infohash - activeExists, err := redis.Int(conn.Do("HSET", hashkey, "active", true)) - if err != nil { - return err - } - torrent.Active = true - // HSET returns 1 if hash didn't exist before - if activeExists == 1 { - return ErrMarkActive - } - return nil -} - -// MarkInactive sets the active field of the torrent to false. -// This modifies the argument as well as the hash field in Redis. -// This function will return ErrMarkActive if the torrent does not exist. -func (conn *Conn) MarkInactive(torrent *storage.Torrent) error { - hashkey := conn.conf.Prefix + TorrentPrefix + torrent.Infohash - activeExists, err := redis.Int(conn.Do("HSET", hashkey, "active", false)) - if err != nil { - return err - } - torrent.Active = false - // HSET returns 1 if hash didn't exist before - if activeExists == 1 { - // Clean-up incomplete torrent - _, err = conn.Do("DEL", hashkey) - if err != nil { - return err - } - return ErrMarkActive - } - return nil -} - -// AddLeecher adds a new peer to a torrent's leecher set. -// This modifies the torrent argument, as well as the torrent's set and peer's hash in Redis. -// This function does not return an error if the leecher already exists. -// This is a multiple action command, it's not internally atomic. -func (conn *Conn) AddLeecher(torrent *storage.Torrent, peer *storage.Peer) error { - setKey := conn.conf.Prefix + LeechersPrefix + strconv.FormatUint(torrent.ID, 36) - _, err := conn.Do("SADD", setKey, getPeerHashKey(peer)) - if err != nil { - return err - } - err = conn.setPeer(peer) - if err != nil { - return err - } - if torrent.Leechers == nil { - torrent.Leechers = make(map[string]storage.Peer) - } - torrent.Leechers[storage.PeerMapKey(peer)] = *peer - return nil -} - -// SetLeecher updates a torrent's leecher. -// This modifies the torrent argument, as well as the peer's hash in Redis. -// Setting assumes that the peer is already a leecher, and only needs to be updated. -// This function does not return an error if the leecher does not exist or is not in the torrent's leecher set. -func (conn *Conn) SetLeecher(t *storage.Torrent, p *storage.Peer) error { - err := conn.setPeer(p) - if err != nil { - return err - } - t.Leechers[storage.PeerMapKey(p)] = *p - return nil -} - -// RemoveLeecher removes the given peer from a torrent's leecher set. -// This modifies the torrent argument, as well as the torrent's set and peer's hash in Redis. -// This function does not return an error if the peer doesn't exist, or is not in the set. -func (conn *Conn) RemoveLeecher(t *storage.Torrent, p *storage.Peer) error { - err := conn.removePeer(p, LeechersPrefix) - if err != nil { - return err - } - delete(t.Leechers, storage.PeerMapKey(p)) - return nil -} - -// LeecherFinished moves a peer's hashkey from a torrent's leecher set to the seeder set and updates the peer. -// This modifies the torrent argument, as well as the torrent's set and peer's hash in Redis. -// This function does not return an error if the peer doesn't exist or is not in the torrent's leecher set. -func (conn *Conn) LeecherFinished(torrent *storage.Torrent, peer *storage.Peer) error { - torrentIdKey := strconv.FormatUint(torrent.ID, 36) - seederSetKey := conn.conf.Prefix + SeedersPrefix + torrentIdKey - leecherSetKey := conn.conf.Prefix + LeechersPrefix + torrentIdKey - - _, err := conn.Do("SMOVE", leecherSetKey, seederSetKey, getPeerHashKey(peer)) - if err != nil { - return err - } - torrent.Seeders[storage.PeerMapKey(peer)] = *peer - delete(torrent.Leechers, storage.PeerMapKey(peer)) - - err = conn.setPeer(peer) - return err -} - -// AddSeeder adds a new peer to a torrent's seeder set. -// This modifies the torrent argument, as well as the torrent's set and peer's hash in Redis. -// This function does not return an error if the seeder already exists. -// This is a multiple action command, it's not internally atomic. -func (conn *Conn) AddSeeder(torrent *storage.Torrent, peer *storage.Peer) error { - setKey := conn.conf.Prefix + SeedersPrefix + strconv.FormatUint(torrent.ID, 36) - _, err := conn.Do("SADD", setKey, getPeerHashKey(peer)) - if err != nil { - return err - } - err = conn.setPeer(peer) - if err != nil { - return err - } - if torrent.Seeders == nil { - torrent.Seeders = make(map[string]storage.Peer) - } - torrent.Seeders[storage.PeerMapKey(peer)] = *peer - return nil -} - -// SetSeeder updates a torrent's seeder. -// This modifies the torrent argument, as well as the peer's hash in Redis. -// Setting assumes that the peer is already a seeder, and only needs to be updated. -// This function does not return an error if the seeder does not exist or is not in the torrent's seeder set. -func (conn *Conn) SetSeeder(t *storage.Torrent, p *storage.Peer) error { - err := conn.setPeer(p) - if err != nil { - return err - } - t.Seeders[storage.PeerMapKey(p)] = *p - return nil -} - -// RemoveSeeder removes the given peer from a torrent's seeder set. -// This modifies the torrent argument, as well as the torrent's set and peer's hash in Redis. -// This function does not return an error if the peer doesn't exist, or is not in the set. -func (conn *Conn) RemoveSeeder(t *storage.Torrent, p *storage.Peer) error { - err := conn.removePeer(p, SeedersPrefix) - if err != nil { - return err - } - delete(t.Seeders, storage.PeerMapKey(p)) - return nil -} - -// IncrementSlots increment a user's Slots by one. -// This function modifies the argument as well as the hash field in Redis. -func (conn *Conn) IncrementSlots(u *storage.User) error { - hashkey := conn.conf.Prefix + UserPrefix + u.Passkey - slotCount, err := redis.Int(conn.Do("HINCRBY", hashkey, "slots", 1)) - if err != nil { - return err - } - u.Slots = int64(slotCount) - return nil -} - -// IncrementSlots increment a user's Slots by one. -// This function modifies the argument as well as the hash field in Redis. -func (conn *Conn) DecrementSlots(u *storage.User) error { - hashkey := conn.conf.Prefix + UserPrefix + u.Passkey - slotCount, err := redis.Int(conn.Do("HINCRBY", hashkey, "slots", -1)) - if err != nil { - return err - } - u.Slots = int64(slotCount) - return nil -} - -// init registers the redis driver -func init() { - tracker.Register("redis", &driver{}) -} diff --git a/storage/tracker/redis/redis_bench_test.go b/storage/tracker/redis/redis_bench_test.go deleted file mode 100644 index e8634c6..0000000 --- a/storage/tracker/redis/redis_bench_test.go +++ /dev/null @@ -1,288 +0,0 @@ -// Copyright 2013 The Chihaya Authors. All rights reserved. -// Use of this source code is governed by the BSD 2-Clause license, -// which can be found in the LICENSE file. - -package redis - -import ( - "math/rand" - "testing" - "time" -) - -func BenchmarkSuccessfulFindUser(b *testing.B) { - b.StopTimer() - conn := createTestConn() - testUser := createTestUser() - panicOnErr(conn.AddUser(testUser)) - b.StartTimer() - - for bCount := 0; bCount < b.N; bCount++ { - - foundUser, found, err := conn.FindUser(testUser.Passkey) - panicOnErr(err) - if !found { - b.Error("user not found", testUser) - } - if *foundUser != *testUser { - b.Error("found user mismatch", *foundUser, testUser) - } - } - // Cleanup - b.StopTimer() - panicOnErr(conn.RemoveUser(testUser)) - b.StartTimer() -} - -func BenchmarkFailedFindUser(b *testing.B) { - b.StopTimer() - conn := createTestConn() - testUser := createTestUser() - b.StartTimer() - - for bCount := 0; bCount < b.N; bCount++ { - - _, found, err := conn.FindUser(testUser.Passkey) - panicOnErr(err) - if found { - b.Error("user not found", testUser) - } - } -} - -func BenchmarkSuccessfulFindTorrent(b *testing.B) { - b.StopTimer() - conn := createTestConn() - testTorrent := createTestTorrent() - - panicOnErr(conn.AddTorrent(testTorrent)) - b.StartTimer() - - for bCount := 0; bCount < b.N; bCount++ { - foundTorrent, found, err := conn.FindTorrent(testTorrent.Infohash) - panicOnErr(err) - if !found { - b.Error("torrent not found", testTorrent) - } - // Incomplete comparison as maps make struct not nativly comparable - if foundTorrent.Infohash != testTorrent.Infohash { - b.Error("found torrent mismatch", foundTorrent, testTorrent) - } - } - // Cleanup - b.StopTimer() - panicOnErr(conn.RemoveTorrent(testTorrent)) - b.StartTimer() -} - -func BenchmarkFailFindTorrent(b *testing.B) { - b.StopTimer() - conn := createTestConn() - testTorrent := createTestTorrent() - b.StartTimer() - - for bCount := 0; bCount < b.N; bCount++ { - foundTorrent, found, err := conn.FindTorrent(testTorrent.Infohash) - panicOnErr(err) - if found { - b.Error("torrent found", foundTorrent) - } - } -} - -func BenchmarkSuccessfulClientWhitelisted(b *testing.B) { - b.StopTimer() - conn := createTestConn() - testPeerID := "-lt0D30-" - panicOnErr(conn.WhitelistClient(testPeerID)) - b.StartTimer() - - for bCount := 0; bCount < b.N; bCount++ { - found, err := conn.ClientWhitelisted(testPeerID) - panicOnErr(err) - if !found { - b.Error("peerID not found", testPeerID) - } - } - // Cleanup - b.StopTimer() - panicOnErr(conn.UnWhitelistClient(testPeerID)) - b.StartTimer() -} - -func BenchmarkFailClientWhitelisted(b *testing.B) { - b.StopTimer() - conn := createTestConn() - testPeerID2 := "TIX0192" - b.StartTimer() - - for bCount := 0; bCount < b.N; bCount++ { - found, err := conn.ClientWhitelisted(testPeerID2) - panicOnErr(err) - if found { - b.Error("peerID found", testPeerID2) - } - } -} - -func BenchmarkRecordSnatch(b *testing.B) { - b.StopTimer() - conn := createTestConn() - testTorrent := createTestTorrent() - testUser := createTestUser() - panicOnErr(conn.AddTorrent(testTorrent)) - panicOnErr(conn.AddUser(testUser)) - b.StartTimer() - - for bCount := 0; bCount < b.N; bCount++ { - panicOnErr(conn.RecordSnatch(testUser, testTorrent)) - } - // Cleanup - b.StopTimer() - panicOnErr(conn.RemoveTorrent(testTorrent)) - panicOnErr(conn.RemoveUser(testUser)) - b.StartTimer() -} - -func BenchmarkMarkActive(b *testing.B) { - b.StopTimer() - conn := createTestConn() - testTorrent := createTestTorrent() - testTorrent.Active = false - panicOnErr(conn.AddTorrent(testTorrent)) - b.StartTimer() - - for bCount := 0; bCount < b.N; bCount++ { - panicOnErr(conn.MarkActive(testTorrent)) - } - // Cleanup - b.StopTimer() - panicOnErr(conn.RemoveTorrent(testTorrent)) - b.StartTimer() -} - -func BenchmarkAddSeeder(b *testing.B) { - b.StopTimer() - conn := createTestConn() - testTorrent := createTestTorrent() - panicOnErr(conn.AddTorrent(testTorrent)) - b.StartTimer() - - for bCount := 0; bCount < b.N; bCount++ { - b.StopTimer() - testSeeder := createTestPeer(createTestUserID(), testTorrent.ID) - b.StartTimer() - - panicOnErr(conn.AddSeeder(testTorrent, testSeeder)) - } - // Cleanup - b.StopTimer() - panicOnErr(conn.RemoveTorrent(testTorrent)) - b.StartTimer() -} - -func BenchmarkRemoveSeeder(b *testing.B) { - b.StopTimer() - conn := createTestConn() - testTorrent := createTestTorrent() - panicOnErr(conn.AddTorrent(testTorrent)) - testSeeder := createTestPeer(createTestUserID(), testTorrent.ID) - b.StartTimer() - - for bCount := 0; bCount < b.N; bCount++ { - b.StopTimer() - conn.AddSeeder(testTorrent, testSeeder) - b.StartTimer() - - panicOnErr(conn.RemoveSeeder(testTorrent, testSeeder)) - } - // Cleanup - b.StopTimer() - panicOnErr(conn.RemoveTorrent(testTorrent)) - b.StartTimer() -} - -func BenchmarkSetSeeder(b *testing.B) { - b.StopTimer() - conn := createTestConn() - testTorrent := createTestTorrent() - panicOnErr(conn.AddTorrent(testTorrent)) - testSeeder := createTestPeer(createTestUserID(), testTorrent.ID) - panicOnErr(conn.AddSeeder(testTorrent, testSeeder)) - r := rand.New(rand.NewSource(time.Now().UnixNano())) - b.StartTimer() - - for bCount := 0; bCount < b.N; bCount++ { - b.StopTimer() - testSeeder.Uploaded += uint64(r.Int63()) - b.StartTimer() - - conn.SetSeeder(testTorrent, testSeeder) - } - // Cleanup - b.StopTimer() - panicOnErr(conn.RemoveTorrent(testTorrent)) - b.StartTimer() -} - -func BenchmarkIncrementSlots(b *testing.B) { - b.StopTimer() - conn := createTestConn() - testUser := createTestUser() - panicOnErr(conn.AddUser(testUser)) - b.StartTimer() - - for bCount := 0; bCount < b.N; bCount++ { - panicOnErr(conn.IncrementSlots(testUser)) - } - // Cleanup - b.StopTimer() - panicOnErr(conn.RemoveUser(testUser)) - b.StartTimer() -} - -func BenchmarkLeecherFinished(b *testing.B) { - b.StopTimer() - conn := createTestConn() - testTorrent := createTestTorrent() - panicOnErr(conn.AddTorrent(testTorrent)) - b.StartTimer() - - for bCount := 0; bCount < b.N; bCount++ { - b.StopTimer() - testLeecher := createTestPeer(createTestUserID(), testTorrent.ID) - panicOnErr(conn.AddLeecher(testTorrent, testLeecher)) - testLeecher.Left = 0 - b.StartTimer() - - panicOnErr(conn.LeecherFinished(testTorrent, testLeecher)) - } - // Cleanup - b.StopTimer() - panicOnErr(conn.RemoveTorrent(testTorrent)) - b.StartTimer() -} - -// This is a comparision to the Leecher finished function -func BenchmarkRemoveLeecherAddSeeder(b *testing.B) { - b.StopTimer() - conn := createTestConn() - testTorrent := createTestTorrent() - panicOnErr(conn.AddTorrent(testTorrent)) - b.StartTimer() - - for bCount := 0; bCount < b.N; bCount++ { - b.StopTimer() - testLeecher := createTestPeer(createTestUserID(), testTorrent.ID) - panicOnErr(conn.AddLeecher(testTorrent, testLeecher)) - testLeecher.Left = 0 - b.StartTimer() - - panicOnErr(conn.RemoveLeecher(testTorrent, testLeecher)) - panicOnErr(conn.AddSeeder(testTorrent, testLeecher)) - } - // Cleanup - b.StopTimer() - conn.RemoveTorrent(testTorrent) - b.StartTimer() -} diff --git a/storage/tracker/redis/redis_test.go b/storage/tracker/redis/redis_test.go deleted file mode 100644 index 8849eed..0000000 --- a/storage/tracker/redis/redis_test.go +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright 2013 The Chihaya Authors. All rights reserved. -// Use of this source code is governed by the BSD 2-Clause license, -// which can be found in the LICENSE file. - -package redis - -import ( - "crypto/rand" - "fmt" - "io" - "os" - "strconv" - "testing" - - "github.com/garyburd/redigo/redis" - - "github.com/chihaya/chihaya/config" - "github.com/chihaya/chihaya/storage" -) - -var ( - testTorrentIDChannel chan uint64 - testUserIDChannel chan uint64 - testPeerIDChannel chan int -) - -func init() { - testTorrentIDChannel = make(chan uint64, 100) - testUserIDChannel = make(chan uint64, 100) - testPeerIDChannel = make(chan int, 100) - // Sync access to ID counter with buffered global channels - go func() { - for i := 0; ; i++ { - testTorrentIDChannel <- uint64(i) - } - }() - go func() { - for i := 0; ; i++ { - testUserIDChannel <- uint64(i) - } - }() - go func() { - for i := 0; ; i++ { - testPeerIDChannel <- i - } - }() -} - -func createTestTorrentID() uint64 { - return <-testTorrentIDChannel -} - -func createTestUserID() uint64 { - return <-testUserIDChannel -} - -func createTestPeerID() string { - return "-testPeerID-" + strconv.Itoa(<-testPeerIDChannel) -} - -func createTestInfohash() string { - uuid := make([]byte, 40) - n, err := io.ReadFull(rand.Reader, uuid) - if n != len(uuid) || err != nil { - panic(err) - } - return string(uuid) -} - -func createTestPasskey() string { - uuid := make([]byte, 40) - n, err := io.ReadFull(rand.Reader, uuid) - if n != len(uuid) || err != nil { - panic(err) - } - return string(uuid) -} - -func panicOnErr(err error) { - if err != nil { - fmt.Println(err) - panic(err) - } -} - -func createTestRedisConn() *Conn { - testConfig, err := config.Open(os.Getenv("TESTCONFIGPATH")) - conf := &testConfig.Cache - panicOnErr(err) - - testPool := &Pool{ - conf: conf, - pool: redis.Pool{ - MaxIdle: conf.MaxIdleConns, - IdleTimeout: conf.IdleTimeout.Duration, - Dial: makeDialFunc(conf), - TestOnBorrow: testOnBorrow, - }, - } - - newConn := &Conn{ - conf: testPool.conf, - done: false, - Conn: testPool.pool.Get(), - } - panicOnErr(err) - - // Test connection before returning - _, err = newConn.Do("PING") - panicOnErr(err) - return newConn -} - -func createTestUser() *storage.User { - return &storage.User{ID: createTestUserID(), Passkey: createTestPasskey(), - UpMultiplier: 1.01, DownMultiplier: 1.0, Slots: 4, SlotsUsed: 2, Snatches: 7} -} - -func createTestPeer(userID uint64, torrentID uint64) *storage.Peer { - - return &storage.Peer{ID: createTestPeerID(), UserID: userID, TorrentID: torrentID, - IP: "127.0.0.1", Port: 6889, Uploaded: 1024, Downloaded: 3000, Left: 4200, LastAnnounce: 11} -} - -func createTestPeers(torrentID uint64, num int) map[string]storage.Peer { - testPeers := make(map[string]storage.Peer) - for i := 0; i < num; i++ { - tempPeer := createTestPeer(createTestUserID(), torrentID) - testPeers[storage.PeerMapKey(tempPeer)] = *tempPeer - } - return testPeers -} - -func createTestTorrent() *storage.Torrent { - - torrentInfohash := createTestInfohash() - torrentID := createTestTorrentID() - - testSeeders := createTestPeers(torrentID, 4) - testLeechers := createTestPeers(torrentID, 2) - - testTorrent := storage.Torrent{ID: torrentID, Infohash: torrentInfohash, Active: true, - Seeders: testSeeders, Leechers: testLeechers, Snatches: 11, UpMultiplier: 1.0, DownMultiplier: 1.0, LastAction: 0} - return &testTorrent -} - -func TestValidPeers(t *testing.T) { - testConn := createTestRedisConn() - testTorrentID := createTestTorrentID() - testPeers := createTestPeers(testTorrentID, 3) - - panicOnErr(testConn.addPeers(testPeers, "test:")) - peerMap, err := testConn.getPeers(testTorrentID, "test:") - panicOnErr(err) - if len(peerMap) != len(testPeers) { - t.Error("Num Peers not equal ", len(peerMap), len(testPeers)) - } - panicOnErr(testConn.removePeers(testTorrentID, testPeers, "test:")) -} - -func TestInvalidPeers(t *testing.T) { - testConn := createTestRedisConn() - testTorrentID := createTestTorrentID() - testPeers := createTestPeers(testTorrentID, 3) - tempPeer := createTestPeer(createTestUserID(), testTorrentID) - testPeers[storage.PeerMapKey(tempPeer)] = *tempPeer - - panicOnErr(testConn.addPeers(testPeers, "test:")) - // Imitate a peer being removed during get - hashKey := testConn.conf.Prefix + getPeerHashKey(tempPeer) - _, err := testConn.Do("DEL", hashKey) - panicOnErr(err) - - peerMap, err := testConn.getPeers(testTorrentID, "test:") - panicOnErr(err) - // Expect 1 less peer due to delete - if len(peerMap) != len(testPeers)-1 { - t.Error("Num Peers not equal ", len(peerMap), len(testPeers)-1) - } - panicOnErr(testConn.removePeers(testTorrentID, testPeers, "test:")) - if len(testPeers) != 0 { - t.Errorf("All peers not removed, %d peers remain!", len(testPeers)) - } -}