LBRYCommunity/seeder
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge #36: Add support to filter nodes by node flags

Browse source 
c9679dc Add whitelist for service filter, use a map for cacheTime (Jonas Schnelli)
5c8b9e3 IP-Query: Use a vector instead a set and random pick an IP (Jonas Schnelli)
839db15 Add support to filter nodes by node flags (Jonas Schnelli)
This commit is contained in:
Pieter Wuille 2016-06-07 17:54:49 +02:00
commit ab0ac73e5f
No known key found for this signature in database
GPG key ID: DBA1A67379A1A931
5 changed files with 68 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
db.cpp
View file

@ -166,7 +166,7 @@ void CAddrDb::Add_(const CAddress &addr, bool force) {
nDirty++; nDirty++;
} }
void CAddrDb::GetIPs_(set<CNetAddr>& ips, int max, const bool* nets) { void CAddrDb::GetIPs_(set<CNetAddr>& ips, uint64_t requestedFlags, int max, const bool* nets) {
if (goodId.size() == 0) { if (goodId.size() == 0) {
int id = -1; int id = -1;
if (ourId.size() == 0) { if (ourId.size() == 0) {
@ -175,23 +175,28 @@ void CAddrDb::GetIPs_(set<CNetAddr>& ips, int max, const bool* nets) {
} else { } else {
id = *ourId.begin(); id = *ourId.begin();
} }
if (id >= 0) { if (id >= 0 && idToInfo[id].services & requestedFlags) {
ips.insert(idToInfo[id].ip); ips.insert(idToInfo[id].ip);
} }
return; return;
} }
if (max > goodId.size() / 2) std::vector<int> goodIdFiltered;
max = goodId.size() / 2; for (std::set<int>::const_iterator it = goodId.begin(); it != goodId.end(); it++) {
if ((idToInfo[*it].services & requestedFlags) == requestedFlags)
goodIdFiltered.push_back(*it);
}
if (!goodIdFiltered.size())
return;
if (max > goodIdFiltered.size() / 2)
max = goodIdFiltered.size() / 2;
if (max < 1) if (max < 1)
max = 1; max = 1;
int low = *goodId.begin();
int high = *goodId.rbegin();
set<int> ids; set<int> ids;
while (ids.size() < max) { while (ids.size() < max) {
int range = high-low+1; ids.insert(goodIdFiltered[rand() % goodIdFiltered.size()]);
int pos = low + (rand() % range);
int id = *(goodId.lower_bound(pos));
ids.insert(id);
} }
for (set<int>::const_iterator it = ids.begin(); it != ids.end(); it++) { for (set<int>::const_iterator it = ids.begin(); it != ids.end(); it++) {
CService &ip = idToInfo[*it].ip; CService &ip = idToInfo[*it].ip;

6
db.h
View file

@ -219,7 +219,7 @@ protected:
void Bad_(const CService &ip, int ban); // mark an IP as bad (and optionally ban it) (must have been returned by Get_) void Bad_(const CService &ip, int ban); // mark an IP as bad (and optionally ban it) (must have been returned by Get_)
void Skipped_(const CService &ip); // mark an IP as skipped (must have been returned by Get_) void Skipped_(const CService &ip); // mark an IP as skipped (must have been returned by Get_)
int Lookup_(const CService &ip); // look up id of an IP int Lookup_(const CService &ip); // look up id of an IP
void GetIPs_(std::set<CNetAddr>& ips, int max, const bool *nets); // get a random set of IPs (shared lock only) void GetIPs_(std::set<CNetAddr>& ips, uint64_t requestedFlags, int max, const bool *nets); // get a random set of IPs (shared lock only)
public: public:
std::map<CService, time_t> banned; // nodes that are banned, with their unban time (a) std::map<CService, time_t> banned; // nodes that are banned, with their unban time (a)
@ -351,8 +351,8 @@ public:
} }
} }
} }
void GetIPs(std::set<CNetAddr>& ips, int max, const bool *nets) { void GetIPs(std::set<CNetAddr>& ips, uint64_t requestedFlags, int max, const bool *nets) {
SHARED_CRITICAL_BLOCK(cs) SHARED_CRITICAL_BLOCK(cs)
GetIPs_(ips, max, nets); GetIPs_(ips, requestedFlags, max, nets);
} }
}; };

2
dns.c
View file

@ -327,7 +327,7 @@ ssize_t static dnshandle(dns_opt_t *opt, const unsigned char *inbuf, size_t insi
// A/AAAA records // A/AAAA records
if ((typ == TYPE_A || typ == TYPE_AAAA || typ == QTYPE_ANY) && (cls == CLASS_IN || cls == QCLASS_ANY)) { if ((typ == TYPE_A || typ == TYPE_AAAA || typ == QTYPE_ANY) && (cls == CLASS_IN || cls == QCLASS_ANY)) {
addr_t addr[32]; addr_t addr[32];
int naddr = opt->cb((void*)opt, addr, 32, typ == TYPE_A || typ == QTYPE_ANY, typ == TYPE_AAAA || typ == QTYPE_ANY); int naddr = opt->cb((void*)opt, name, addr, 32, typ == TYPE_A || typ == QTYPE_ANY, typ == TYPE_AAAA || typ == QTYPE_ANY);
int n = 0; int n = 0;
while (n < naddr) { while (n < naddr) {
int ret = 1; int ret = 1;

2
dns.h
View file

@ -18,7 +18,7 @@ typedef struct {
const char *host; const char *host;
const char *ns; const char *ns;
const char *mbox; const char *mbox;
int (*cb)(void *opt, addr_t *addr, int max, int ipv4, int ipv6); int (*cb)(void *opt, char *requested_hostname, addr_t *addr, int max, int ipv4, int ipv6);
// stats // stats
uint64_t nRequests; uint64_t nRequests;
} dns_opt_t; } dns_opt_t;

72
main.cpp
View file

@ -15,6 +15,17 @@ using namespace std;
bool fTestNet = false; bool fTestNet = false;
uint64_t filter_whitelist[] = {
0x0000000000000001,
0x0000000000000003,
0x0000000000000005,
0x0000000000000007,
0x0000000000000009,
0x000000000000000B,
0x000000000000000D,
0x000000000000000F,
};
class CDnsSeedOpts { class CDnsSeedOpts {
public: public:
int nThreads; int nThreads;
@ -167,19 +178,20 @@ extern "C" void* ThreadCrawler(void* data) {
} while(1); } while(1);
} }
extern "C" int GetIPList(void *thread, addr_t *addr, int max, int ipv4, int ipv6); extern "C" int GetIPList(void *thread, char *requestedHostname, addr_t *addr, int max, int ipv4, int ipv6);
class CDnsThread { class CDnsThread {
public: public:
dns_opt_t dns_opt; // must be first dns_opt_t dns_opt; // must be first
const int id; const int id;
vector<addr_t> cache; std::map<uint64_t, vector<addr_t> > cache;
int nIPv4, nIPv6; int nIPv4, nIPv6;
time_t cacheTime; std::map<uint64_t, time_t> cacheTime;
unsigned int cacheHits; unsigned int cacheHits;
uint64_t dbQueries; uint64_t dbQueries;
std::vector<uint64_t> filterWhitelist;
void cacheHit(bool force = false) { void cacheHit(uint64_t requestedFlags, bool force = false) {
static bool nets[NET_MAX] = {}; static bool nets[NET_MAX] = {};
if (!nets[NET_IPV4]) { if (!nets[NET_IPV4]) {
nets[NET_IPV4] = true; nets[NET_IPV4] = true;
@ -187,14 +199,14 @@ public:
} }
time_t now = time(NULL); time_t now = time(NULL);
cacheHits++; cacheHits++;
if (force || cacheHits > (cache.size()*cache.size()/400) || (cacheHits*cacheHits > cache.size() / 20 && (now - cacheTime > 5))) { if (force || cacheHits > (cache[requestedFlags].size()*cache[requestedFlags].size()/400) || (cacheHits*cacheHits > cache[requestedFlags].size() / 20 && (now - cacheTime[requestedFlags] > 5))) {
set<CNetAddr> ips; set<CNetAddr> ips;
db.GetIPs(ips, 1000, nets); db.GetIPs(ips, requestedFlags, 1000, nets);
dbQueries++; dbQueries++;
cache.clear(); cache[requestedFlags].clear();
nIPv4 = 0; nIPv4 = 0;
nIPv6 = 0; nIPv6 = 0;
cache.reserve(ips.size()); cache[requestedFlags].reserve(ips.size());
for (set<CNetAddr>::iterator it = ips.begin(); it != ips.end(); it++) { for (set<CNetAddr>::iterator it = ips.begin(); it != ips.end(); it++) {
struct in_addr addr; struct in_addr addr;
struct in6_addr addr6; struct in6_addr addr6;
@ -202,18 +214,18 @@ public:
addr_t a; addr_t a;
a.v = 4; a.v = 4;
memcpy(&a.data.v4, &addr, 4); memcpy(&a.data.v4, &addr, 4);
cache.push_back(a); cache[requestedFlags].push_back(a);
nIPv4++; nIPv4++;
} else if ((*it).GetIn6Addr(&addr6)) { } else if ((*it).GetIn6Addr(&addr6)) {
addr_t a; addr_t a;
a.v = 6; a.v = 6;
memcpy(&a.data.v6, &addr6, 16); memcpy(&a.data.v6, &addr6, 16);
cache.push_back(a); cache[requestedFlags].push_back(a);
nIPv6++; nIPv6++;
} }
} }
cacheHits = 0; cacheHits = 0;
cacheTime = now; cacheTime[requestedFlags] = now;
} }
} }
@ -227,13 +239,12 @@ public:
dns_opt.port = opts->nPort; dns_opt.port = opts->nPort;
dns_opt.nRequests = 0; dns_opt.nRequests = 0;
cache.clear(); cache.clear();
cache.reserve(1000); cacheTime.clear();
cacheTime = 0;
cacheHits = 0; cacheHits = 0;
dbQueries = 0; dbQueries = 0;
nIPv4 = 0; nIPv4 = 0;
nIPv6 = 0; nIPv6 = 0;
cacheHit(true); filterWhitelist = std::vector<uint64_t>(filter_whitelist, filter_whitelist + (sizeof filter_whitelist / sizeof filter_whitelist[0]));
} }
void run() { void run() {
@ -241,10 +252,23 @@ public:
} }
}; };
extern "C" int GetIPList(void *data, addr_t* addr, int max, int ipv4, int ipv6) { extern "C" int GetIPList(void *data, char *requestedHostname, addr_t* addr, int max, int ipv4, int ipv6) {
CDnsThread *thread = (CDnsThread*)data; CDnsThread *thread = (CDnsThread*)data;
thread->cacheHit();
unsigned int size = thread->cache.size(); uint64_t requestedFlags = 0;
int hostlen = strlen(requestedHostname);
if (hostlen > 1 && requestedHostname[0] == 'x' && requestedHostname[1] != '0') {
char *pEnd;
uint64_t flags = (uint64_t)strtoull(requestedHostname+1, &pEnd, 16);
if (*pEnd == '.' && pEnd <= requestedHostname+17 && std::find(thread->filterWhitelist.begin(), thread->filterWhitelist.end(), flags) != thread->filterWhitelist.end())
requestedFlags = flags;
else
return 0;
}
else if (strcasecmp(requestedHostname, thread->dns_opt.host))
return 0;
thread->cacheHit(requestedFlags);
unsigned int size = thread->cache[requestedFlags].size();
unsigned int maxmax = (ipv4 ? thread->nIPv4 : 0) + (ipv6 ? thread->nIPv6 : 0); unsigned int maxmax = (ipv4 ? thread->nIPv4 : 0) + (ipv6 ? thread->nIPv6 : 0);
if (max > size) if (max > size)
max = size; max = size;
@ -254,16 +278,16 @@ extern "C" int GetIPList(void *data, addr_t* addr, int max, int ipv4, int ipv6)
while (i<max) { while (i<max) {
int j = i + (rand() % (size - i)); int j = i + (rand() % (size - i));
do { do {
bool ok = (ipv4 && thread->cache[j].v == 4) || bool ok = (ipv4 && thread->cache[requestedFlags][j].v == 4) ||
(ipv6 && thread->cache[j].v == 6); (ipv6 && thread->cache[requestedFlags][j].v == 6);
if (ok) break; if (ok) break;
j++; j++;
if (j==size) if (j==size)
j=i; j=i;
} while(1); } while(1);
addr[i] = thread->cache[j]; addr[i] = thread->cache[requestedFlags][j];
thread->cache[j] = thread->cache[i]; thread->cache[requestedFlags][j] = thread->cache[requestedFlags][i];
thread->cache[i] = addr[i]; thread->cache[requestedFlags][i] = addr[i];
i++; i++;
} }
return max; return max;
@ -306,11 +330,11 @@ extern "C" void* ThreadDumper(void*) {
rename("dnsseed.dat.new", "dnsseed.dat"); rename("dnsseed.dat.new", "dnsseed.dat");
} }
FILE *d = fopen("dnsseed.dump", "w"); FILE *d = fopen("dnsseed.dump", "w");
fprintf(d, "# address good lastSuccess %%(2h) %%(8h) %%(1d) %%(7d) %%(30d) blocks svcs version\n"); fprintf(d, "# address servicebits good lastSuccess %%(2h) %%(8h) %%(1d) %%(7d) %%(30d) blocks svcs version\n");
double stat[5]={0,0,0,0,0}; double stat[5]={0,0,0,0,0};
for (vector<CAddrReport>::const_iterator it = v.begin(); it < v.end(); it++) { for (vector<CAddrReport>::const_iterator it = v.begin(); it < v.end(); it++) {
CAddrReport rep = *it; CAddrReport rep = *it;
fprintf(d, "%-47s %4d %11"PRId64" %6.2f%% %6.2f%% %6.2f%% %6.2f%% %6.2f%% %6i %08"PRIx64" %5i \"%s\"\n", rep.ip.ToString().c_str(), (int)rep.fGood, rep.lastSuccess, 100.0*rep.uptime[0], 100.0*rep.uptime[1], 100.0*rep.uptime[2], 100.0*rep.uptime[3], 100.0*rep.uptime[4], rep.blocks, rep.services, rep.clientVersion, rep.clientSubVersion.c_str()); fprintf(d, "%-47s %8lld %4d %11"PRId64" %6.2f%% %6.2f%% %6.2f%% %6.2f%% %6.2f%% %6i %08"PRIx64" %5i \"%s\"\n", rep.ip.ToString().c_str(), (uint64_t)rep.services, (int)rep.fGood, rep.lastSuccess, 100.0*rep.uptime[0], 100.0*rep.uptime[1], 100.0*rep.uptime[2], 100.0*rep.uptime[3], 100.0*rep.uptime[4], rep.blocks, rep.services, rep.clientVersion, rep.clientSubVersion.c_str());
stat[0] += rep.uptime[0]; stat[0] += rep.uptime[0];
stat[1] += rep.uptime[1]; stat[1] += rep.uptime[1];
stat[2] += rep.uptime[2]; stat[2] += rep.uptime[2];