LBRYCommunity/seeder
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Multiple DNS threads

Browse source
This commit is contained in:
Pieter Wuille 2012-05-04 01:15:49 +02:00
parent ca1ca62a8a
commit 54fb24d7c3
4 changed files with 114 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
Makefile
View file

@ -1,14 +1,10 @@
dnsseed: dns.o bitcoin.o netbase.o protocol.o db.o main.o
g++ -pthread -lcrypto -o dnsseed dns.o bitcoin.o netbase.o protocol.o db.o main.o
strip -s dnsseed
dnsseed.dbg: dns.o bitcoin.o netbase.o protocol.o db.o main.o
g++ -pthread -lcrypto -o dnsseed.dbg dns.o bitcoin.o netbase.o protocol.o db.o main.o
%.o: %.cpp bitcoin.h netbase.h protocol.h db.h serialize.h uint256.h util.h
g++ -pthread -O3 -ggdb3 -march=nocona -Wno-invalid-offsetof -c -o $@ $<
g++ -pthread -O2 -ggdb3 -march=nocona -Wno-invalid-offsetof -c -o $@ $<
dns.o: dns.c
gcc -pthread -std=c99 -O3 -g0 -march=nocona dns.c -c -o dns.o
gcc -pthread -std=c99 -O2 -ggdb3 -march=nocona dns.c -c -o dns.o
%.o: %.cpp

28
dns.c
View file

@ -296,7 +296,7 @@ ssize_t static dnshandle(dns_opt_t *opt, const unsigned char *inbuf, size_t insi
// A records
if ((typ == TYPE_A || typ == QTYPE_ANY) && (cls == CLASS_IN || cls == QCLASS_ANY)) {
struct in_addr addr[32];
int naddr = opt->cb(addr, 32, 1);
int naddr = opt->cb((void*)opt, addr, 32, 1);
int n = 0;
while (n < naddr) {
int ret = write_record_a(&outpos, outend - auth_size, "", offset, CLASS_IN, opt->datattl, &addr[n]);
@ -333,27 +333,39 @@ error:
return 12;
}
static int listenSocket = -1;
int dnsserver(dns_opt_t *opt) {
struct sockaddr_in si_me, si_other;
socklen_t s, slen=sizeof(si_other);
unsigned char inbuf[BUFLEN], outbuf[BUFLEN];
if ((s=socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP))==-1)
struct sockaddr_in si_other;
int senderSocket = -1;
senderSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
if (senderSocket == -1)
return -3;
if (listenSocket == -1) {
struct sockaddr_in si_me;
if ((listenSocket=socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP))==-1) {
listenSocket = -1;
return -1;
}
memset((char *) &si_me, 0, sizeof(si_me));
si_me.sin_family = AF_INET;
si_me.sin_port = htons(opt->port);
si_me.sin_addr.s_addr = INADDR_ANY;
if (bind(s, (struct sockaddr*)&si_me, sizeof(si_me))==-1)
if (bind(listenSocket, (struct sockaddr*)&si_me, sizeof(si_me))==-1)
return -2;
}
unsigned char inbuf[BUFLEN], outbuf[BUFLEN];
do {
ssize_t insize = recvfrom(s, inbuf, BUFLEN, 0, (struct sockaddr*)&si_other, &slen);
socklen_t si_other_len = sizeof(si_other);
ssize_t insize = recvfrom(listenSocket, inbuf, BUFLEN, 0, (struct sockaddr*)&si_other, &si_other_len);
unsigned char *addr = (unsigned char*)&si_other.sin_addr.s_addr;
// printf("DNS: Request %llu from %i.%i.%i.%i:%i of %i bytes\n", (unsigned long long)(opt->nRequests), addr[0], addr[1], addr[2], addr[3], ntohs(si_other.sin_port), (int)insize);
opt->nRequests++;
if (insize > 0) {
ssize_t ret = dnshandle(opt, inbuf, insize, outbuf);
if (ret > 0)
sendto(s, outbuf, ret, 0, (struct sockaddr*)&si_other, slen);
sendto(listenSocket, outbuf, ret, 0, (struct sockaddr*)&si_other, sizeof(si_other));
}
} while(1);
return 0;

2
dns.h
View file

@ -10,7 +10,7 @@ typedef struct {
const char *host;
const char *ns;
const char *mbox;
int (*cb)(struct in_addr *addr, int max, int ipv4only);
int (*cb)(void *opt, struct in_addr *addr, int max, int ipv4only);
// stats
uint64_t nRequests;
} dns_opt_t;

98
main.cpp
View file

@ -16,11 +16,12 @@ class CDnsSeedOpts {
public:
int nThreads;
int nPort;
int nDnsThreads;
const char *mbox;
const char *ns;
const char *host;
CDnsSeedOpts() : nThreads(24), nPort(53), mbox(NULL), ns(NULL), host(NULL) {}
CDnsSeedOpts() : nThreads(24), nDnsThreads(24), nPort(53), mbox(NULL), ns(NULL), host(NULL) {}
void ParseCommandLine(int argc, char **argv) {
static const char *help = "Bitcoin-seeder\n"
@ -31,6 +32,7 @@ public:
"-n <ns> Hostname of the nameserver\n"
"-m <mbox> E-Mail address reported in SOA records\n"
"-t <threads> Number of crawlers to run in parallel (default 24)\n"
"-d <threads> Number of DNS server threads (default 24)\n"
"-p <port> UDP port to listen on (default 53)\n"
"-?, --help Show this text\n"
"\n";
@ -42,12 +44,13 @@ public:
{"ns", required_argument, 0, 'n'},
{"mbox", required_argument, 0, 'm'},
{"threads", required_argument, 0, 't'},
{"dnsthreads", required_argument, 0, 'd'},
{"port", required_argument, 0, 'p'},
{"help", no_argument, 0, 'h'},
{0, 0, 0, 0}
};
int option_index = 0;
int c = getopt_long(argc, argv, "h:n:m:t:p:", long_options, &option_index);
int c = getopt_long(argc, argv, "h:n:m:t:p:d:", long_options, &option_index);
if (c == -1) break;
switch (c) {
case 'h': {
@ -71,6 +74,12 @@ public:
break;
}
case 'd': {
int n = strtol(optarg, NULL, 10);
if (n > 0 && n < 1000) nDnsThreads = n;
break;
}
case 'p': {
int p = strtol(optarg, NULL, 10);
if (p > 0 && p < 65536) nPort = p;
@ -118,15 +127,20 @@ extern "C" void* ThreadCrawler(void* data) {
} while(1);
}
static vector<struct in_addr> cache;
static time_t cacheTime;
static unsigned int cacheHits = 1000000000;
static uint64_t dbQueries = 0;
extern "C" int GetIPList(void *thread, struct in_addr *addr, int max, int ipv4only);
void static cacheRefresh(int ipv4only) {
class CDnsThread {
public:
dns_opt_t dns_opt;
vector<struct in_addr> cache;
time_t cacheTime;
unsigned int cacheHits;
uint64_t dbQueries;
void cacheHit(int ipv4only, bool force = false) {
time_t now = time(NULL);
cacheHits++;
if (cacheHits > (cache.size()*cache.size()/400) || (cacheHits*cacheHits > cache.size() / 20 && (now - cacheTime > 5))) {
if (force || cacheHits > (cache.size()*cache.size()/400) || (cacheHits*cacheHits > cache.size() / 20 && (now - cacheTime > 5))) {
set<CIP> ips;
db.GetIPs(ips, 1000, ipv4only);
dbQueries++;
@ -141,25 +155,9 @@ void static cacheRefresh(int ipv4only) {
cacheHits = 0;
cacheTime = now;
}
}
extern "C" int GetIPList(struct in_addr *addr, int max, int ipv4only) {
cacheRefresh(ipv4only);
if (max > cache.size())
max = cache.size();
for (int i=0; i<max; i++) {
int j = i + (rand() % (cache.size() - i));
addr[i] = cache[j];
cache[j] = cache[i];
cache[i] = addr[i];
}
return max;
}
static dns_opt_t dns_opt;
extern "C" void* ThreadDNS(void* arg) {
CDnsSeedOpts *opts = (CDnsSeedOpts*)arg;
CDnsThread(CDnsSeedOpts* opts) {
dns_opt.host = opts->host;
dns_opt.ns = opts->ns;
dns_opt.mbox = opts->mbox;
@ -168,7 +166,39 @@ extern "C" void* ThreadDNS(void* arg) {
dns_opt.cb = GetIPList;
dns_opt.port = opts->nPort;
dns_opt.nRequests = 0;
cache.clear();
cache.reserve(1000);
cacheTime = 0;
cacheHits = 0;
dbQueries = 0;
cacheHit(true, true);
}
void run() {
dnsserver(&dns_opt);
}
};
extern "C" int GetIPList(void *data, struct in_addr *addr, int max, int ipv4only) {
CDnsThread *thread = (CDnsThread*)data;
thread->cacheHit(ipv4only);
unsigned int size = thread->cache.size();
if (max > size)
max = size;
for (int i=0; i<max; i++) {
int j = i + (rand() % (size - i));
addr[i] = thread->cache[j];
thread->cache[j] = thread->cache[i];
thread->cache[i] = addr[i];
}
return max;
}
vector<CDnsThread*> dnsThread;
extern "C" void* ThreadDNS(void* arg) {
CDnsThread *thread = (CDnsThread*)arg;
thread->run();
}
int StatCompare(const CAddrReport& a, const CAddrReport& b) {
@ -223,7 +253,13 @@ extern "C" void* ThreadStats(void*) {
CAddrDbStats stats;
db.GetStats(stats);
printf("\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b");
printf("%s %i/%i available (%i tried in %is, %i new, %i active), %i banned; %llu DNS requests, %llu db reads", c, stats.nGood, stats.nAvail, stats.nTracked, stats.nAge, stats.nNew, stats.nAvail - stats.nTracked - stats.nNew, stats.nBanned, (unsigned long long)dns_opt.nRequests, (unsigned long long)dbQueries);
uint64_t requests = 0;
uint64_t queries = 0;
for (unsigned int i=0; i<dnsThread.size(); i++) {
requests += dnsThread[i]->dns_opt.nRequests;
queries += dnsThread[i]->dbQueries;
}
printf("%s %i/%i available (%i tried in %is, %i new, %i active), %i banned; %llu DNS requests, %llu db queries", c, stats.nGood, stats.nAvail, stats.nTracked, stats.nAge, stats.nNew, stats.nAvail - stats.nTracked - stats.nNew, stats.nBanned, (unsigned long long)requests, (unsigned long long)queries);
Sleep(1000);
} while(1);
}
@ -276,8 +312,14 @@ int main(int argc, char **argv) {
printf("done\n");
pthread_create(&threadDump, NULL, ThreadDumper, NULL);
if (fDNS) {
printf("Starting DNS server for %s on %s (port %i)...", opts.host, opts.ns, opts.nPort);
pthread_create(&threadDns, NULL, ThreadDNS, &opts);
printf("Starting %i DNS threads for %s on %s (port %i)...", opts.nDnsThreads, opts.host, opts.ns, opts.nPort);
dnsThread.clear();
for (int i=0; i<opts.nDnsThreads; i++) {
dnsThread.push_back(new CDnsThread(&opts));
pthread_create(&threadDns, NULL, ThreadDNS, dnsThread[i]);
printf(".");
Sleep(20);
}
printf("done\n");
}
pthread_create(&threadStats, NULL, ThreadStats, NULL);