Hi,
We found a problem with list requests_inflight while using evdns on
highly loaded system (about 2000 DNS requests per second). First of all
linear search is slow and second is that this list is not big enough to
handle all requests, so many requests were in requests_waiting list,
that has no limits on length. Due to this application grows in size
constantly and is not able to serve requests. I've replaced linear list
with hashed list, that improves performance. The one thing that is not
logical (but I don't know how to make it better) is that I'm using bzero
of hash nests array in reading resolv conf function. The second part of
patch allows using evhttp module for listening on unix sockets. This is
our project specific requirement, but it may be useful for others.
--- a/evdns.c Wed Feb 13 15:31:39 2008 +0300
+++ b/evdns.c Wed Feb 13 16:24:17 2008 +0300
@@ -156,6 +156,7 @@ typedef unsigned int uint;
#endif
#define MAX_ADDRS 32 /* maximum number of addresses from a single packet */
+#define MAX_NESTS 1024 /* numbers of nests in hashed list of pending requests
*/
/* which we bother recording */
#define TYPE_A EVDNS_TYPE_A
@@ -231,7 +232,7 @@ struct nameserver {
char write_waiting; /* true if we are waiting for EV_WRITE events */
};
-static struct request *req_head = NULL, *req_waiting_head = NULL;
+static struct request *req_head[MAX_NESTS], *req_waiting_head = NULL;
static struct nameserver *server_head = NULL;
/* Represents a local port where we're listening for DNS requests. Right now,
*/
@@ -310,7 +311,7 @@ static int global_requests_inflight = 0;
/* and are counted here */
static int global_requests_waiting = 0;
-static int global_max_requests_inflight = 64;
+static int global_max_requests_inflight = 2048;
static struct timeval global_timeout = {5, 0}; /* 5 seconds */
static int global_max_reissues = 1; /* a reissue occurs when we get some
errors from the server */
@@ -443,7 +444,7 @@ _evdns_log(int warn, const char *fmt, ..
/* failure */
static struct request *
request_find_from_trans_id(u16 trans_id) {
- struct request *req = req_head, *const started_at = req_head;
+ struct request *req = req_head[trans_id % MAX_NESTS], *const started_at
= req_head[trans_id % MAX_NESTS];
if (req) {
do {
@@ -498,6 +499,7 @@ static void
static void
nameserver_failed(struct nameserver *const ns, const char *msg) {
struct request *req, *started_at;
+ int i;
/* if this nameserver has already been marked as failed */
/* then don't do anything */
if (!ns->state) return;
@@ -529,17 +531,19 @@ nameserver_failed(struct nameserver *con
/* trying to reassign requests to one */
if (!global_good_nameservers) return;
- req = req_head;
- started_at = req_head;
- if (req) {
- do {
- if (req->tx_count == 0 && req->ns == ns) {
- /* still waiting to go out, can be moved */
- /* to another server */
- req->ns = nameserver_pick();
- }
- req = req->next;
- } while (req != started_at);
+ for (i = 0; i < MAX_NESTS; i ++) {
+ req = req_head[i];
+ started_at = req_head[i];
+ if (req) {
+ do {
+ if (req->tx_count == 0 && req->ns == ns) {
+ /* still waiting to go out, can be
moved */
+ /* to another server */
+ req->ns = nameserver_pick();
+ }
+ req = req->next;
+ } while (req != started_at);
+ }
}
}
@@ -650,7 +654,7 @@ evdns_requests_pump_waiting_queue(void)
req->ns = nameserver_pick();
request_trans_id_set(req, transaction_id_pick());
- evdns_request_insert(req, &req_head);
+ evdns_request_insert(req, &req_head[req->trans_id % MAX_NESTS]);
evdns_request_transmit(req);
evdns_transmit();
}
@@ -742,19 +746,19 @@ reply_handle(struct request *const req,
/* a new request was issued so this request is
finished and */
/* the user callback will be made when that
request (or a */
/* child of it) finishes. */
- request_finished(req, &req_head);
+ request_finished(req, &req_head[req->trans_id %
MAX_NESTS]);
return;
}
}
/* all else failed. Pass the failure up */
reply_callback(req, 0, error, NULL);
- request_finished(req, &req_head);
+ request_finished(req, &req_head[req->trans_id % MAX_NESTS]);
} else {
/* all ok, tell the user */
reply_callback(req, ttl, 0, reply);
nameserver_up(req->ns);
- request_finished(req, &req_head);
+ request_finished(req, &req_head[req->trans_id % MAX_NESTS]);
}
}
@@ -1090,12 +1094,13 @@ static u16
static u16
transaction_id_pick(void) {
for (;;) {
- const struct request *req = req_head, *started_at;
+ const struct request *req, *started_at;
u16 trans_id = trans_id_function();
+ req = req_head[trans_id % MAX_NESTS];
if (trans_id == 0xffff) continue;
/* now check to see if that id is already inflight */
- req = started_at = req_head;
+ req = started_at = req_head[trans_id % MAX_NESTS];
if (req) {
do {
if (req->trans_id == trans_id) break;
@@ -1869,7 +1874,7 @@ evdns_request_timeout_callback(int fd, s
if (req->tx_count >= global_max_retransmits) {
/* this request has failed */
reply_callback(req, 0, DNS_ERR_TIMEOUT, NULL);
- request_finished(req, &req_head);
+ request_finished(req, &req_head[req->trans_id % MAX_NESTS]);
} else {
/* retransmit it */
evdns_request_transmit(req);
@@ -1981,19 +1986,22 @@ nameserver_send_probe(struct nameserver
/* 1 tried to transmit something */
static int
evdns_transmit(void) {
+ int i;
char did_try_to_transmit = 0;
-
- if (req_head) {
- struct request *const started_at = req_head, *req = req_head;
- /* first transmit all the requests which are currently waiting
*/
- do {
- if (req->transmit_me) {
- did_try_to_transmit = 1;
- evdns_request_transmit(req);
- }
-
- req = req->next;
- } while (req != started_at);
+
+ for (i = 0; i < MAX_NESTS; i ++) {
+ if (req_head[i]) {
+ struct request *const started_at = req_head[i], *req =
req_head[i];
+ /* first transmit all the requests which are currently
waiting */
+ do {
+ if (req->transmit_me) {
+ did_try_to_transmit = 1;
+ evdns_request_transmit(req);
+ }
+
+ req = req->next;
+ } while (req != started_at);
+ }
}
return did_try_to_transmit;
@@ -2019,7 +2027,8 @@ evdns_clear_nameservers_and_suspend(void
evdns_clear_nameservers_and_suspend(void)
{
struct nameserver *server = server_head, *started_at = server_head;
- struct request *req = req_head, *req_started_at = req_head;
+ struct request *req, *req_started_at;
+ int i;
if (!server)
return 0;
@@ -2036,29 +2045,35 @@ evdns_clear_nameservers_and_suspend(void
}
server_head = NULL;
global_good_nameservers = 0;
-
- while (req) {
- struct request *next = req->next;
- req->tx_count = req->reissue_count = 0;
- req->ns = NULL;
- /* ???? What to do about searches? */
- (void) evtimer_del(&req->timeout_event);
- req->trans_id = 0;
- req->transmit_me = 0;
-
- global_requests_waiting++;
- evdns_request_insert(req, &req_waiting_head);
- /* We want to insert these suspended elements at the front of
- * the waiting queue, since they were pending before any of
- * the waiting entries were added. This is a circular list,
- * so we can just shift the start back by one.*/
- req_waiting_head = req_waiting_head->prev;
-
- if (next == req_started_at)
- break;
- req = next;
- }
- req_head = NULL;
+
+ for (i = 0; i < MAX_NESTS; i ++) {
+ req = req_head[i];
+ req_started_at = req;
+
+ while (req) {
+ struct request *next = req->next;
+ req->tx_count = req->reissue_count = 0;
+ req->ns = NULL;
+ /* ???? What to do about searches? */
+ (void) evtimer_del(&req->timeout_event);
+ req->trans_id = 0;
+ req->transmit_me = 0;
+
+ global_requests_waiting++;
+ evdns_request_insert(req, &req_waiting_head);
+ /* We want to insert these suspended elements at the
front of
+ * the waiting queue, since they were pending before
any of
+ * the waiting entries were added. This is a circular
list,
+ * so we can just shift the start back by one.*/
+ req_waiting_head = req_waiting_head->prev;
+
+ if (next == req_started_at)
+ break;
+ req = next;
+ }
+ req_head[i] = NULL;
+ }
+
global_requests_inflight = 0;
return 0;
@@ -2245,7 +2260,7 @@ request_submit(struct request *const req
if (req->ns) {
/* if it has a nameserver assigned then this is going */
/* straight into the inflight queue */
- evdns_request_insert(req, &req_head);
+ evdns_request_insert(req, &req_head[req->trans_id % MAX_NESTS]);
global_requests_inflight++;
evdns_request_transmit(req);
} else {
@@ -2714,6 +2729,9 @@ evdns_resolv_conf_parse(int flags, const
u8 *resolv;
char *start;
int err = 0;
+
+ /* XXX: remove this hack from here */
+ bzero (req_head, MAX_NESTS * sizeof (req_head[0]));
log(EVDNS_LOG_DEBUG, "Parsing resolv.conf file %s", filename);
@@ -3002,11 +3020,14 @@ evdns_shutdown(int fail_requests)
{
struct nameserver *server, *server_next;
struct search_domain *dom, *dom_next;
-
- while (req_head) {
- if (fail_requests)
- reply_callback(req_head, 0, DNS_ERR_SHUTDOWN, NULL);
- request_finished(req_head, &req_head);
+ int i;
+
+ for (i = 0; i < MAX_NESTS; i ++) {
+ while (req_head[i]) {
+ if (fail_requests)
+ reply_callback(req_head[i], 0,
DNS_ERR_SHUTDOWN, NULL);
+ request_finished(req_head[i], &req_head[i]);
+ }
}
while (req_waiting_head) {
if (fail_requests)
--- a/http.c Wed Feb 13 15:31:39 2008 +0300
+++ b/http.c Wed Feb 13 16:24:17 2008 +0300
@@ -78,6 +78,9 @@
#ifdef HAVE_FCNTL_H
#include <fcntl.h>
#endif
+#ifndef WIN32
+#include <sys/un.h>
+#endif
#undef timeout_pending
#undef timeout_initialized
@@ -2349,18 +2352,25 @@ name_from_addr(struct sockaddr *sa, sock
static char ntop[NI_MAXHOST];
static char strport[NI_MAXSERV];
int ni_result;
-
- if ((ni_result = getnameinfo(sa, salen,
- ntop, sizeof(ntop), strport, sizeof(strport),
- NI_NUMERICHOST|NI_NUMERICSERV)) != 0) {
- if (ni_result == EAI_SYSTEM)
- event_err(1, "getnameinfo failed");
- else
- event_errx(1, "getnameinfo failed: %s",
gai_strerror(ni_result));
- }
-
+
+ if (sa->sa_family == PF_UNIX) {
+ strncpy (ntop, ((struct sockaddr_un *)sa)->sun_path, NI_MAXHOST
- 1);
+ strport[0] = '0';
+ strport[1] = '\0';
+ }
+ else {
+ if ((ni_result = getnameinfo(sa, salen,
+ ntop, sizeof(ntop), strport, sizeof(strport),
+ NI_NUMERICHOST|NI_NUMERICSERV)) != 0) {
+ if (ni_result == EAI_SYSTEM)
+ event_err(1, "getnameinfo failed");
+ else
+ event_errx(1, "getnameinfo failed: %s",
gai_strerror(ni_result));
+ }
+ }
*phost = ntop;
*pport = strport;
+
#else
/* XXXX */
#endif
_______________________________________________
Libevent-users mailing list
Libevent-users@monkey.org
http://monkeymail.org/mailman/listinfo/libevent-users
