apr-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "RCHAPACH Rochester" <rchap...@us.ibm.com>
Subject [PATCH] Dynamic socket descriptor mem allocation (repost)
Date Fri, 14 Sep 2001 15:35:21 GMT

 This patch is from two co-workers of mine - Saeid Sakhitab and Cale Rath.

Rob Simonson
simo@us.ibm.com



=======================================================================================



The following patch, which is currently running on the iSeries platform,
will dynamically allocate memory for descriptors according to the highest
listening socket descriptor which is used for the select in poll.c on
platforms that do not use poll.  Since socket and file descriptors are used
hand-in-hand, FD_SIZE which is normally set to 200, can be too small if
many files are cached according to their descriptors, and/or if many
listening sockets are present because of the use of multiple Listens or
multiple virtual hosts which could affect a large ISP.

--- networkio.h     Wed Sep  5 10:59:21 2001
+++ ../new/networkio.h   Wed Sep  5 10:53:19 2001
@@ -136,6 +136,8 @@
     int local_interface_unknown;
     apr_int32_t netmask;
     apr_int32_t inherit;
+    unsigned int *fdset;
+    int fdset_size;
 };

 struct apr_pollfd_t {
@@ -145,13 +147,13 @@
     int num;
     int curpos;
 #else
-    fd_set *read;
-    fd_set *write;
-    fd_set *except;
+    unsigned int *read;
+    unsigned int *write;
+    unsigned int *except;
     int highsock;
-    fd_set *read_set;
-    fd_set *write_set;
-    fd_set *except_set;
+    unsigned int *read_set;
+    unsigned int *write_set;
+    unsigned int *except_set;
 #endif
     apr_int16_t *events;
     apr_int16_t *revents;
@@ -172,5 +174,18 @@
             *mask &= ~option;            \
     } while (0)

+#define APR_SOCK_SETSIZE(fd,cntxt) \
+      (unsigned int *)apr_pcalloc(cntxt,(fd/(8*sizeof(int)) + 1) *
sizeof(int))
+#define APR_FDSET_SIZE(fd_size,size) \
+      (fd_size = (size/(8*sizeof(int)) + 1) * sizeof(int))
+#define APR_FD_ZERO(fds,fd_size)  \
+      (memset(fds,0,fd_size))  /* set all 0  */
+#define APR_FD_SET(fd, fds)  \
+      (fds[(fd)/32] |= (1 << ((fd)%32)))      /* set bits   */
+#define APR_FD_CLR(fd, fds)   \
+      (fds[(fd)/32] &= ~(1 << ((fd)%32)))     /* clear bits */
+#define APR_FD_ISSET(fd, fds)  \
+      ((fds[(fd)/32]&(1 << ((fd)%32)))? 1 : 0)      /* check bits */
+
 #endif  /* ! NETWORK_IO_H */

==============================================
--- poll.c     Wed Sep  5 10:59:21 2001
+++ ../new/poll.c   Wed Sep  5 10:53:19 2001
@@ -222,40 +222,43 @@

 #else    /* Use select to mimic poll */

-apr_status_t apr_poll_setup(apr_pollfd_t **new, apr_int32_t num,
apr_pool_t *cont)
+/*  use the highest descriptor to set the size for read, write, and except
+ *  Also pull out FD_ZERO since apr_pcalloc zero's the memory it allocates
+ */
+apr_status_t apr_poll_setup(apr_pollfd_t **new, apr_int32_t high_sock,
apr_pool_t *cont)
 {
-    (*new) = (apr_pollfd_t *)apr_pcalloc(cont, sizeof(apr_pollfd_t) *
num);
+    apr_int32_t fd_set_size;
+    APR_FDSET_SIZE(fd_set_size,high_sock);
+
+    (*new) = (apr_pollfd_t *)apr_pcalloc(cont, sizeof(apr_pollfd_t));
     if ((*new) == NULL) {
         return APR_ENOMEM;
     }
     (*new)->cntxt = cont;
-    (*new)->read = (fd_set *)apr_pcalloc(cont, sizeof(fd_set));
-    (*new)->write = (fd_set *)apr_pcalloc(cont, sizeof(fd_set));
-    (*new)->except = (fd_set *)apr_pcalloc(cont, sizeof(fd_set));
-    (*new)->read_set = (fd_set *)apr_pcalloc(cont, sizeof(fd_set));
-    (*new)->write_set = (fd_set *)apr_pcalloc(cont, sizeof(fd_set));
-    (*new)->except_set = (fd_set *)apr_pcalloc(cont, sizeof(fd_set));
-    FD_ZERO((*new)->read);
-    FD_ZERO((*new)->write);
-    FD_ZERO((*new)->except);
-    FD_ZERO((*new)->read_set);
-    FD_ZERO((*new)->write_set);
-    FD_ZERO((*new)->except_set);
-    (*new)->highsock = -1;
+    (*new)->read = (unsigned int *)apr_pcalloc(cont, fd_set_size);
+    (*new)->write = (unsigned int *)apr_pcalloc(cont, fd_set_size);
+    (*new)->except = (unsigned int *)apr_pcalloc(cont, fd_set_size);
+    (*new)->read_set = (unsigned int *)apr_pcalloc(cont, fd_set_size);
+    (*new)->write_set = (unsigned int *)apr_pcalloc(cont, fd_set_size);
+    (*new)->except_set = (unsigned int *)apr_pcalloc(cont, fd_set_size);
+    (*new)->highsock = high_sock;
     return APR_SUCCESS;
 }

+/*   -- use APR_FD_SET defined in networkio.h for dynamic
+ *      memory allocation for read, write, and except
+ */
 apr_status_t apr_poll_socket_add(apr_pollfd_t *aprset,
                                apr_socket_t *sock, apr_int16_t event)
 {
     if (event & APR_POLLIN) {
-        FD_SET(sock->socketdes, aprset->read_set);
+        APR_FD_SET(sock->socketdes, aprset->read_set);
     }
     if (event & APR_POLLPRI) {
-        FD_SET(sock->socketdes, aprset->except_set);
+        APR_FD_SET(sock->socketdes, aprset->except_set);
     }
     if (event & APR_POLLOUT) {
-        FD_SET(sock->socketdes, aprset->write_set);
+        APR_FD_SET(sock->socketdes, aprset->write_set);
     }
     if (sock->socketdes > aprset->highsock) {
         aprset->highsock = sock->socketdes;
@@ -263,27 +266,35 @@
     return APR_SUCCESS;
 }

+/* -- use APR_FD_CLR defined in networkio.h for dynamic
+ *    memory allocation for read, write, and except
+ */
 apr_status_t apr_poll_socket_mask(apr_pollfd_t *aprset,
                                   apr_socket_t *sock,
                                   apr_int16_t events)
 {
     if (events & APR_POLLIN) {
-        FD_CLR(sock->socketdes, aprset->read_set);
+        APR_FD_CLR(sock->socketdes, aprset->read_set);
     }
     if (events & APR_POLLPRI) {
-        FD_CLR(sock->socketdes, aprset->except_set);
+        APR_FD_CLR(sock->socketdes, aprset->except_set);
     }
     if (events & APR_POLLOUT) {
-        FD_CLR(sock->socketdes, aprset->write_set);
+        APR_FD_CLR(sock->socketdes, aprset->write_set);
     }
     return APR_SUCCESS;
 }

+/* -- use APR_FDSET_SIZE defined in networkio.h for dynamic
+ *    memory allocation for read, write, and except
+ *    Also cast read, write, and except to fd_set *
+ */
 apr_status_t apr_poll(apr_pollfd_t *aprset, apr_int32_t *nsds,
              apr_interval_time_t timeout)
 {
     int rv;
     struct timeval tv, *tvptr;
+    apr_int32_t fd_set_size;

     if (timeout < 0) {
         tvptr = NULL;
@@ -294,12 +305,14 @@
         tvptr = &tv;
     }

-    memcpy(aprset->read, aprset->read_set, sizeof(fd_set));
-    memcpy(aprset->write, aprset->write_set, sizeof(fd_set));
-    memcpy(aprset->except, aprset->except_set, sizeof(fd_set));
+    APR_FDSET_SIZE(fd_set_size,aprset->highsock);
+
+    memcpy(aprset->read, aprset->read_set, fd_set_size);
+    memcpy(aprset->write, aprset->write_set, fd_set_size);
+    memcpy(aprset->except, aprset->except_set, fd_set_size);

-    rv = select(aprset->highsock + 1, aprset->read, aprset->write,
-                aprset->except, tvptr);
+    rv = select(aprset->highsock + 1, (fd_set *)aprset->read, (fd_set
*)aprset->write,
+                (fd_set *)aprset->except, tvptr);

     (*nsds) = rv;
     if ((*nsds) == 0) {
@@ -310,7 +323,9 @@
     }
     return APR_SUCCESS;
 }
-
+/* -- use APR_FD_ISSET defined in networkio.h for dynamic
+ *    memory allocation for read, write, and except
+ */
 apr_status_t apr_poll_revents_get(apr_int16_t *event, apr_socket_t *sock,
apr_pollfd_t *aprset)
 {
     apr_int16_t revents = 0;
@@ -320,7 +335,7 @@
     /* We just want to PEEK at the data, so I am setting up a dummy WSABUF
      * variable here.
      */
-    if (FD_ISSET(sock->socketdes, aprset->read)) {
+    if (APR_FD_ISSET(sock->socketdes, aprset->read)) {
         revents |= APR_POLLIN;
         if (sock->connected
         && recv(sock->socketdes, data, sizeof data, flags) == -1) {
@@ -343,14 +358,14 @@
             }
         }
     }
-    if (FD_ISSET(sock->socketdes, aprset->write)) {
+    if (APR_FD_ISSET(sock->socketdes, aprset->write)) {
         revents |= APR_POLLOUT;
     }
     /* I am assuming that the except is for out of band data, not a failed
      * connection on a non-blocking socket.  Might be a bad assumption,
but
      * it works for now. rbb.
      */
-    if (FD_ISSET(sock->socketdes, aprset->except)) {
+    if (APR_FD_ISSET(sock->socketdes, aprset->except)) {
         revents |= APR_POLLPRI;
     }

@@ -358,24 +373,31 @@
     return APR_SUCCESS;
 }

+
+/* -- use APR_FD_CLR defined in networkio.h for dynamic
+ *    memory allocation for read, write, and except
+ */
 apr_status_t apr_poll_socket_remove(apr_pollfd_t *aprset, apr_socket_t
*sock)
 {
-    FD_CLR(sock->socketdes, aprset->read_set);
-    FD_CLR(sock->socketdes, aprset->except_set);
-    FD_CLR(sock->socketdes, aprset->write_set);
+    APR_FD_CLR(sock->socketdes, aprset->read_set);
+    APR_FD_CLR(sock->socketdes, aprset->except_set);
+    APR_FD_CLR(sock->socketdes, aprset->write_set);
     return APR_SUCCESS;
 }

+/* -- use APR_FD_ZERO defined in networkio.h for dynamic
+ *    memory allocation for read, write, and except
+ */
 apr_status_t apr_poll_socket_clear(apr_pollfd_t *aprset, apr_int16_t
event)
 {
     if (event & APR_POLLIN) {
-        FD_ZERO(aprset->read_set);
+        APR_FD_ZERO(aprset->read_set,aprset->highsock);
     }
     if (event & APR_POLLPRI) {
-        FD_ZERO(aprset->except_set);
+        APR_FD_ZERO(aprset->except_set,aprset->highsock);
     }
     if (event & APR_POLLOUT) {
-        FD_ZERO(aprset->write_set);
+        APR_FD_ZERO(aprset->write_set,aprset->highsock);
     }
     aprset->highsock = 0;
     return APR_SUCCESS;

=================================
+++ ../new/sendrecv.c    Wed Sep  5 10:53:20 2001
@@ -62,12 +62,12 @@
 apr_status_t apr_wait_for_io_or_timeout(apr_socket_t *sock, int for_read)
 {
     struct timeval tv, *tvptr;
-    fd_set fdset;
     int srv;

     do {
-        FD_ZERO(&fdset);
-        FD_SET(sock->socketdes, &fdset);
+        /* -- zero out the the descriptor mem in apr_socket_t */
+        APR_FD_ZERO(sock->fdset,sock->fdset_size);
+    APR_FD_SET(sock->socketdes,sock->fdset);
         if (sock->timeout < 0) {
             tvptr = NULL;
         }
@@ -76,9 +76,10 @@
             tv.tv_usec = sock->timeout % APR_USEC_PER_SEC;
             tvptr = &tv;
         }
-        srv = select(sock->socketdes + 1,
-            for_read ? &fdset : NULL,
-            for_read ? NULL : &fdset,
+    /* -- use vals set in apr_socket_t structure that were previously
dynamically set */
+    srv = select(sock->socketdes + 1,
+            for_read ? (fd_set *)sock->fdset : NULL,
+            for_read ? NULL : (fd_set*)sock->fdset,
             NULL,
             tvptr);
         /* TODO - timeout should be smaller on repeats of this loop */

=================================================
--- sockets.c  Wed Sep  5 10:59:22 2001
+++ ../new/sockets.c     Wed Sep  5 10:53:20 2001
@@ -219,6 +219,9 @@
     if ((*new)->socketdes < 0) {
         return errno;
     }
+    (*new)->fdset = APR_SOCK_SETSIZE((*new)->socketdes,(*new)->cntxt);
+    APR_FDSET_SIZE((*new)->fdset_size,(*new)->socketdes);
+
     *(*new)->local_addr = *sock->local_addr;
     /* fix up any pointers which are no longer valid */
     if (sock->local_addr->sa.sin.sin_family == AF_INET) {

===================================
--- threaded.c Wed Sep  5 10:59:22 2001
+++ ../new/threaded.c    Thu Sep  6 09:52:16 2001
@@ -462,16 +462,6 @@

     (void) apr_os_sock_get(&csd, sock);

-    if (csd >= FD_SETSIZE) {
-        ap_log_error(APLOG_MARK, APLOG_NOERRNO|APLOG_WARNING, 0, NULL,
-                     "new file descriptor %d is too large; you probably
need "
-                     "to rebuild Apache with a larger FD_SETSIZE "
-                     "(currently %d)",
-                     csd, FD_SETSIZE);
-        apr_socket_close(sock);
-        return;
-    }
-
     ap_sock_disable_nagle(sock);

     current_conn = ap_new_connection(p, ap_server_conf, sock, conn_id);
@@ -542,6 +532,8 @@
     int curr_pollfd, last_pollfd = 0;
     apr_pollfd_t *pollset;
     apr_status_t rv;
+    int local_hisock = -1;
+    int csd;

     free(ti);

@@ -551,7 +543,18 @@
     worker_thread_count++;
     apr_lock_release(worker_thread_count_mutex);

+#ifdef HAVE_POLL
     apr_poll_setup(&pollset, num_listensocks+1, tpool);
+#else
+    for(n=0 ; n <= num_listensocks ; ++n){
+    (void) apr_os_sock_get(&csd,listensocks[n]);
+    if (csd  > local_hisock) {
+        local_hisock = csd;
+    }
+    }
+    apr_poll_setup(&pollset, local_hisock, tpool);
+#endif
+
     for(n=0 ; n <= num_listensocks ; ++n)
     apr_poll_socket_add(pollset, listensocks[n], APR_POLLIN);
--

______________________________________________________________
Ryan Bloom                    rbb@apache.org
Covalent Technologies              rbb@covalent.net
--------------------------------------------------------------




Mime
View raw message