Return-Path: Delivered-To: apmail-apr-dev-archive@apr.apache.org Received: (qmail 58377 invoked by uid 500); 6 Sep 2001 21:07:52 -0000 Mailing-List: contact dev-help@apr.apache.org; run by ezmlm Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: Delivered-To: mailing list dev@apr.apache.org Received: (qmail 58366 invoked from network); 6 Sep 2001 21:07:51 -0000 Content-Type: text/plain; charset="iso-8859-1" From: Ryan Bloom Reply-To: rbb@covalent.net Organization: Covalent Technologies To: dev@apr.apache.org Subject: [FWD] [PATCH] Dynamic socket descriptor mem allocation Date: Thu, 6 Sep 2001 14:07:48 -0700 X-Mailer: KMail [version 1.3] MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Message-Id: <20010906210748.7FBEA46DF4@koj.rkbloom.net> X-Spam-Rating: daedalus.apache.org 1.6.2 0/1000/N From: "Robert Simonson" To: dev@apr.apache.org This patch is from two co-workers of mine - Saeid Sakhitab and Cale Rath. Rob Simonson simo@us.ibm.com ======================================================================================= The following patch, which is currently running on the iSeries platform, will dynamically allocate memory for descriptors according to the highest listening socket descriptor which is used for the select in poll.c on platforms that do not use poll. Since socket and file descriptors are used hand-in-hand, FD_SIZE which is normally set to 200, can be too small if many files are cached according to their descriptors, and/or if many listening sockets are present because of the use of multiple Listens or multiple virtual hosts which could affect a large ISP. --- networkio.h Wed Sep 5 10:59:21 2001 +++ ../new/networkio.h Wed Sep 5 10:53:19 2001 @@ -136,6 +136,8 @@ int local_interface_unknown; apr_int32_t netmask; apr_int32_t inherit; + unsigned int *fdset; + int fdset_size; }; struct apr_pollfd_t { @@ -145,13 +147,13 @@ int num; int curpos; #else - fd_set *read; - fd_set *write; - fd_set *except; + unsigned int *read; + unsigned int *write; + unsigned int *except; int highsock; - fd_set *read_set; - fd_set *write_set; - fd_set *except_set; + unsigned int *read_set; + unsigned int *write_set; + unsigned int *except_set; #endif apr_int16_t *events; apr_int16_t *revents; @@ -172,5 +174,18 @@ *mask &= ~option; \ } while (0) +#define APR_SOCK_SETSIZE(fd,cntxt) \ + (unsigned int *)apr_pcalloc(cntxt,(fd/(8*sizeof(int)) + 1) * sizeof(int)) +#define APR_FDSET_SIZE(fd_size,size) \ + (fd_size = (size/(8*sizeof(int)) + 1) * sizeof(int)) +#define APR_FD_ZERO(fds,fd_size) \ + (memset(fds,0,fd_size)) /* set all 0 */ +#define APR_FD_SET(fd, fds) \ + (fds[(fd)/32] |= (1 << ((fd)%32))) /* set bits */ +#define APR_FD_CLR(fd, fds) \ + (fds[(fd)/32] &= ~(1 << ((fd)%32))) /* clear bits */ +#define APR_FD_ISSET(fd, fds) \ + ((fds[(fd)/32]&(1 << ((fd)%32)))? 1 : 0) /* check bits */ + #endif /* ! NETWORK_IO_H */ ============================================== --- poll.c Wed Sep 5 10:59:21 2001 +++ ../new/poll.c Wed Sep 5 10:53:19 2001 @@ -222,40 +222,43 @@ #else /* Use select to mimic poll */ -apr_status_t apr_poll_setup(apr_pollfd_t **new, apr_int32_t num, apr_pool_t *cont) +/* use the highest descriptor to set the size for read, write, and except + * Also pull out FD_ZERO since apr_pcalloc zero's the memory it allocates + */ +apr_status_t apr_poll_setup(apr_pollfd_t **new, apr_int32_t high_sock, apr_pool_t *cont) { - (*new) = (apr_pollfd_t *)apr_pcalloc(cont, sizeof(apr_pollfd_t) * num); + apr_int32_t fd_set_size; + APR_FDSET_SIZE(fd_set_size,high_sock); + + (*new) = (apr_pollfd_t *)apr_pcalloc(cont, sizeof(apr_pollfd_t)); if ((*new) == NULL) { return APR_ENOMEM; } (*new)->cntxt = cont; - (*new)->read = (fd_set *)apr_pcalloc(cont, sizeof(fd_set)); - (*new)->write = (fd_set *)apr_pcalloc(cont, sizeof(fd_set)); - (*new)->except = (fd_set *)apr_pcalloc(cont, sizeof(fd_set)); - (*new)->read_set = (fd_set *)apr_pcalloc(cont, sizeof(fd_set)); - (*new)->write_set = (fd_set *)apr_pcalloc(cont, sizeof(fd_set)); - (*new)->except_set = (fd_set *)apr_pcalloc(cont, sizeof(fd_set)); - FD_ZERO((*new)->read); - FD_ZERO((*new)->write); - FD_ZERO((*new)->except); - FD_ZERO((*new)->read_set); - FD_ZERO((*new)->write_set); - FD_ZERO((*new)->except_set); - (*new)->highsock = -1; + (*new)->read = (unsigned int *)apr_pcalloc(cont, fd_set_size); + (*new)->write = (unsigned int *)apr_pcalloc(cont, fd_set_size); + (*new)->except = (unsigned int *)apr_pcalloc(cont, fd_set_size); + (*new)->read_set = (unsigned int *)apr_pcalloc(cont, fd_set_size); + (*new)->write_set = (unsigned int *)apr_pcalloc(cont, fd_set_size); + (*new)->except_set = (unsigned int *)apr_pcalloc(cont, fd_set_size); + (*new)->highsock = high_sock; return APR_SUCCESS; } +/* -- use APR_FD_SET defined in networkio.h for dynamic + * memory allocation for read, write, and except + */ apr_status_t apr_poll_socket_add(apr_pollfd_t *aprset, apr_socket_t *sock, apr_int16_t event) { if (event & APR_POLLIN) { - FD_SET(sock->socketdes, aprset->read_set); + APR_FD_SET(sock->socketdes, aprset->read_set); } if (event & APR_POLLPRI) { - FD_SET(sock->socketdes, aprset->except_set); + APR_FD_SET(sock->socketdes, aprset->except_set); } if (event & APR_POLLOUT) { - FD_SET(sock->socketdes, aprset->write_set); + APR_FD_SET(sock->socketdes, aprset->write_set); } if (sock->socketdes > aprset->highsock) { aprset->highsock = sock->socketdes; @@ -263,27 +266,35 @@ return APR_SUCCESS; } +/* -- use APR_FD_CLR defined in networkio.h for dynamic + * memory allocation for read, write, and except + */ apr_status_t apr_poll_socket_mask(apr_pollfd_t *aprset, apr_socket_t *sock, apr_int16_t events) { if (events & APR_POLLIN) { - FD_CLR(sock->socketdes, aprset->read_set); + APR_FD_CLR(sock->socketdes, aprset->read_set); } if (events & APR_POLLPRI) { - FD_CLR(sock->socketdes, aprset->except_set); + APR_FD_CLR(sock->socketdes, aprset->except_set); } if (events & APR_POLLOUT) { - FD_CLR(sock->socketdes, aprset->write_set); + APR_FD_CLR(sock->socketdes, aprset->write_set); } return APR_SUCCESS; } +/* -- use APR_FDSET_SIZE defined in networkio.h for dynamic + * memory allocation for read, write, and except + * Also cast read, write, and except to fd_set * + */ apr_status_t apr_poll(apr_pollfd_t *aprset, apr_int32_t *nsds, apr_interval_time_t timeout) { int rv; struct timeval tv, *tvptr; + apr_int32_t fd_set_size; if (timeout < 0) { tvptr = NULL; @@ -294,12 +305,14 @@ tvptr = &tv; } - memcpy(aprset->read, aprset->read_set, sizeof(fd_set)); - memcpy(aprset->write, aprset->write_set, sizeof(fd_set)); - memcpy(aprset->except, aprset->except_set, sizeof(fd_set)); + APR_FDSET_SIZE(fd_set_size,aprset->highsock); + + memcpy(aprset->read, aprset->read_set, fd_set_size); + memcpy(aprset->write, aprset->write_set, fd_set_size); + memcpy(aprset->except, aprset->except_set, fd_set_size); - rv = select(aprset->highsock + 1, aprset->read, aprset->write, - aprset->except, tvptr); + rv = select(aprset->highsock + 1, (fd_set *)aprset->read, (fd_set *)aprset->write, + (fd_set *)aprset->except, tvptr); (*nsds) = rv; if ((*nsds) == 0) { @@ -310,7 +323,9 @@ } return APR_SUCCESS; } - +/* -- use APR_FD_ISSET defined in networkio.h for dynamic + * memory allocation for read, write, and except + */ apr_status_t apr_poll_revents_get(apr_int16_t *event, apr_socket_t *sock, apr_pollfd_t *aprset) { apr_int16_t revents = 0; @@ -320,7 +335,7 @@ /* We just want to PEEK at the data, so I am setting up a dummy WSABUF * variable here. */ - if (FD_ISSET(sock->socketdes, aprset->read)) { + if (APR_FD_ISSET(sock->socketdes, aprset->read)) { revents |= APR_POLLIN; if (sock->connected && recv(sock->socketdes, data, sizeof data, flags) == -1) { @@ -343,14 +358,14 @@ } } } - if (FD_ISSET(sock->socketdes, aprset->write)) { + if (APR_FD_ISSET(sock->socketdes, aprset->write)) { revents |= APR_POLLOUT; } /* I am assuming that the except is for out of band data, not a failed * connection on a non-blocking socket. Might be a bad assumption, but * it works for now. rbb. */ - if (FD_ISSET(sock->socketdes, aprset->except)) { + if (APR_FD_ISSET(sock->socketdes, aprset->except)) { revents |= APR_POLLPRI; } @@ -358,24 +373,31 @@ return APR_SUCCESS; } + +/* -- use APR_FD_CLR defined in networkio.h for dynamic + * memory allocation for read, write, and except + */ apr_status_t apr_poll_socket_remove(apr_pollfd_t *aprset, apr_socket_t *sock) { - FD_CLR(sock->socketdes, aprset->read_set); - FD_CLR(sock->socketdes, aprset->except_set); - FD_CLR(sock->socketdes, aprset->write_set); + APR_FD_CLR(sock->socketdes, aprset->read_set); + APR_FD_CLR(sock->socketdes, aprset->except_set); + APR_FD_CLR(sock->socketdes, aprset->write_set); return APR_SUCCESS; } +/* -- use APR_FD_ZERO defined in networkio.h for dynamic + * memory allocation for read, write, and except + */ apr_status_t apr_poll_socket_clear(apr_pollfd_t *aprset, apr_int16_t event) { if (event & APR_POLLIN) { - FD_ZERO(aprset->read_set); + APR_FD_ZERO(aprset->read_set,aprset->highsock); } if (event & APR_POLLPRI) { - FD_ZERO(aprset->except_set); + APR_FD_ZERO(aprset->except_set,aprset->highsock); } if (event & APR_POLLOUT) { - FD_ZERO(aprset->write_set); + APR_FD_ZERO(aprset->write_set,aprset->highsock); } aprset->highsock = 0; return APR_SUCCESS; ================================= +++ ../new/sendrecv.c Wed Sep 5 10:53:20 2001 @@ -62,12 +62,12 @@ apr_status_t apr_wait_for_io_or_timeout(apr_socket_t *sock, int for_read) { struct timeval tv, *tvptr; - fd_set fdset; int srv; do { - FD_ZERO(&fdset); - FD_SET(sock->socketdes, &fdset); + /* -- zero out the the descriptor mem in apr_socket_t */ + APR_FD_ZERO(sock->fdset,sock->fdset_size); + APR_FD_SET(sock->socketdes,sock->fdset); if (sock->timeout < 0) { tvptr = NULL; } @@ -76,9 +76,10 @@ tv.tv_usec = sock->timeout % APR_USEC_PER_SEC; tvptr = &tv; } - srv = select(sock->socketdes + 1, - for_read ? &fdset : NULL, - for_read ? NULL : &fdset, + /* -- use vals set in apr_socket_t structure that were previously dynamically set */ + srv = select(sock->socketdes + 1, + for_read ? (fd_set *)sock->fdset : NULL, + for_read ? NULL : (fd_set*)sock->fdset, NULL, tvptr); /* TODO - timeout should be smaller on repeats of this loop */ ================================================= --- sockets.c Wed Sep 5 10:59:22 2001 +++ ../new/sockets.c Wed Sep 5 10:53:20 2001 @@ -219,6 +219,9 @@ if ((*new)->socketdes < 0) { return errno; } + (*new)->fdset = APR_SOCK_SETSIZE((*new)->socketdes,(*new)->cntxt); + APR_FDSET_SIZE((*new)->fdset_size,(*new)->socketdes); + *(*new)->local_addr = *sock->local_addr; /* fix up any pointers which are no longer valid */ if (sock->local_addr->sa.sin.sin_family == AF_INET) { =================================== --- threaded.c Wed Sep 5 10:59:22 2001 +++ ../new/threaded.c Thu Sep 6 09:52:16 2001 @@ -462,16 +462,6 @@ (void) apr_os_sock_get(&csd, sock); - if (csd >= FD_SETSIZE) { - ap_log_error(APLOG_MARK, APLOG_NOERRNO|APLOG_WARNING, 0, NULL, - "new file descriptor %d is too large; you probably need " - "to rebuild Apache with a larger FD_SETSIZE " - "(currently %d)", - csd, FD_SETSIZE); - apr_socket_close(sock); - return; - } - ap_sock_disable_nagle(sock); current_conn = ap_new_connection(p, ap_server_conf, sock, conn_id); @@ -542,6 +532,8 @@ int curr_pollfd, last_pollfd = 0; apr_pollfd_t *pollset; apr_status_t rv; + int local_hisock = -1; + int csd; free(ti); @@ -551,7 +543,18 @@ worker_thread_count++; apr_lock_release(worker_thread_count_mutex); +#ifdef HAVE_POLL apr_poll_setup(&pollset, num_listensocks+1, tpool); +#else + for(n=0 ; n <= num_listensocks ; ++n){ + (void) apr_os_sock_get(&csd,listensocks[n]); + if (csd > local_hisock) { + local_hisock = csd; + } + } + apr_poll_setup(&pollset, local_hisock, tpool); +#endif + for(n=0 ; n <= num_listensocks ; ++n) apr_poll_socket_add(pollset, listensocks[n], APR_POLLIN); -- ______________________________________________________________ Ryan Bloom rbb@apache.org Covalent Technologies rbb@covalent.net --------------------------------------------------------------