httpd-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Roy T. Fielding" <field...@kiwi.ICS.UCI.EDU>
Subject [PATCH] Last lingering cleanup, setsockopt error messages
Date Sun, 06 Apr 1997 14:09:09 GMT
I've gone through all the lingering_close stuff again and found
a few areas where things can now be cleaned up, better documented,
and coded in a less disorienting fashion.  I found that the nonblocking
stuff is not needed after all, the error messages are never useful,
and the timeout needed to be specialized to avoid double-logging.
I also fixed the error messages for the setsockopt options, and
removed one case where sock_disable_nagle was being called for no
good reason.  Also added Unixware, NeXT, and Irix to the NO_LINGCLOSE
list, mostly in conf.h if there is a -D<platform> for it.

.....Roy


Index: Configure
===================================================================
RCS file: /export/home/cvs/apache/src/Configure,v
retrieving revision 1.87
diff -c -r1.87 Configure
*** Configure	1997/04/05 04:31:42	1.87
--- Configure	1997/04/06 13:55:55
***************
*** 347,359 ****
      *-unixware1)
  	DEF_WANTHSREGEX=yes
  	OS='Unixware'
! 	CFLAGS="$CFLAGS -DSVR4"
  	LIBS="$LIBS -lsocket -lnsl -lcrypt"
  	;;
      *-unixware2)
  	DEF_WANTHSREGEX=yes
  	OS='Unixware'
! 	CFLAGS="$CFLAGS -DSVR4"
  	LIBS="$LIBS -lsocket -lnsl -lcrypt"
  	;;
      *-unixware211)
--- 347,359 ----
      *-unixware1)
  	DEF_WANTHSREGEX=yes
  	OS='Unixware'
! 	CFLAGS="$CFLAGS -DSVR4 -DNO_LINGCLOSE"
  	LIBS="$LIBS -lsocket -lnsl -lcrypt"
  	;;
      *-unixware2)
  	DEF_WANTHSREGEX=yes
  	OS='Unixware'
! 	CFLAGS="$CFLAGS -DSVR4 -DNO_LINGCLOSE"
  	LIBS="$LIBS -lsocket -lnsl -lcrypt"
  	;;
      *-unixware211)
Index: conf.h
===================================================================
RCS file: /export/home/cvs/apache/src/conf.h,v
retrieving revision 1.90
diff -c -r1.90 conf.h
*** conf.h	1997/04/02 13:25:16	1.90
--- conf.h	1997/04/06 13:55:55
***************
*** 71,77 ****
  #define NEED_STRCASECMP
  #define NEED_STRDUP
  #define NEED_STRNCASECMP
- #define FNDELAY O_NDELAY
  extern void GETPRIVMODE();
  extern void GETUSERMODE();
  extern char *inet_ntoa();
--- 71,76 ----
***************
*** 113,118 ****
--- 112,118 ----
  #define HAVE_CRYPT_H
  #define NO_LONG_DOUBLE
  #define HAVE_BSTRING_H
+ #define NO_LINGCLOSE
  
  #elif defined(HIUX)
  #define HAVE_SYS_RESOURCE_H
***************
*** 198,203 ****
--- 198,204 ----
  #undef NO_KILLPG
  #define NO_SETSID
  #define NEED_STRDUP
+ #define NO_LINGCLOSE
  #define NO_UNISTD_H
  #undef _POSIX_SOURCE
  #ifndef FD_CLOEXEC
***************
*** 333,338 ****
--- 334,340 ----
  #define USE_FCNTL_SERIALIZED_ACCEPT
  
  #elif defined(UW)
+ #define NO_LINGCLOSE
  #define NO_KILLPG
  #undef  NO_SETSID
  #undef NEED_STRDUP
Index: http_main.c
===================================================================
RCS file: /export/home/cvs/apache/src/http_main.c,v
retrieving revision 1.134
diff -c -r1.134 http_main.c
*** http_main.c	1997/04/06 07:43:39	1.134
--- http_main.c	1997/04/06 13:55:55
***************
*** 301,401 ****
  #define accept_mutex_off()
  #endif
  
- /*
-  * More machine-dependant networking gooo... on some systems,
-  * you've got to be *really* sure that all the packets are acknowledged
-  * before closing the connection.
-  */
- 
- #ifndef NO_LINGCLOSE
- static void lingering_close (request_rec *r)
- {
-     int dummybuf[512];
-     struct timeval tv;
-     fd_set lfds, fds_read, fds_err;
-     int select_rv = 0, read_rv = 0;
-     int lsd;
- 
-     /* Prevent a slow-drip client from holding us here indefinitely */
- 
-     hard_timeout("lingering_close", r);
- 
-     /* Send any leftover data to the client, but never try to again */
- 
-     bflush(r->connection->client);
-     bsetflag(r->connection->client, B_EOUT, 1);
- 
-     /* Close our half of the connection --- send the client a FIN and
-      * set the socket to non-blocking for later reads.
-      */
-     lsd = r->connection->client->fd;
- 
- #ifdef MPE
-     if (((shutdown(lsd, 1)) != 0) || (sfcntl(lsd, F_SETFL, FNDELAY) == -1)) {
- #else
-     if (((shutdown(lsd, 1)) != 0) || (fcntl(lsd, F_SETFL, FNDELAY) == -1)) {
- #endif
- 	/* if it fails, no need to go through the rest of the routine */
- 	if (errno != ENOTCONN)
- 	    log_unixerr("shutdown", NULL, "lingering_close", r->server);
- 	bclose(r->connection->client);
- 	kill_timeout(r);
- 	return;
-     }
- 
-     /* Set up to wait for readable data on socket... */
- 
-     FD_ZERO(&lfds);
-     FD_SET(lsd, &lfds);
- 
-     /* Wait for readable data or error condition on socket;
-      * slurp up any data that arrives...  We exit when we go for 
-      * an interval of tv length without getting any more data, get an
-      * error from select(), get an exception on lsd, get an error or EOF
-      * on a read, or the timer expires.
-      */
- 
-     do {
-         /* We use a 1 second timeout because current (Feb 97) browsers
-          * fail to close a connection after the server closes it.  Thus,
-          * to avoid keeping the child busy, we are only lingering long enough
-          * for a client that is actively sending data on a connection.
-          * This should be sufficient unless the connection is massively
-          * losing packets, in which case we might have missed the RST anyway.
-          * These parameters are reset on each pass, since they might be
-          * changed by select.
-          */
-         tv.tv_sec  = 1;
-         tv.tv_usec = 0;
-         read_rv    = 0;
-         fds_read   = lfds;
-         fds_err    = lfds;
-     
- #ifdef SELECT_NEEDS_CAST
-         select_rv = select(lsd+1, (int*)&fds_read, NULL, (int*)&fds_err, &tv);
- #else
-         select_rv = select(lsd+1, &fds_read, NULL, &fds_err, &tv);
- #endif
-     } while ((select_rv > 0) &&           /* Something to see on socket    */
-              !FD_ISSET(lsd, &fds_err) &&   /* that isn't an error condition
*/
-              FD_ISSET(lsd, &fds_read) &&   /* and is worth trying to read  
*/
-              ((read_rv = read(lsd, dummybuf, sizeof dummybuf)) > 0));
- 
-     /* Log any errors that occurred (client close or reset is not an error) */
-     
-     if (select_rv < 0)
-         log_unixerr("select", NULL, "lingering_close", r->server);
-     else if (read_rv < 0 && errno != ECONNRESET)
-         log_unixerr("read", NULL, "lingering_close", r->server);
- 
-     /* Should now have seen final ack.  Safe to finally kill socket */
- 
-     bclose(r->connection->client);
- 
-     kill_timeout(r);
- }
- #endif /* ndef NO_LINGCLOSE */
- 
  void usage(char *bin)
  {
      fprintf(stderr,"Usage: %s [-d directory] [-f file] [-v] [-h] [-l]\n",bin);
--- 301,306 ----
***************
*** 558,563 ****
--- 463,619 ----
      }
  }
  
+ /*
+  * More machine-dependent networking gooo... on some systems,
+  * you've got to be *really* sure that all the packets are acknowledged
+  * before closing the connection, since the client will not be able
+  * to see the last response if their TCP buffer is flushed by a RST
+  * packet from us, which is what the server's TCP stack will send
+  * if it receives any request data after closing the connection.
+  *
+  * In an ideal world, this function would be accomplished by simply
+  * setting the socket option SO_LINGER and handling it within the
+  * server's TCP stack without blocking the server process on close().
+  * Unfortunately, many OS vendors just haven't figured that out.
+  * For those that have, see USE_SO_LINGER below.  For the others,
+  * we have created a home-brew lingering_close.
+  *
+  * Many operating systems tend to block, puke, or otherwise mishandle
+  * calls to shutdown only half of the connection.  You should define
+  * NO_LINGCLOSE in conf.h if such is the case for your system.
+  */
+ #ifdef USE_SO_LINGER
+ #define NO_LINGCLOSE    /* The two lingering options are exclusive */
+ 
+ static void sock_enable_linger (int s)
+ {
+     struct linger li;
+ 
+     li.l_onoff = 1;
+     li.l_linger = 30;
+ 
+     if (setsockopt(s, SOL_SOCKET, SO_LINGER,
+                    (char *)&li, sizeof(struct linger)) < 0) {
+         log_unixerr("setsockopt", "(SO_LINGER)", NULL, server_conf);
+         /* not a fatal error */
+     }
+ }
+ 
+ #else
+ #define sock_enable_linger(s) /* NOOP */
+ #endif  /* USE_SO_LINGER */
+ 
+ #ifndef NO_LINGCLOSE
+ 
+ /* Special version of timeout for lingering_close */
+ 
+ static void lingerout(sig)
+ int sig;
+ {
+     if (alarms_blocked) {
+ 	alarm_pending = 1;
+ 	return;
+     }
+     
+     if (!current_conn) {
+ #if defined(USE_LONGJMP)
+ 	longjmp(jmpbuffer,1);
+ #else
+ 	siglongjmp(jmpbuffer,1);
+ #endif
+     }
+     current_conn->aborted = 1;
+ }
+     
+ static void linger_timeout ()
+ {
+     const int max_secs_to_linger = 30;
+ 
+     timeout_name = "lingering close";
+     
+     signal(SIGALRM,(void (*)())lingerout);
+     alarm(max_secs_to_linger);
+ }
+ 
+ /* Since many clients will abort a connection instead of closing it,
+  * attempting to log an error message from this routine will only
+  * confuse the webmaster.  There doesn't seem to be any portable way to
+  * distinguish between a dropped connection and something that might be
+  * worth logging.
+  */
+ static void lingering_close (request_rec *r)
+ {
+     int dummybuf[512];
+     struct timeval tv;
+     fd_set lfds, fds_read, fds_err;
+     int select_rv = 0, read_rv = 0;
+     int lsd;
+ 
+     /* Prevent a slow-drip client from holding us here indefinitely */
+ 
+     linger_timeout();
+ 
+     /* Send any leftover data to the client, but never try to again */
+ 
+     bflush(r->connection->client);
+     bsetflag(r->connection->client, B_EOUT, 1);
+ 
+     /* Close our half of the connection --- send the client a FIN */
+ 
+     lsd = r->connection->client->fd;
+ 
+     if ((shutdown(lsd, 1) != 0) || r->connection->aborted) {
+ 	kill_timeout(r);
+ 	bclose(r->connection->client);
+ 	return;
+     }
+ 
+     /* Set up to wait for readable data on socket... */
+ 
+     FD_ZERO(&lfds);
+     FD_SET(lsd, &lfds);
+ 
+     /* Wait for readable data or error condition on socket;
+      * slurp up any data that arrives...  We exit when we go for 
+      * an interval of tv length without getting any more data, get an
+      * error from select(), get an exception on lsd, get an error or EOF
+      * on a read, or the timer expires.
+      */
+ 
+     do {
+         /* We use a 2 second timeout because current (Feb 97) browsers
+          * fail to close a connection after the server closes it.  Thus,
+          * to avoid keeping the child busy, we are only lingering long enough
+          * for a client that is actively sending data on a connection.
+          * This should be sufficient unless the connection is massively
+          * losing packets, in which case we might have missed the RST anyway.
+          * These parameters are reset on each pass, since they might be
+          * changed by select.
+          */
+         tv.tv_sec  = 2;
+         tv.tv_usec = 0;
+         read_rv    = 0;
+         fds_read   = lfds;
+         fds_err    = lfds;
+     
+ #ifdef SELECT_NEEDS_CAST
+         select_rv = select(lsd+1, (int*)&fds_read, NULL, (int*)&fds_err, &tv);
+ #else
+         select_rv = select(lsd+1, &fds_read, NULL, &fds_err, &tv);
+ #endif
+     } while ((select_rv > 0) &&            /* Something to see on socket    */
+              !FD_ISSET(lsd, &fds_err) &&   /* that isn't an error condition
*/
+              FD_ISSET(lsd, &fds_read) &&   /* and is worth trying to read  
*/
+              ((read_rv = read(lsd, dummybuf, sizeof dummybuf)) > 0));
+ 
+     /* Should now have seen final ack.  Safe to finally kill socket */
+ 
+     bclose(r->connection->client);
+ 
+     kill_timeout(r);
+ }
+ #endif /* ndef NO_LINGCLOSE */
+ 
  /*****************************************************************
   *
   * Dealing with the scoreboard... a lot of these variables are global
***************
*** 1566,1573 ****
  
      if (setsockopt(s, IPPROTO_TCP, TCP_NODELAY, (char*)&just_say_no,
                     sizeof(int)) < 0) {
! 	perror ("setsockopt(TCP_NODELAY)");
! 	fprintf(stderr, "httpd: could not set socket option TCP_NODELAY\n");
      }
  }
  #else
--- 1622,1628 ----
  
      if (setsockopt(s, IPPROTO_TCP, TCP_NODELAY, (char*)&just_say_no,
                     sizeof(int)) < 0) {
!         log_unixerr("setsockopt", "(TCP_NODELAY)", NULL, server_conf);
      }
  }
  #else
***************
*** 1739,1746 ****
  	    continue;
  	}
  
- 	sock_disable_nagle(csd);
- 
  	(void)update_child_status(child_num, SERVER_BUSY_READ,
  	                          (request_rec*)NULL);
  
--- 1794,1799 ----
***************
*** 1840,1902 ****
      return 0;
  }
  
! static int
! make_sock(pool *pconf, const struct sockaddr_in *server)
  {
      int s;
      int one = 1;
  
      if ((s = socket(AF_INET,SOCK_STREAM,IPPROTO_TCP)) == -1) {
!         perror("socket");
!         fprintf(stderr,"httpd: could not get socket\n");
          exit(1);
      }
  
!     note_cleanups_for_fd (pconf, s); /* arrange to close on exec or restart */
      
  #ifndef MPE
  /* MPE does not support SO_REUSEADDR and SO_KEEPALIVE */
      if (setsockopt(s, SOL_SOCKET,SO_REUSEADDR,(char *)&one,sizeof(int)) < 0) {
! 	perror("setsockopt(SO_REUSEADDR)");
! 	fprintf(stderr,"httpd: could not set socket option SO_REUSEADDR\n");
          exit(1);
      }
      one = 1;
      if (setsockopt(s, SOL_SOCKET,SO_KEEPALIVE,(char *)&one,sizeof(int)) < 0) {
! 	perror("setsockopt(SO_KEEPALIVE)"); 
!         fprintf(stderr,"httpd: could not set socket option SO_KEEPALIVE\n"); 
!         exit(1); 
      }
  #endif
  
      sock_disable_nagle(s);
      
- #ifdef USE_SO_LINGER   /* If puts don't complete, you could try this. */
-     {
- 	/* Unfortunately, SO_LINGER causes problems as severe as it
- 	 * cures on many of the affected systems; now trying the
- 	 * lingering_close trick (see routine by that name above)
- 	 * instead...
- 	 */
- 	struct linger li;
- 	li.l_onoff = 1;
- 	li.l_linger = 900;
- 
- 	if (setsockopt(s, SOL_SOCKET, SO_LINGER,
- 	               (char *)&li, sizeof(struct linger)) < 0) {
- 	    perror("setsockopt(SO_LINGER)");
- 	    fprintf(stderr,"httpd: could not set socket option SO_LINGER\n");
- 	    exit(1);
- 	}
-     }
- #endif  /* USE_SO_LINGER */
- 
      /*
       * To send data over high bandwidth-delay connections at full
!      * speed we must the TCP window to open wide enough to keep the
!      * pipe full.  Default the default window size on many systems
       * is only 4kB.  Cross-country WAN connections of 100ms
!      * at 1Mb/s are not impossible for well connected sites in 1995.
       * If we assume 100ms cross-country latency,
       * a 4kB buffer limits throughput to 40kB/s.
       *
--- 1893,1933 ----
      return 0;
  }
  
! static int make_sock(pool *pconf, const struct sockaddr_in *server)
  {
      int s;
      int one = 1;
  
      if ((s = socket(AF_INET,SOCK_STREAM,IPPROTO_TCP)) == -1) {
!         log_unixerr("socket", NULL, "Failed to get a socket, exiting child",
!                     server_conf);
          exit(1);
      }
  
!     note_cleanups_for_fd(pconf, s); /* arrange to close on exec or restart */
      
  #ifndef MPE
  /* MPE does not support SO_REUSEADDR and SO_KEEPALIVE */
      if (setsockopt(s, SOL_SOCKET,SO_REUSEADDR,(char *)&one,sizeof(int)) < 0) {
!         log_unixerr("setsockopt", "(SO_REUSEADDR)", NULL, server_conf);
          exit(1);
      }
      one = 1;
      if (setsockopt(s, SOL_SOCKET,SO_KEEPALIVE,(char *)&one,sizeof(int)) < 0) {
!         log_unixerr("setsockopt", "(SO_KEEPALIVE)", NULL, server_conf);
!         exit(1);
      }
  #endif
  
      sock_disable_nagle(s);
+     sock_enable_linger(s);
      
      /*
       * To send data over high bandwidth-delay connections at full
!      * speed we must force the TCP window to open wide enough to keep the
!      * pipe full.  The default window size on many systems
       * is only 4kB.  Cross-country WAN connections of 100ms
!      * at 1Mb/s are not impossible for well connected sites.
       * If we assume 100ms cross-country latency,
       * a 4kB buffer limits throughput to 40kB/s.
       *
***************
*** 1908,1921 ****
       *
       * -John Heidemann <johnh@isi.edu> 25-Oct-96
       *
-      *
       * If no size is specified, use the kernel default.
       */
      if (server_conf->send_buffer_size) {
          if (setsockopt(s, SOL_SOCKET, SO_SNDBUF,
                (char *)&server_conf->send_buffer_size, sizeof(int)) < 0) {
! 	    perror("setsockopt(SO_SNDBUF), using default buffer size"); 
! 	    /* Fail soft. */
  	}
      }
  
--- 1939,1953 ----
       *
       * -John Heidemann <johnh@isi.edu> 25-Oct-96
       *
       * If no size is specified, use the kernel default.
       */
      if (server_conf->send_buffer_size) {
          if (setsockopt(s, SOL_SOCKET, SO_SNDBUF,
                (char *)&server_conf->send_buffer_size, sizeof(int)) < 0) {
!             log_unixerr("setsockopt", "(SO_SNDBUF)",
!                         "Failed to set SendBufferSize, using default",
!                         server_conf);
! 	    /* not a fatal error */
  	}
      }
  

Mime
View raw message