httpd-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Dean Gaudet <dgau...@arctic.org>
Subject graceful restarts, take 3
Date Sun, 20 Apr 1997 02:32:16 GMT
(No you didn't miss take 2, I didn't send it.)

This patch, superceding the one I posted yesterday, keeps the same
scoreboard across a graceful restart.  It adds another state --
SERVER_GRACEFUL for the purposes of debugging graceful restarts, servers
in this state were busy when they were sent the SIGUSR1 to just_die() and
are "gracefully" handling their last request.

A side-effect of keeping the scoreboard means that statistics aren't
reset.  Rather than reset them all (it's more complicated than just a
memset) I chose to just not reset the server restart_time.  So none of the
statistics will be clearned until you do a SIGHUP or hard restart the
server.

Another side-effect of keeping the scoreboard is that it's tricky to make
sure StartServers children are started immediately.  See the comments in
standalone_main for more details.  My reasoning for how I did this was
that at the time the USR1 is received the server is likely to be "tuned" 
to the current load (total #servers) and so it should be able to climb
right back up to that number really rapidly.  That is what happens, except
if the total #servers pre-restart is fewer than StartServers it will pick
up the slack rapidly after it thinks all the previous generation have been
reaped.

The important properties of MaxClients/MinSpareServers/MaxSpareServers
should all be implemented properly across a graceful restart now.

I've also made the child_main code more aggressive about dying on USR1... 
in particular I wasn't happy with the old servers sitting in KEEPALIVE
after a USR1.  But the sacrifice here is that it's possible for us to have
read the request before we die, and we'll die without telling the client
anything.  The window of opportunity could be lessened by putting a signal
(SIGUSR1, SIG_IGN); into the read_request_line code. 

I've been running this one on a 10 hit/second server with hourly USR1s for
12 hours with nothing bad to report yet.  Linux with LINUX_TWEAK. 

Dean

Index: http_main.c
===================================================================
RCS file: /export/home/cvs/apache/src/http_main.c,v
retrieving revision 1.137
diff -c -3 -r1.137 http_main.c
*** http_main.c	1997/04/12 04:24:57	1.137
--- http_main.c	1997/04/20 02:13:37
***************
*** 1173,1195 ****
  #if 1
  
  static int wait_or_timeout(int *status)
!     {
  #ifndef NEED_WAITPID
      int ret;
  
      ret=waitpid(-1,status,WNOHANG);
!     if(ret <= 0)
! 	{
  	sleep(1);
  	return -1;
! 	}
      return ret;
  #else
      if(!reap_children())
  	sleep(1);
      return -1;
  #endif
!     }
  
  #else
  
--- 1173,1194 ----
  #if 1
  
  static int wait_or_timeout(int *status)
! {
  #ifndef NEED_WAITPID
      int ret;
  
      ret=waitpid(-1,status,WNOHANG);
!     if(ret <= 0) {
  	sleep(1);
  	return -1;
!     }
      return ret;
  #else
      if(!reap_children())
  	sleep(1);
      return -1;
  #endif
! }
  
  #else
  
***************
*** 1420,1426 ****
  }
  
  void graceful_restart()
!     {
      scoreboard_image->global.exit_generation=generation;
      is_graceful=1;
      update_scoreboard_global();
--- 1419,1427 ----
  }
  
  void graceful_restart()
! {
!     signal (SIGALRM, SIG_IGN);
!     alarm (0);
      scoreboard_image->global.exit_generation=generation;
      is_graceful=1;
      update_scoreboard_global();
***************
*** 1429,1435 ****
  #else
      siglongjmp(restart_buffer,1);
  #endif
!     }
  
  void set_signals()
  {
--- 1430,1436 ----
  #else
      siglongjmp(restart_buffer,1);
  #endif
! }
  
  void set_signals()
  {
***************
*** 1450,1455 ****
--- 1451,1464 ----
      sa.sa_handler=(void (*)())sig_term;
      if(sigaction(SIGTERM,&sa,NULL) < 0)
  	log_unixerr("sigaction(SIGTERM)", NULL, NULL, server_conf);
+ 
+     /* wait_or_timeout uses sleep() which could deliver a SIGALRM just as
+      * we're trying to process the restart requests.  That's not good.
+      * restart and graceful_restart both clean out the SIGALRM handler,
+      * but this totally avoids the race condition between when the restart
+      * request is made and when the handler is invoked.
+      */
+     sigaddset (&sa.sa_mask, SIGALRM);
      sa.sa_handler=(void (*)())restart;
      if(sigaction(SIGHUP,&sa,NULL) < 0)
  	log_unixerr("sigaction(SIGHUP)", NULL, NULL, server_conf);
***************
*** 1715,1720 ****
--- 1724,1735 ----
  	BUFF *conn_io;
  	request_rec *r;
        
+ 	/* Prepare to receive a SIGUSR1 due to graceful restart so that
+ 	 * we can exit cleanly.  Since we're between connections right
+ 	 * now it's the right time to exit, but we might be blocked in a
+ 	 * system call when the graceful restart request is made. */
+ 	signal (SIGUSR1, (void (*)())just_die);
+ 
          /*
           * (Re)initialize this child to a pre-connection state.
           */
***************
*** 1840,1846 ****
--- 1855,1875 ----
           */
  
          for (;;) {
+ 	    /* In case we get a graceful restart while we're blocked
+ 	     * waiting for the request.
+ 	     * XXX: This isn't perfect, we might actually read the
+ 	     * request and then just die without saying anything to
+ 	     * the client.
+ 	     */
+ 	    signal (SIGUSR1, (void (*)())just_die);
+ 
              r = read_request(current_conn);
+ 
+ 	    /* ok we've read the request... it's a little too late
+ 	     * to do a graceful restart, so ignore them for now.
+ 	     */
+ 	    signal (SIGUSR1, SIG_IGN);
+ 
              (void)update_child_status(child_num, SERVER_BUSY_WRITE, r);
  
              if (r) process_request(r); /* else premature EOF --- ignore */
***************
*** 2050,2061 ****
   * Executive routines.
   */
  
- static int num_children = 0;
- 
  void standalone_main(int argc, char **argv)
  {
      struct sockaddr_in sa_server;
      int saved_sd;
  
      standalone = 1;
      sd = listenmaxfd = -1;
--- 2079,2089 ----
   * Executive routines.
   */
  
  void standalone_main(int argc, char **argv)
  {
      struct sockaddr_in sa_server;
      int saved_sd;
+     int remaining_children_to_start;
  
      standalone = 1;
      sd = listenmaxfd = -1;
***************
*** 2071,2076 ****
--- 2099,2105 ----
      ++generation;
  
      signal (SIGHUP, SIG_IGN);	/* Until we're done (re)reading config */
+     signal (SIGUSR1, SIG_IGN);
      
      if(!one_process && !is_graceful)
      {
***************
*** 2082,2094 ****
          log_unixerr ("killpg SIGHUP", NULL, NULL, server_conf);
      }
      
!     if(is_graceful)
! 	{
  	/* USE WITH EXTREME CAUTION. Graceful restarts are known to break */
  	/*  problems will be dealt with in a future release */
  	log_error("SIGUSR1 received.  Doing graceful restart",server_conf);
  	kill_cleanups_for_fd(pconf,sd);
  	}
      else if (sd != -1 || listenmaxfd != -1) {
  	reclaim_child_processes(); /* Not when just starting up */
  	log_error ("SIGHUP received.  Attempting to restart", server_conf);
--- 2111,2147 ----
          log_unixerr ("killpg SIGHUP", NULL, NULL, server_conf);
      }
      
!     if(is_graceful) {
! 	int i;
! 
  	/* USE WITH EXTREME CAUTION. Graceful restarts are known to break */
  	/*  problems will be dealt with in a future release */
  	log_error("SIGUSR1 received.  Doing graceful restart",server_conf);
  	kill_cleanups_for_fd(pconf,sd);
+ 	if (!one_process) {
+ #ifndef NO_KILLPG
+ 	    if (killpg(pgrp, SIGUSR1) < 0)    /* kill off the idle ones */
+ #else
+ 	    if (kill(-pgrp, SIGUSR1) < 0)
+ #endif
+ 		log_unixerr ("killpg SIGUSR1", NULL, NULL, server_conf);
  	}
+ 	/* This is mostly for debugging... so that we know what is still
+ 	 * gracefully dealing with existing request.
+ 	 * XXX: clean this up a bit?
+ 	 */
+ 	sync_scoreboard_image();
+ 	for (i = 0; i < daemons_limit; ++i ) {
+ 	    if (scoreboard_image->servers[i].status != SERVER_DEAD) {
+ 		scoreboard_image->servers[i].status = SERVER_GRACEFUL;
+ 	    }
+ 	}
+ #if !defined(HAVE_MMAP) && !defined(HAVE_SHMGET)
+ 	lseek (scoreboard_fd, 0L, 0);
+ 	force_write (scoreboard_fd, (char*)scoreboard_image,
+ 		    sizeof(*scoreboard_image));
+ #endif
+     }
      else if (sd != -1 || listenmaxfd != -1) {
  	reclaim_child_processes(); /* Not when just starting up */
  	log_error ("SIGHUP received.  Attempting to restart", server_conf);
***************
*** 2096,2102 ****
      
      copy_listeners(pconf);
      saved_sd=sd;
!     restart_time = time(NULL);
      clear_pool (pconf);
      ptrans = make_sub_pool (pconf);
      
--- 2149,2157 ----
      
      copy_listeners(pconf);
      saved_sd=sd;
!     if (!is_graceful) {
! 	restart_time = time(NULL);
!     }
      clear_pool (pconf);
      ptrans = make_sub_pool (pconf);
      
***************
*** 2104,2110 ****
      open_logs(server_conf, pconf);
      set_group_privs();
      accept_mutex_init(pconf);
!     reinit_scoreboard(pconf);
      
      default_server_hostnames (server_conf);
  
--- 2159,2167 ----
      open_logs(server_conf, pconf);
      set_group_privs();
      accept_mutex_init(pconf);
!     if (!is_graceful) {
! 	reinit_scoreboard(pconf);
!     }
      
      default_server_hostnames (server_conf);
  
***************
*** 2144,2164 ****
      set_signals();
      log_pid(pconf, pid_fname);
  
-     num_children = 0;
-     
      if (daemons_max_free < daemons_min_free + 1) /* Don't thrash... */
  	daemons_max_free = daemons_min_free + 1;
  
!     while (num_children < daemons_to_start && num_children < daemons_limit)
{
! 	make_child(server_conf, num_children++);
      }
  
      log_error ("Server configured -- resuming normal operations", server_conf);
!     
      while (1) {
  	int status, child_slot;
  	int pid = wait_or_timeout(&status);
! 	
  	if (pid >= 0) {
  	    /* Child died... note that it's gone in the scoreboard. */
  	    sync_scoreboard_image();
--- 2201,2237 ----
      set_signals();
      log_pid(pconf, pid_fname);
  
      if (daemons_max_free < daemons_min_free + 1) /* Don't thrash... */
  	daemons_max_free = daemons_min_free + 1;
  
!     /* If we're doing a graceful_restart then we're going to see a lot
!      * of children exiting immediately when we get into the main loop
!      * below (because we just sent them SIGUSR1).  This happens pretty
!      * rapidly... and for each one that exits we'll start a new one until
!      * we reach at least daemons_min_free.  But we may be permitted to
!      * start more than that, so we'll just keep track of how many we're
!      * supposed to start up without the 1 second penalty between each fork.
!      */
!     remaining_children_to_start = daemons_to_start;
!     if( remaining_children_to_start > daemons_limit ) {
! 	remaining_children_to_start = daemons_limit;
!     }
!     if (!is_graceful) {
! 	while (remaining_children_to_start) {
! 	    --remaining_children_to_start;
! 	    make_child(server_conf, remaining_children_to_start);
! 	}
      }
  
      log_error ("Server configured -- resuming normal operations", server_conf);
! 
      while (1) {
  	int status, child_slot;
  	int pid = wait_or_timeout(&status);
! 
! 	/* XXX: if it takes longer than 1 second for all our children to start
! 	 * up and get into IDLE state then we may spawn an extra child
! 	 */
  	if (pid >= 0) {
  	    /* Child died... note that it's gone in the scoreboard. */
  	    sync_scoreboard_image();
***************
*** 2167,2176 ****
  	    if (child_slot >= 0)
  		(void)update_child_status (child_slot, SERVER_DEAD,
  		 (request_rec*)NULL);
!         }
  
  	sync_scoreboard_image();
! 	if ((count_idle_servers() < daemons_min_free)
  	 && (child_slot = find_free_child_num()) >= 0
  	 && child_slot < daemons_limit) {
  	    Explain1("Starting new child in slot %d",child_slot);
--- 2240,2272 ----
  	    if (child_slot >= 0)
  		(void)update_child_status (child_slot, SERVER_DEAD,
  		 (request_rec*)NULL);
!         } else if (remaining_children_to_start) {
! 	    /* we hit a 1 second timeout in which none of the previous
! 	     * generation of children needed to be reaped... so assume
! 	     * they're all done, and pick up the slack if any is left.
! 	     */
! 	    while (remaining_children_to_start > 0) {
! 		child_slot = find_free_child_num();
! 		if (child_slot < 0 || child_slot >= daemons_limit) {
! 		    remaining_children_to_start = 0;
! 		    break;
! 		}
! 		if (make_child (server_conf, child_slot) < 0) {
! 		    remaining_children_to_start = 0;
! 		    break;
! 		}
! 		--remaining_children_to_start;
! 	    }
! 	    /* In any event we really shouldn't do the code below because
! 	     * few of the servers we just started are in the IDLE state
! 	     * yet, so we'd mistakenly create an extra server.
! 	     */
! 	    continue;
! 	}
  
  	sync_scoreboard_image();
! 	if ( ( remaining_children_to_start
! 	    || (count_idle_servers() < daemons_min_free) )
  	 && (child_slot = find_free_child_num()) >= 0
  	 && child_slot < daemons_limit) {
  	    Explain1("Starting new child in slot %d",child_slot);
***************
*** 2184,2198 ****
  	    }
  
  	}
! 
! 	/*
! 	if(scoreboard_image->global.please_exit && !count_live_servers())
! #if defined(USE_LONGJMP)
! 	    longjmp(restart_buffer,1);
! #else
! 	    siglongjmp(restart_buffer,1);
! #endif
! 	*/
      }
  
  } /* standalone_main */
--- 2280,2288 ----
  	    }
  
  	}
! 	if (remaining_children_to_start) {
! 	    --remaining_children_to_start;
! 	}
      }
  
  } /* standalone_main */
Index: mod_status.c
===================================================================
RCS file: /export/home/cvs/apache/src/mod_status.c,v
retrieving revision 1.45
diff -c -3 -r1.45 mod_status.c
*** mod_status.c	1997/04/06 07:43:42	1.45
--- mod_status.c	1997/04/20 02:13:37
***************
*** 227,232 ****
--- 227,233 ----
      status[SERVER_BUSY_KEEPALIVE]='K';
      status[SERVER_BUSY_LOG]='L';
      status[SERVER_BUSY_DNS]='D';
+     status[SERVER_GRACEFUL]='G';
  
      if (r->method_number != M_GET) return NOT_IMPLEMENTED;
      r->content_type = "text/html";
***************
*** 279,285 ****
  	    ready++;
          else if (res == SERVER_BUSY_READ || res==SERVER_BUSY_WRITE || 
  		 res == SERVER_STARTING || res==SERVER_BUSY_KEEPALIVE ||
! 		 res == SERVER_BUSY_LOG || res==SERVER_BUSY_DNS)
  	    busy++;
  #if defined(STATUS)
          lres = score_record.access_count;
--- 280,287 ----
  	    ready++;
          else if (res == SERVER_BUSY_READ || res==SERVER_BUSY_WRITE || 
  		 res == SERVER_STARTING || res==SERVER_BUSY_KEEPALIVE ||
! 		 res == SERVER_BUSY_LOG || res==SERVER_BUSY_DNS ||
! 		 res == SERVER_GRACEFUL)
  	    busy++;
  #if defined(STATUS)
          lres = score_record.access_count;
***************
*** 407,412 ****
--- 409,415 ----
  	rputs("\"<B><code>K</code></B>\" Keepalive (read), \n",r);
  	rputs("\"<B><code>D</code></B>\" DNS Lookup,<BR>\n",r);
  	rputs("\"<B><code>L</code></B>\" Logging, \n",r);
+ 	rputs("\"<B><code>G</code></B>\" Gracefully finishing, \n",r);
  	rputs("\"<B><code>.</code></B>\" Open slot with no current process<P>\n",r);
      }
  
***************
*** 468,473 ****
--- 471,482 ----
  		        case SERVER_DEAD:
  		            rputs("Dead",r);
  		            break;
+ 			case SERVER_GRACEFUL:
+ 			    rputs("Graceful",r);
+ 			    break;
+ 			default:
+ 			    rputs("?STATE?",r);
+ 			    break;
  		    }
  #ifdef __EMX__
                      /* Allow for OS/2 not having CPU stats */
***************
*** 521,526 ****
--- 530,541 ----
  		        case SERVER_DEAD:
  		            rputs("<td>.",r);
  		            break;
+ 			case SERVER_GRACEFUL:
+ 			    rputs("<td>G",r);
+ 			    break;
+ 			default:
+ 			    rputs("<td>?",r);
+ 			    break;
  		    }
  #ifdef __EMX__
  	            /* Allow for OS/2 not having CPU stats */
Index: scoreboard.h
===================================================================
RCS file: /export/home/cvs/apache/src/scoreboard.h,v
retrieving revision 1.20
diff -c -3 -r1.20 scoreboard.h
*** scoreboard.h	1997/01/01 18:10:44	1.20
--- scoreboard.h	1997/04/20 02:13:37
***************
*** 71,76 ****
--- 71,77 ----
  #define SERVER_BUSY_KEEPALIVE 5 /* Waiting for more requests via keepalive */
  #define SERVER_BUSY_LOG 6       /* Logging the request */
  #define SERVER_BUSY_DNS 7       /* Looking up a hostname */
+ #define SERVER_GRACEFUL 8	/* server is gracefully finishing request */
  
  typedef struct {
      pid_t pid;


Mime
View raw message