httpd-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Jeff Trawick <trawi...@bellsouth.net>
Subject [PATCH] make prefork.c use an APR lock for accept mutex
Date Wed, 07 Jun 2000 20:07:37 GMT
I've played with this before, but I always seem to get carried away
making changes until I've made a bigger mess...

Note that prefork.c is not currently getting any accept() mutex since
the preprocessor defines it looks for are not defined anywhere.  Thus,
this patch will slow down prefork on systems where we don't need a
mutex for single listen (until somebody tweaks hints.m4 to turn on
SINGLE_LISTEN_NO_SERIALIZED_ACCEPT) but will speed up prefork on
systems with the thundering herd problem with accept().  The
determination of what sort of mutex to use is left to APR.  

Not addressed here: a straightforward way to put something in hints.m4
which will convince APR to use a particular sort of lock 

Index: src/modules/mpm/prefork/prefork.c
===================================================================
RCS file: /cvs/apache/apache-2.0/src/modules/mpm/prefork/prefork.c,v
retrieving revision 1.95
diff -u -r1.95 prefork.c
--- prefork.c	2000/06/02 15:33:18	1.95
+++ prefork.c	2000/06/07 20:00:20
@@ -120,6 +120,7 @@
 
 static int ap_max_requests_per_child=0;
 static char *ap_pid_fname=NULL;
+static ap_lock_t *accept_lock;
 static char *ap_scoreboard_fname=NULL;
 static char *ap_lock_fname;
 static int ap_daemons_to_start=0;
@@ -226,552 +227,46 @@
     exit(code);
 }
 
-#if defined(USE_FCNTL_SERIALIZED_ACCEPT) || defined(USE_FLOCK_SERIALIZED_ACCEPT)
 static void expand_lock_fname(ap_pool_t *p)
 {
     /* XXXX possibly bogus cast */
     ap_lock_fname = ap_psprintf(p, "%s.%lu",
 	ap_server_root_relative(p, ap_lock_fname), (unsigned long)getpid());
 }
-#endif
-
-#if defined (USE_USLOCK_SERIALIZED_ACCEPT)
-
-#include <ulocks.h>
-
-static ulock_t uslock = NULL;
-
-#define accept_mutex_child_init(x)
-
-static void accept_mutex_init(ap_pool_t *p)
-{
-    ptrdiff_t old;
-    usptr_t *us;
-
-
-    /* default is 8, allocate enough for all the children plus the parent */
-    if ((old = usconfig(CONF_INITUSERS, HARD_SERVER_LIMIT + 1)) == -1) {
-	perror("usconfig(CONF_INITUSERS)");
-	exit(-1);
-    }
-
-    if ((old = usconfig(CONF_LOCKTYPE, US_NODEBUG)) == -1) {
-	perror("usconfig(CONF_LOCKTYPE)");
-	exit(-1);
-    }
-    if ((old = usconfig(CONF_ARENATYPE, US_SHAREDONLY)) == -1) {
-	perror("usconfig(CONF_ARENATYPE)");
-	exit(-1);
-    }
-    if ((us = usinit("/dev/zero")) == NULL) {
-	perror("usinit");
-	exit(-1);
-    }
-
-    if ((uslock = usnewlock(us)) == NULL) {
-	perror("usnewlock");
-	exit(-1);
-    }
-}
-
-static void accept_mutex_on(void)
-{
-    switch (ussetlock(uslock)) {
-    case 1:
-	/* got lock */
-	break;
-    case 0:
-	ap_log_error(APLOG_MARK, APLOG_STARTUP | APLOG_NOERRNO, 0, NULL, 
-                     "didn't get lock");
-	clean_child_exit(APEXIT_CHILDFATAL);
-    case -1:
-	perror("ussetlock");
-	clean_child_exit(APEXIT_CHILDFATAL);
-    }
-}
-
-static void accept_mutex_off(void)
-{
-    if (usunsetlock(uslock) == -1) {
-	perror("usunsetlock");
-	clean_child_exit(APEXIT_CHILDFATAL);
-    }
-}
-
-#elif defined (USE_PTHREAD_SERIALIZED_ACCEPT)
-
-/* This code probably only works on Solaris ... but it works really fast
- * on Solaris.  Note that pthread mutexes are *NOT* released when a task
- * dies ... the task has to free it itself.  So we block signals and
- * try to be nice about releasing the mutex.
- */
-
-#include <pthread.h>
-
-static pthread_mutex_t *accept_mutex = (void *)(caddr_t) -1;
-static int have_accept_mutex;
-static sigset_t accept_block_mask;
-static sigset_t accept_previous_mask;
-
-static void accept_mutex_child_cleanup(void *foo)
-{
-    if (accept_mutex != (void *)(caddr_t)-1
-	&& have_accept_mutex) {
-	pthread_mutex_unlock(accept_mutex);
-    }
-}
-
-static void accept_mutex_child_init(ap_pool_t *p)
-{
-    ap_register_cleanup(p, NULL, accept_mutex_child_cleanup, ap_null_cleanup);
-}
-
-static void accept_mutex_cleanup(void *foo)
-{
-    if (accept_mutex != (void *)(caddr_t)-1
-	&& munmap((caddr_t) accept_mutex, sizeof(*accept_mutex))) {
-	perror("munmap");
-    }
-    accept_mutex = (void *)(caddr_t)-1;
-}
-
-static void accept_mutex_init(ap_pool_t *p)
-{
-    pthread_mutexattr_t mattr;
-    int fd;
-
-    fd = open("/dev/zero", O_RDWR);
-    if (fd == -1) {
-	perror("open(/dev/zero)");
-	exit(APEXIT_INIT);
-    }
-    accept_mutex = (pthread_mutex_t *) mmap((caddr_t) 0, sizeof(*accept_mutex),
-				 PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
-    if (accept_mutex == (void *) (caddr_t) - 1) {
-	perror("mmap");
-	exit(APEXIT_INIT);
-    }
-    close(fd);
-    if ((errno = pthread_mutexattr_init(&mattr))) {
-	perror("pthread_mutexattr_init");
-	exit(APEXIT_INIT);
-    }
-    if ((errno = pthread_mutexattr_setpshared(&mattr,
-						PTHREAD_PROCESS_SHARED))) {
-	perror("pthread_mutexattr_setpshared");
-	exit(APEXIT_INIT);
-    }
-    if ((errno = pthread_mutex_init(accept_mutex, &mattr))) {
-	perror("pthread_mutex_init");
-	exit(APEXIT_INIT);
-    }
-    sigfillset(&accept_block_mask);
-    sigdelset(&accept_block_mask, SIGHUP);
-    sigdelset(&accept_block_mask, SIGTERM);
-    sigdelset(&accept_block_mask, SIGUSR1);
-    ap_register_cleanup(p, NULL, accept_mutex_cleanup, ap_null_cleanup);
-}
-
-static void accept_mutex_on(void)
-{
-    int err;
-
-    if (sigprocmask(SIG_BLOCK, &accept_block_mask, &accept_previous_mask)) {
-	perror("sigprocmask(SIG_BLOCK)");
-	clean_child_exit(APEXIT_CHILDFATAL);
-    }
-    if ((err = pthread_mutex_lock(accept_mutex))) {
-	errno = err;
-	perror("pthread_mutex_lock");
-	clean_child_exit(APEXIT_CHILDFATAL);
-    }
-    have_accept_mutex = 1;
-}
-
-static void accept_mutex_off(void)
-{
-    int err;
-
-    if ((err = pthread_mutex_unlock(accept_mutex))) {
-	errno = err;
-	perror("pthread_mutex_unlock");
-	clean_child_exit(APEXIT_CHILDFATAL);
-    }
-    /* There is a slight race condition right here... if we were to die right
-     * now, we'd do another pthread_mutex_unlock.  Now, doing that would let
-     * another process into the mutex.  pthread mutexes are designed to be
-     * fast, as such they don't have protection for things like testing if the
-     * thread owning a mutex is actually unlocking it (or even any way of
-     * testing who owns the mutex).
-     *
-     * If we were to unset have_accept_mutex prior to releasing the mutex
-     * then the race could result in the server unable to serve hits.  Doing
-     * it this way means that the server can continue, but an additional
-     * child might be in the critical section ... at least it's still serving
-     * hits.
-     */
-    have_accept_mutex = 0;
-    if (sigprocmask(SIG_SETMASK, &accept_previous_mask, NULL)) {
-	perror("sigprocmask(SIG_SETMASK)");
-	clean_child_exit(1);
-    }
-}
-
-#elif defined (USE_SYSVSEM_SERIALIZED_ACCEPT)
-
-#include <sys/types.h>
-#include <sys/ipc.h>
-#include <sys/sem.h>
-
-#ifdef NEED_UNION_SEMUN
-/* it makes no sense, but this isn't defined on solaris */
-union semun {
-    long val;
-    struct semid_ds *buf;
-    ushort *array;
-};
-
-#endif
-
-static int sem_id = -1;
-static struct sembuf op_on;
-static struct sembuf op_off;
-
-/* We get a random semaphore ... the lame sysv semaphore interface
- * means we have to be sure to clean this up or else we'll leak
- * semaphores.
- */
-static void accept_mutex_cleanup(void *foo)
-{
-    union semun ick;
-
-    if (sem_id < 0)
-	return;
-    /* this is ignored anyhow */
-    ick.val = 0;
-    semctl(sem_id, 0, IPC_RMID, ick);
-}
-
-#define accept_mutex_child_init(x)
-
-static void accept_mutex_init(ap_pool_t *p)
-{
-    union semun ick;
-    struct semid_ds buf;
-
-    /* acquire the semaphore */
-    sem_id = semget(IPC_PRIVATE, 1, IPC_CREAT | 0600);
-    if (sem_id < 0) {
-	perror("semget");
-	exit(APEXIT_INIT);
-    }
-    ick.val = 1;
-    if (semctl(sem_id, 0, SETVAL, ick) < 0) {
-	perror("semctl(SETVAL)");
-	exit(APEXIT_INIT);
-    }
-    if (!getuid()) {
-	/* restrict it to use only by the appropriate user_id ... not that this
-	 * stops CGIs from acquiring it and dinking around with it.
-	 */
-	buf.sem_perm.uid = unixd_config.user_id;
-	buf.sem_perm.gid = unixd_config.group_id;
-	buf.sem_perm.mode = 0600;
-	ick.buf = &buf;
-	if (semctl(sem_id, 0, IPC_SET, ick) < 0) {
-	    perror("semctl(IPC_SET)");
-	    exit(APEXIT_INIT);
-	}
-    }
-    ap_register_cleanup(p, NULL, accept_mutex_cleanup, ap_null_cleanup);
-
-    /* preinitialize these */
-    op_on.sem_num = 0;
-    op_on.sem_op = -1;
-    op_on.sem_flg = SEM_UNDO;
-    op_off.sem_num = 0;
-    op_off.sem_op = 1;
-    op_off.sem_flg = SEM_UNDO;
-}
-
-static void accept_mutex_on(void)
-{
-    while (semop(sem_id, &op_on, 1) < 0) {
-	if (errno != EINTR) {
-	    perror("accept_mutex_on");
-	    clean_child_exit(APEXIT_CHILDFATAL);
-	}
-    }
-}
-
-static void accept_mutex_off(void)
-{
-    while (semop(sem_id, &op_off, 1) < 0) {
-	if (errno != EINTR) {
-	    perror("accept_mutex_off");
-	    clean_child_exit(APEXIT_CHILDFATAL);
-	}
-    }
-}
-
-#elif defined(USE_FCNTL_SERIALIZED_ACCEPT)
-static struct flock lock_it;
-static struct flock unlock_it;
-
-static int lock_fd = -1;
-
-#define accept_mutex_child_init(x)
-
-/*
- * Initialize mutex lock.
- * Must be safe to call this on a restart.
- */
-static void accept_mutex_init(ap_pool_t *p)
-{
-    ap_file_t *tempfile = NULL;
-    lock_it.l_whence = SEEK_SET;	/* from current point */
-    lock_it.l_start = 0;		/* -"- */
-    lock_it.l_len = 0;			/* until end of file */
-    lock_it.l_type = F_WRLCK;		/* set exclusive/write lock */
-    lock_it.l_pid = 0;			/* pid not actually interesting */
-    unlock_it.l_whence = SEEK_SET;	/* from current point */
-    unlock_it.l_start = 0;		/* -"- */
-    unlock_it.l_len = 0;		/* until end of file */
-    unlock_it.l_type = F_UNLCK;		/* set exclusive/write lock */
-    unlock_it.l_pid = 0;		/* pid not actually interesting */
-
-    expand_lock_fname(p);
-    ap_open(&tempfile, ap_lock_fname, APR_CREATE | APR_WRITE | APR_EXCL,
-            APR_UREAD | APR_UWRITE | APR_GREAD | APR_WREAD, p);
-    ap_get_os_file(&lock_fd, tempfile);
-    if (lock_fd == -1) {
-	perror("open");
-	ap_log_error(APLOG_MARK, APLOG_STARTUP | APLOG_NOERRNO, 0, NULL, 
-                     "Cannot open lock file: %s", ap_lock_fname);
-	exit(APEXIT_INIT);
-    }
-    unlink(ap_lock_fname);
-}
-
-static void accept_mutex_on(void)
-{
-    int ret;
-
-    while ((ret = fcntl(lock_fd, F_SETLKW, &lock_it)) < 0 && errno == EINTR)
{
-	/* nop */
-    }
-
-    if (ret < 0) {
-	ap_log_error(APLOG_MARK, APLOG_EMERG, errno, ap_server_conf,
-		    "fcntl: F_SETLKW: Error getting accept lock, exiting!  "
-		    "Perhaps you need to use the LockFile directive to place "
-		    "your lock file on a local disk!");
-	clean_child_exit(APEXIT_CHILDFATAL);
-    }
-}
-
-static void accept_mutex_off(void)
-{
-    int ret;
 
-    while ((ret = fcntl(lock_fd, F_SETLKW, &unlock_it)) < 0 && errno == EINTR)
{
-	/* nop */
-    }
-    if (ret < 0) {
-	ap_log_error(APLOG_MARK, APLOG_EMERG, errno, ap_server_conf,
-		    "fcntl: F_SETLKW: Error freeing accept lock, exiting!  "
-		    "Perhaps you need to use the LockFile directive to place "
-		    "your lock file on a local disk!");
-	clean_child_exit(APEXIT_CHILDFATAL);
-    }
-}
-
-#elif defined(USE_FLOCK_SERIALIZED_ACCEPT)
-
-static int lock_fd = -1;
-
-static ap_status_t accept_mutex_cleanup(void *foo)
-{
-    unlink(ap_lock_fname);
-
-    return APR_SUCCESS;
-}
-
-/*
- * Initialize mutex lock.
- * Done by each child at it's birth
+/* Initialize mutex lock.
+ * Done by each child at its birth
  */
 static void accept_mutex_child_init(ap_pool_t *p)
 {
-    ap_file_t *tempfile = NULL;
-    ap_status_t ret;
-
-    ret=ap_open(&tempfile, ap_lock_fname, APR_WRITE, APR_UREAD|APR_UWRITE, p);
-    if (ret != APR_SUCCESS) {
-	ap_log_error(APLOG_MARK, APLOG_EMERG, ret, ap_server_conf,
-		    "Child cannot open lock file: %s", ap_lock_fname);
-	clean_child_exit(APEXIT_CHILDINIT);
-    }
-    ap_get_os_file(&lock_fd, tempfile);
+    ap_child_init_lock(&accept_lock, ap_lock_fname, p);
 }
 
-/*
- * Initialize mutex lock.
+/* Initialize mutex lock.
  * Must be safe to call this on a restart.
  */
 static void accept_mutex_init(ap_pool_t *p)
 {
-    ap_file_t *tempfile = NULL;
-    ap_status_t ret;
+    ap_status_t rv;
 
     expand_lock_fname(p);
-    unlink(ap_lock_fname);
-    ret=ap_open(&tempfile, ap_lock_fname, APR_CREATE|APR_WRITE|APR_EXCL,
-	    APR_UREAD|APR_UWRITE, p);
-    if (ret != APR_SUCCESS) {
-	ap_log_error(APLOG_MARK, APLOG_EMERG, ret, ap_server_conf,
-		    "Parent cannot open lock file: %s", ap_lock_fname);
-	exit(APEXIT_INIT);
-    }
-    ap_get_os_file(&lock_fd, tempfile);
-    ap_register_cleanup(p, NULL, accept_mutex_cleanup, ap_null_cleanup);
-}
-
-static void accept_mutex_on(void)
-{
-    int ret;
-
-    while ((ret = flock(lock_fd, LOCK_EX)) < 0 && errno == EINTR)
-	continue;
-
-    if (ret < 0) {
-	ap_log_error(APLOG_MARK, APLOG_EMERG, errno, ap_ap_server_conf,
-		    "flock: LOCK_EX: Error getting accept lock. Exiting!");
-	clean_child_exit(APEXIT_CHILDFATAL);
-    }
-}
-
-static void accept_mutex_off(void)
-{
-    if (flock(lock_fd, LOCK_UN) < 0) {
-	ap_log_error(APLOG_MARK, APLOG_EMERG, errno, ap_server_conf,
-		    "flock: LOCK_UN: Error freeing accept lock. Exiting!");
-	clean_child_exit(APEXIT_CHILDFATAL);
-    }
-}
-
-#elif defined(USE_OS2SEM_SERIALIZED_ACCEPT)
-
-static HMTX lock_sem = -1;
-
-static void accept_mutex_cleanup(void *foo)
-{
-    DosReleaseMutexSem(lock_sem);
-    DosCloseMutexSem(lock_sem);
-}
-
-/*
- * Initialize mutex lock.
- * Done by each child at it's birth
- */
-static void accept_mutex_child_init(ap_pool_t *p)
-{
-    int rc = DosOpenMutexSem(NULL, &lock_sem);
-
-    if (rc != 0) {
-	ap_log_error(APLOG_MARK, APLOG_NOERRNO|APLOG_EMERG, ap_server_conf,
-		    "Child cannot open lock semaphore, rc=%d", rc);
-	clean_child_exit(APEXIT_CHILDINIT);
-    } else {
-        ap_register_cleanup(p, NULL, accept_mutex_cleanup, ap_null_cleanup);
-    }
-}
-
-/*
- * Initialize mutex lock.
- * Must be safe to call this on a restart.
- */
-static void accept_mutex_init(ap_pool_t *p)
-{
-    int rc = DosCreateMutexSem(NULL, &lock_sem, DC_SEM_SHARED, FALSE);
-
-    if (rc != 0) {
-	ap_log_error(APLOG_MARK, APLOG_NOERRNO|APLOG_EMERG, ap_server_conf,
-		    "Parent cannot create lock semaphore, rc=%d", rc);
-	exit(APEXIT_INIT);
-    }
-
-    ap_register_cleanup(p, NULL, accept_mutex_cleanup, ap_null_cleanup);
-}
-
-static void accept_mutex_on(void)
-{
-    int rc = DosRequestMutexSem(lock_sem, SEM_INDEFINITE_WAIT);
-
-    if (rc != 0) {
-	ap_log_error(APLOG_MARK, APLOG_NOERRNO|APLOG_EMERG, ap_server_conf,
-		    "OS2SEM: Error %d getting accept lock. Exiting!", rc);
-	clean_child_exit(APEXIT_CHILDFATAL);
+    rv = ap_create_lock(&accept_lock, APR_MUTEX, APR_CROSS_PROCESS, ap_lock_fname, p);
+    if (rv) {
+	ap_log_error(APLOG_MARK, APLOG_ERR, rv, NULL, "couldn't create accept mutex");
     }
 }
 
-static void accept_mutex_off(void)
-{
-    int rc = DosReleaseMutexSem(lock_sem);
-    
-    if (rc != 0) {
-	ap_log_error(APLOG_MARK, APLOG_NOERRNO|APLOG_EMERG, ap_server_conf,
-		    "OS2SEM: Error %d freeing accept lock. Exiting!", rc);
-	clean_child_exit(APEXIT_CHILDFATAL);
-    }
-}
-
-#elif defined(USE_TPF_CORE_SERIALIZED_ACCEPT)
-
-static int tpf_core_held;
-
-static void accept_mutex_cleanup(void *foo)
-{
-    if(tpf_core_held)
-        coruc(RESOURCE_KEY);
-}
-
-#define accept_mutex_init(x)
-
-static void accept_mutex_child_init(ap_pool_t *p)
-{
-    ap_register_cleanup(p, NULL, accept_mutex_cleanup, ap_null_cleanup);
-    tpf_core_held = 0;
-}
-
 static void accept_mutex_on(void)
 {
-    corhc(RESOURCE_KEY);
-    tpf_core_held = 1;
-    ap_check_signals();
+    ap_status_t rv = ap_lock(accept_lock);
+    ap_assert(!rv);
 }
 
 static void accept_mutex_off(void)
 {
-    coruc(RESOURCE_KEY);
-    tpf_core_held = 0;
-    ap_check_signals();
+    ap_status_t rv = ap_unlock(accept_lock);
+    ap_assert(!rv);
 }
-
-#else
-/* Default --- no serialization.  Other methods *could* go here,
- * as #elifs...
- */
-#if !defined(MULTITHREAD)
-/* Multithreaded systems don't complete between processes for
- * the sockets. */
-#define NO_SERIALIZED_ACCEPT
-#define accept_mutex_child_init(x)
-#define accept_mutex_init(x)
-#define accept_mutex_on()
-#define accept_mutex_off()
-#endif
-#endif
 
 /* On some architectures it's safe to do unserialized accept()s in the single
  * Listen case.  But it's never safe to do it in the case where there's


-- 
Jeff Trawick | trawick@ibm.net | PGP public key at web site:
     http://www.geocities.com/SiliconValley/Park/9289/
          Born in Roswell... married an alien...

Mime
View raw message