httpd-cvs mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pque...@apache.org
Subject svn commit: r721987 - in /httpd/httpd/trunk: CHANGES modules/proxy/config.m4 modules/proxy/mod_lbmethod_heartbeat.c
Date Mon, 01 Dec 2008 07:25:11 GMT
Author: pquerna
Date: Sun Nov 30 23:25:11 2008
New Revision: 721987

URL: http://svn.apache.org/viewvc?rev=721987&view=rev
Log:
Add a new module to read in the heartbeat file and do load balancing for
mod_proxy based upon it.

Added:
    httpd/httpd/trunk/modules/proxy/mod_lbmethod_heartbeat.c   (contents, props changed)
      - copied, changed from r721944, httpd/httpd/trunk/modules/proxy/examples/mod_lbmethod_rr.c
Modified:
    httpd/httpd/trunk/CHANGES
    httpd/httpd/trunk/modules/proxy/config.m4

Modified: httpd/httpd/trunk/CHANGES
URL: http://svn.apache.org/viewvc/httpd/httpd/trunk/CHANGES?rev=721987&r1=721986&r2=721987&view=diff
==============================================================================
--- httpd/httpd/trunk/CHANGES [utf-8] (original)
+++ httpd/httpd/trunk/CHANGES [utf-8] Sun Nov 30 23:25:11 2008
@@ -2,6 +2,9 @@
 Changes with Apache 2.3.0
 [ When backported to 2.2.x, remove entry from this file ]
 
+  *) mod_lbmethod_heartbeat: New module to load balance mod_proxy workers
+     based on heartbeats. [Paul Querna]
+
   *) mod_heartmonitor: New module to collect heartbeats, and write out a file
      so that other modules can load balance traffic as needed. [Paul Querna]
 

Modified: httpd/httpd/trunk/modules/proxy/config.m4
URL: http://svn.apache.org/viewvc/httpd/httpd/trunk/modules/proxy/config.m4?rev=721987&r1=721986&r2=721987&view=diff
==============================================================================
--- httpd/httpd/trunk/modules/proxy/config.m4 (original)
+++ httpd/httpd/trunk/modules/proxy/config.m4 Sun Nov 30 23:25:11 2008
@@ -19,6 +19,7 @@
 proxy_fcgi_objs="mod_proxy_fcgi.lo"
 proxy_ajp_objs="mod_proxy_ajp.lo ajp_header.lo ajp_link.lo ajp_msg.lo ajp_utils.lo"
 proxy_balancer_objs="mod_proxy_balancer.lo"
+proxy_lb_hb_objs="mod_lbmethod_heartbeat.lo"
 
 case "$host" in
   *os2*)
@@ -30,6 +31,7 @@
     proxy_fcgi_objs="$proxy_fcgi_objs mod_proxy.la"
     proxy_ajp_objs="$proxy_ajp_objs mod_proxy.la"
     proxy_balancer_objs="$proxy_balancer_objs mod_proxy.la"
+    proxy_lb_hb_objs="$proxy_lb_hb_objs mod_proxy.la"
     ;;
 esac
 
@@ -39,6 +41,8 @@
 APACHE_MODULE(proxy_fcgi, Apache proxy FastCGI module, $proxy_fcgi_objs, , $proxy_mods_enable)
 APACHE_MODULE(proxy_ajp, Apache proxy AJP module, $proxy_ajp_objs, , $proxy_mods_enable)
 APACHE_MODULE(proxy_balancer, Apache proxy BALANCER module, $proxy_balancer_objs, , $proxy_mods_enable)
+APACHE_MODULE(lbmethod_heartbeat, Apache proxy Load balancing from Heartbeats, $proxy_lb_hb_objs,
, $proxy_mods_enable)
+
 
 AC_DEFUN([CHECK_SERF], [
   AC_MSG_CHECKING(for serf)

Copied: httpd/httpd/trunk/modules/proxy/mod_lbmethod_heartbeat.c (from r721944, httpd/httpd/trunk/modules/proxy/examples/mod_lbmethod_rr.c)
URL: http://svn.apache.org/viewvc/httpd/httpd/trunk/modules/proxy/mod_lbmethod_heartbeat.c?p2=httpd/httpd/trunk/modules/proxy/mod_lbmethod_heartbeat.c&p1=httpd/httpd/trunk/modules/proxy/examples/mod_lbmethod_rr.c&r1=721944&r2=721987&rev=721987&view=diff
==============================================================================
--- httpd/httpd/trunk/modules/proxy/examples/mod_lbmethod_rr.c (original)
+++ httpd/httpd/trunk/modules/proxy/mod_lbmethod_heartbeat.c Sun Nov 30 23:25:11 2008
@@ -14,106 +14,351 @@
  * limitations under the License.
  */
 
-/* Round Robin lbmethod EXAMPLE module for Apache proxy */
-
-/* NOTE: This is designed simply to provide some info on how to create
-         extra lbmethods via sub-modules... This code is ugly
-         and needs work to actually do round-robin "right"
-         but that is left as an exercise for the reader */
-
 #include "mod_proxy.h"
 #include "scoreboard.h"
 #include "ap_mpm.h"
 #include "apr_version.h"
 #include "apr_hooks.h"
 
-#if APR_HAVE_UNISTD_H
-#include <unistd.h> /* for getpid() */
-#endif
-
-module AP_MODULE_DECLARE_DATA proxy_balancer_rr_module;
-
-typedef struct {
-    int index;
-} rr_data ;
+module AP_MODULE_DECLARE_DATA lbmethod_heartbeat_module;
+
+typedef struct lb_hb_ctx_t
+{
+    const char *path;
+} lb_hb_ctx_t;
+
+typedef struct hb_server_t {
+    const char *ip;
+    int busy;
+    int ready;
+    int seen;
+    proxy_worker *worker;
+} hb_server_t;
+
+static void
+argstr_to_table(apr_pool_t *p, char *str, apr_table_t *parms)
+{
+    char *key;
+    char *value;
+    char *strtok_state;
+    
+    key = apr_strtok(str, "&", &strtok_state);
+    while (key) {
+        value = strchr(key, '=');
+        if (value) {
+            *value = '\0';      /* Split the string in two */
+            value++;            /* Skip passed the = */
+        }
+        else {
+            value = "1";
+        }
+        ap_unescape_url(key);
+        ap_unescape_url(value);
+        apr_table_set(parms, key, value);
+        /*
+         ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r,
+         "Found query arg: %s = %s", key, value);
+         */
+        key = apr_strtok(NULL, "&", &strtok_state);
+    }
+}
+
+static apr_status_t read_heartbeats(const char *path, apr_hash_t *servers,
+                                    apr_pool_t *pool)
+{
+    apr_finfo_t fi;
+    apr_status_t rv;
+    apr_file_t *fp;
+
+    if (!path) {
+        return APR_SUCCESS;
+    }
+
+    rv = apr_file_open(&fp, path, APR_READ|APR_BINARY|APR_BUFFERED,
+                       APR_OS_DEFAULT, pool);
+
+    if (rv) {
+        return rv;
+    }
+
+    rv = apr_file_info_get(&fi, APR_FINFO_SIZE, fp);
+
+    if (rv) {
+        return rv;
+    }
+
+    {
+        char *t;
+        int lineno = 0;
+        apr_bucket_alloc_t *ba = apr_bucket_alloc_create(pool);
+        apr_bucket_brigade *bb = apr_brigade_create(pool, ba);
+        apr_bucket_brigade *tmpbb = apr_brigade_create(pool, ba);
+        apr_table_t *hbt = apr_table_make(pool, 10);
+
+        apr_brigade_insert_file(bb, fp, 0, fi.size, pool);
+
+        do {
+            hb_server_t *server;
+            char buf[4096];
+            apr_size_t bsize = sizeof(buf);
+
+            apr_brigade_cleanup(tmpbb);
+
+            if (APR_BRIGADE_EMPTY(bb)) {
+                break;
+            }
+
+            rv = apr_brigade_split_line(tmpbb, bb,
+                                        APR_BLOCK_READ, sizeof(buf));
+            lineno++;
+
+            if (rv) {
+                return rv;
+            }
+
+            apr_brigade_flatten(tmpbb, buf, &bsize);
+
+            if (bsize == 0) {
+                break;
+            }
+
+            buf[bsize - 1] = 0;
+
+            /* comment */
+            if (buf[0] == '#') {
+                continue;
+            }
+
+            /* line format: <IP> <query_string>\n */
+            t = strchr(buf, ' ');
+            if (!t) {
+                continue;
+            }
+            
+            const char *ip = apr_pstrndup(pool, buf, t - buf);
+            t++;
+
+            server = apr_hash_get(servers, ip, APR_HASH_KEY_STRING);
+            
+            if (server == NULL) {
+                server = apr_pcalloc(pool, sizeof(hb_server_t));
+                server->ip = ip;
+                server->seen = -1;
+
+                apr_hash_set(servers, server->ip, APR_HASH_KEY_STRING, server);
+            }
+            
+            apr_table_clear(hbt);
+
+            argstr_to_table(pool, apr_pstrdup(pool, t), hbt);
+
+            if (apr_table_get(hbt, "busy")) {
+                server->busy = atoi(apr_table_get(hbt, "busy"));
+            }
+
+            if (apr_table_get(hbt, "ready")) {
+                server->ready = atoi(apr_table_get(hbt, "ready"));
+            }
+
+            if (apr_table_get(hbt, "lastseen")) {
+                server->seen = atoi(apr_table_get(hbt, "lastseen"));
+            }
+
+            if (server->busy == 0 && server->ready != 0) {
+                /* Server has zero threads active, but lots of them ready, 
+                 * it likely just started up, so lets /4 the number ready, 
+                 * to prevent us from completely flooding it with all new 
+                 * requests.
+                 */
+                server->ready = server->ready / 4;
+            }
+
+        } while (1);
+    }
+
+    return APR_SUCCESS;
+}
 
 /*
+ * Finding a random number in a range. 
+ *      n' = a + n(b-a+1)/(M+1)
+ * where:
+ *      n' = random number in range
+ *      a  = low end of range
+ *      b  = high end of range
+ *      n  = random number of 0..M
+ *      M  = maxint
+ * Algorithm 'borrowed' from PHP's rand() function.
  */
-static proxy_worker *find_best_roundrobin(proxy_balancer *balancer,
-                                         request_rec *r)
+#define RAND_RANGE(__n, __min, __max, __tmax) \
+(__n) = (__min) + (long) ((double) ((__max) - (__min) + 1.0) * ((__n) / ((__tmax) + 1.0)))
+
+static apr_status_t random_pick(apr_uint32_t *number,
+                                apr_uint32_t min,
+                                apr_uint32_t max)
+{
+    apr_status_t rv = 
+        apr_generate_random_bytes((void*)number, sizeof(apr_uint32_t));
+
+    if (rv) {
+        return rv;
+    }
+
+    RAND_RANGE(*number, min, max, APR_UINT32_MAX);
+
+    return APR_SUCCESS;
+}
+
+static proxy_worker *find_best_hb(proxy_balancer *balancer,
+                                  request_rec *r)
 {
+    apr_status_t rv;
     int i;
+    apr_uint32_t openslots = 0;
     proxy_worker *worker;
+    hb_server_t *server;
+    apr_array_header_t *up_servers;
     proxy_worker *mycandidate = NULL;
-    int checking_standby;
-    int checked_standby;
-    rr_data *ctx;
-
-    ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
-                 "proxy: Entering roundrobin for BALANCER %s (%d)",
-                 balancer->name, (int)getpid());
-    
-    /* The index of the candidate last chosen is stored in ctx->index */
-    if (!balancer->context) {
-        /* UGLY */
-        ctx = apr_pcalloc(r->server->process->pconf, sizeof(rr_data));
-        balancer->context = (void *)ctx;
-        ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
-                 "proxy: Creating roundrobin ctx for BALANCER %s (%d)",
-                 balancer->name, (int)getpid());
-    } else {
-        ctx = (rr_data *)balancer->context;
-    }
-    ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
-                 "proxy: roundrobin index: %d (%d)",
-                 ctx->index, (int)getpid());
-
-    checking_standby = checked_standby = 0;
-    while (!mycandidate && !checked_standby) {
-        worker = (proxy_worker *)balancer->workers->elts;
+    apr_pool_t *tpool;
+    apr_hash_t *servers;
 
-        for (i = 0; i < balancer->workers->nelts; i++, worker++) {
-            if (i < ctx->index)
-                continue;
-            if ( (checking_standby ? !PROXY_WORKER_IS_STANDBY(worker) : PROXY_WORKER_IS_STANDBY(worker))
)
-                continue;
-            if (!PROXY_WORKER_IS_USABLE(worker))
-                ap_proxy_retry_worker("BALANCER", worker, r->server);
-            if (PROXY_WORKER_IS_USABLE(worker)) {
-                mycandidate = worker;
-                break;
+    lb_hb_ctx_t *ctx = 
+        ap_get_module_config(r->server->module_config,
+                             &lbmethod_heartbeat_module);
+
+    apr_pool_create(&tpool, r->pool);
+
+    servers = apr_hash_make(tpool);
+
+    rv = read_heartbeats(ctx->path, servers, tpool);
+
+    if (rv) {
+        ap_log_rerror(APLOG_MARK, APLOG_ERR, rv, r,
+                      "lb_heartbeat: Unable to read heartbeats at '%s'",
+                      ctx->path);
+        apr_pool_destroy(tpool);
+        return NULL;
+    }
+
+    up_servers = apr_array_make(tpool, apr_hash_count(servers), sizeof(hb_server_t *));
+
+    for (i = 0; i < balancer->workers->nelts; i++) {
+        worker = &APR_ARRAY_IDX(balancer->workers, i, proxy_worker);
+        server = apr_hash_get(servers, worker->hostname, APR_HASH_KEY_STRING);
+
+        if (!server) {
+            continue;
+        }
+
+        if (!PROXY_WORKER_IS_USABLE(worker)) {
+            ap_proxy_retry_worker("BALANCER", worker, r->server);
+        }
+
+        if (PROXY_WORKER_IS_USABLE(worker)) {
+            server->worker = worker;
+            if (server->seen < 10) {
+                openslots += server->ready;
+                APR_ARRAY_PUSH(up_servers, hb_server_t *) = server;
             }
         }
-        checked_standby = checking_standby++;
     }
 
+    if (openslots > 0) {
+        apr_uint32_t c = 0;
+        apr_uint32_t pick = 0;;
+
+        rv = random_pick(&pick, 0, openslots);
+
+        if (rv) {
+            ap_log_rerror(APLOG_MARK, APLOG_ERR, rv, r,
+                          "lb_heartbeat: failed picking a random number. how random.");
+            apr_pool_destroy(tpool);
+            return NULL;
+        }
+
+        for (i = 0; i < up_servers->nelts; i++) {
+            server = APR_ARRAY_IDX(up_servers, i, hb_server_t *);
+            if (pick > c && pick <= c + server->ready) {
+                mycandidate = server->worker;
+            }
 
-    ctx->index += 1;
-    if (ctx->index >= balancer->workers->nelts) {
-        ctx->index = 0;
+            c += server->ready;
+        }
     }
+
+    apr_pool_destroy(tpool);
+
     return mycandidate;
 }
 
-static const proxy_balancer_method roundrobin =
+static const proxy_balancer_method heartbeat =
 {
-    "roundrobin",
-    &find_best_roundrobin,
+    "heartbeat",
+    &find_best_hb,
     NULL
 };
 
+static void register_hooks(apr_pool_t *p)
+{
+    ap_register_provider(p, PROXY_LBMETHOD, "heartbeat", "0", &heartbeat);
+}
 
-static void ap_proxy_rr_register_hook(apr_pool_t *p)
+static void *lb_hb_create_config(apr_pool_t *p, server_rec *s)
 {
-    ap_register_provider(p, PROXY_LBMETHOD, "roundrobin", "0", &roundrobin);
+    lb_hb_ctx_t *ctx = (lb_hb_ctx_t *) apr_palloc(p, sizeof(lb_hb_ctx_t));
+    
+    ctx->path = ap_server_root_relative(p, "logs/hb.dat");
+    
+    return ctx;
 }
 
-module AP_MODULE_DECLARE_DATA proxy_balancer_rr_module = {
+static void *lb_hb_merge_config(apr_pool_t *p, void *basev, void *overridesv)
+{
+    lb_hb_ctx_t *ps = apr_pcalloc(p, sizeof(lb_hb_ctx_t));
+    lb_hb_ctx_t *base = (lb_hb_ctx_t *) basev;
+    lb_hb_ctx_t *overrides = (lb_hb_ctx_t *) overridesv;
+
+    if (overrides->path) {
+        ps->path = apr_pstrdup(p, overrides->path);
+    }
+    else {
+        ps->path = apr_pstrdup(p, base->path);
+    }
+
+    return ps;
+}
+
+static const char *cmd_lb_hb_storage(cmd_parms *cmd,
+                                  void *dconf, const char *path)
+{
+    apr_pool_t *p = cmd->pool;
+    lb_hb_ctx_t *ctx =
+    (lb_hb_ctx_t *) ap_get_module_config(cmd->server->module_config,
+                                         &lbmethod_heartbeat_module);
+
+    const char *err = ap_check_cmd_context(cmd, GLOBAL_ONLY);
+    
+    if (err != NULL) {
+        return err;
+    }
+
+    ctx->path = ap_server_root_relative(p, path);
+
+    return NULL;
+}
+
+static const command_rec cmds[] = {
+    AP_INIT_TAKE1("HeartbeatStorage", cmd_lb_hb_storage, NULL, RSRC_CONF,
+                  "Path to read heartbeat data."),
+    {NULL}
+};
+
+module AP_MODULE_DECLARE_DATA lbmethod_heartbeat_module = {
     STANDARD20_MODULE_STUFF,
-    NULL,       /* create per-directory config structure */
-    NULL,       /* merge per-directory config structures */
-    NULL,       /* create per-server config structure */
-    NULL,       /* merge per-server config structures */
-    NULL,       /* command apr_table_t */
-    ap_proxy_rr_register_hook /* register hooks */
+    NULL,                       /* create per-directory config structure */
+    NULL,                       /* merge per-directory config structures */
+    lb_hb_create_config,        /* create per-server config structure */
+    lb_hb_merge_config,         /* merge per-server config structures */
+    cmds,                       /* command apr_table_t */
+    register_hooks              /* register hooks */
 };

Propchange: httpd/httpd/trunk/modules/proxy/mod_lbmethod_heartbeat.c
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: httpd/httpd/trunk/modules/proxy/mod_lbmethod_heartbeat.c
------------------------------------------------------------------------------
    svn:mergeinfo = 



Mime
View raw message