httpd-cvs mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pa...@apache.org
Subject svn commit: r380618 - in /httpd/mod_cache_requester/trunk: LICENSE README mod_cache_requester.c mod_cache_requester.h
Date Fri, 24 Feb 2006 08:21:55 GMT
Author: parin
Date: Fri Feb 24 00:21:50 2006
New Revision: 380618

URL: http://svn.apache.org/viewcvs?rev=380618&view=rev
Log:
mod-cache-requester source code

Added:
    httpd/mod_cache_requester/trunk/LICENSE
    httpd/mod_cache_requester/trunk/README
    httpd/mod_cache_requester/trunk/mod_cache_requester.c
    httpd/mod_cache_requester/trunk/mod_cache_requester.h

Added: httpd/mod_cache_requester/trunk/LICENSE
URL: http://svn.apache.org/viewcvs/httpd/mod_cache_requester/trunk/LICENSE?rev=380618&view=auto
==============================================================================
--- httpd/mod_cache_requester/trunk/LICENSE (added)
+++ httpd/mod_cache_requester/trunk/LICENSE Fri Feb 24 00:21:50 2006
@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Added: httpd/mod_cache_requester/trunk/README
URL: http://svn.apache.org/viewcvs/httpd/mod_cache_requester/trunk/README?rev=380618&view=auto
==============================================================================
--- httpd/mod_cache_requester/trunk/README (added)
+++ httpd/mod_cache_requester/trunk/README Fri Feb 24 00:21:50 2006
@@ -0,0 +1,105 @@
+/* Copyright 2001-2005 The Apache Software Foundation or its licensors, as
+ * applicable.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+Mod-Cache-Requester
+-------------------
+
+Contents
+--------
+This document covers following topics for mod-cache-requester.
+
+1. Concept
+2. Implementation
+3. Compilation
+4. Configuration.
+
+
+1. Concept
+----------
+a.) Every time a page expires, it is removed from the cache and thus next request would require
server to re-generate the cached page.
+
+b.) this module keeps track of all popular pages and re-requests them before they are removed
from the cache so that server 'refreshes' those pages.
+
+c.) Execution flow.
+	a. request is received by te mod-cache quick handler.
+	b. if page is 'soon-to-expire' then quick-handler lets mod-cache-requester know about this
page and it continues processing the request.
+	c. if page is already there in queue of mod-cache-requester, its priority is incremented
by one. if not, then this page is inserted in the queue.
+	d. mod-cache-requester pops the highest priority page from queue and re-requests (in a seperate
thread) that page, 
+	   with the note indicating that this page is coming from the mod-cache-requester so it
should not be served from cache. 
+	   rather quick handler should add save filter for this request.
+	e. mod-cache-requester receives data which it throws away as the objective of this request
- to reload page in cache - is already achieved.
+
+	f. if no pending request is available in the queue of mod-cache-requester then all threads
go to sleep.
+
+2. Implementation
+-----------------
+a.) a new provider is added in mod_cache.h (cache_req_provider) which provides an interface
(notify-cache-requester) to notify about soon-to-expire pages.
+    This provider would allow us to change/add implementation of mod-cache-requester without
affecting mod_cache.
+
+b.) mod-cache-requester implements a cache-req-provider.
+
+c.) mod_cache_requester.h implements key data structures. (again it would allow us to add/change
the implementation of re-requesting technique. keeping all these data structures in mod_cache_requester.c
would not let us have this flexibility)
+
+d.) mod-cache-requester uses libcurl(function make_curl_request) to make the re-request the
documents. and it adds a note in input headers indicating that this page is being requested
by mod-cache-requester. So server does return cached page and it reloads the page. for security
reasons, 
+
+3. Compilation
+--------------
+a.) copy mod_cache_requester[.h/.c], config.m4 in the cache modules.
+
+b.) apply mod_cache.h.patch and mod_cache.c.patches.
+
+c.) config.m4 should detect the required libraries automatically, if now please follow below
instructions.
+	while compiling and linking this module, provide libcurl path. http://curl.haxx.se/libcurl/using/
+		a. curl-config --cflags command returns list of all files to be  included in CFLAGS
+		b. curl-config --libs command returns list of all libraries to e incuded in LDFLAGS 
+
+4. Configuration
+----------------
+a.) This is a sub-module of a mod_cache
+
+b.) it provides following directives.
+	a. CacheRequesterThreads: Number of threads those run in background and keep re-requesting
popular pages.
+	b. CacheRequesterMaxQueueSize: Maximum size of the expiring pages waiting page queue.
+	c. CacheRequesterSoonToExpireTime: (In Seconds) when expiring time of a page is equal or
less than CacheRequesterSoonToExpireTime, 
+	then such page would be inserted into the queue whose elements are going to be re-requested.
+	d. CacheRequesterSecretCode: This string value is used by libcurl request as an input-header.
This would be used to authenticate requests created by mod-cache-requester.
+
+c.) Sample config file is shown below.
+# I have set CacheRequesterSoonToExpireTime too high just to make most of the pages in the
cache be considered as soon-to-be-expired!
+
+LoadModule cache_module modules/mod_cache.so
+LoadModule disk_cache_module modules/mod_disk_cache.so
+LoadModule mem_cache_module modules/mod_mem_cache.so
+LoadModule cache_requester_module modules/mod_cache_requester.so
+
+<IfModule mod_cache.c>
+	CacheIgnoreCacheControl On
+	<IfModule mod_mem_cache.c>
+		CacheEnable mem /
+		MCacheSize 4096
+		MCacheMaxObjectCount 100
+		MCacheMinObjectSize 1
+		MCacheMaxObjectSize 2048
+	</IfModule>
+
+	<IfModule mod_cache_requester.c>
+		CacheRequesterThreads 2
+		CacheRequesterMaxQueueSize 10
+		CacheRequesterSoonToExpireTime 1000000
+		CacheRequesterSecretCode ABCDEF
+	</IfModule>
+</IfModule> 
+

Added: httpd/mod_cache_requester/trunk/mod_cache_requester.c
URL: http://svn.apache.org/viewcvs/httpd/mod_cache_requester/trunk/mod_cache_requester.c?rev=380618&view=auto
==============================================================================
--- httpd/mod_cache_requester/trunk/mod_cache_requester.c (added)
+++ httpd/mod_cache_requester/trunk/mod_cache_requester.c Fri Feb 24 00:21:50 2006
@@ -0,0 +1,428 @@
+/* Copyright 2001-2005 The Apache Software Foundation or its licensors, as
+ * applicable.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+This mod-cache-requester modules starts background threads which keeps monitoring soon to
expire pages from cache.
+and if such pages are 'popular' then those pages are re-requested using libcurl so that such
pages are not removed from the cache.
+ */
+
+#define CORE_PRIVATE
+#include "mod_cache_requester.h"
+#include <curl/curl.h>
+
+#if APR_HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#if !APR_HAS_THREADS
+#error This module does not currently compile unless you have a thread-capable APR. Sorry!
+#endif
+
+# define DEFAULT_NUMBER_OF_THREADS 2
+# define DEFAULT_QUEUE_SIZE 10
+# define DEFAULT_EXPIRING_SIGNAL_TIME_IN_SECONDS 1000
+# define DEFAULT_SECRET_CODE "SECRET_CODE"
+
+module AP_MODULE_DECLARE_DATA cache_requester_module;
+
+static cache_req_conf_t *sconf;
+static cache_requester *cache_req_obj;
+
+/**
+ * Priority queue management functions
+ */
+static long cache_req_get_priority(void *data) {
+    cache_req_expiring_page *expiring_page  = (cache_req_expiring_page *) data;
+    return expiring_page->priority;
+}
+
+static void cache_req_set_pos(void *a, apr_ssize_t pos)
+{
+    cache_req_expiring_page *obj = (cache_req_expiring_page *)a;
+
+    apr_atomic_set32(&(obj->pos), pos);
+}
+
+static apr_ssize_t cache_req_get_pos(void *a)
+{
+    cache_req_expiring_page *obj = (cache_req_expiring_page *)a;
+    return apr_atomic_read32(&(obj->pos));
+} 
+
+/**
+ * Queue Management operations. all those operations must be thread safe. use cache_req_obj->lock
to ensure mutual exclusion.
+ */
+static void create_expiring_page(cache_req_expiring_page **node, request_rec *r) {
+    char *request_uri;
+
+    *node = (cache_req_expiring_page *) malloc(sizeof(cache_req_expiring_page));
+    if(!node) {
+        return;
+    }
+    (*node)->priority = 0;
+
+    request_uri = apr_pstrcat(r->pool, r->server->server_hostname, 
+                                                    ":", 
+                                                    apr_itoa(r->pool, r->server->port),

+                                                    r->uri, NULL);
+
+    (*node)->request_uri = malloc(strlen(request_uri) + 1);
+    strcpy((*node)->request_uri, request_uri);
+
+    (*node)->uri = malloc(strlen(r->uri) + 1);
+    strcpy((*node)->uri, r->uri);
+
+    (*node)->next = NULL;
+}
+
+static void delete_expiring_page(cache_req_expiring_page *node) {
+    if(node->request_uri) {
+        free(node->request_uri);
+    }
+    if(node->uri) {
+        free(node->uri);
+    }
+
+    if(node) {
+        free(node);
+    }
+}
+
+static cache_req_expiring_page *insert_into_queue(request_rec *r) {
+
+    cache_req_expiring_page *value;
+    cache_req_expiring_page *new_value;
+    int queue_size;
+
+    queue_size = cache_pq_size(cache_req_obj->cache_request_pq);
+
+    if(queue_size >= sconf->queue_size) {
+        ap_log_error(APLOG_MARK, APLOG_INFO, 0, r->server,
+                     "cache_requester: cache size queue exceeded");
+        return NULL;
+    }
+
+    /*check if the entry for this page already exists or not.*/
+    value = apr_hash_get(cache_req_obj->expiring_page_hash, r->uri, APR_HASH_KEY_STRING);
+
+    /* if not, then insert it into the queue, initialize the priority*/
+    if(value == NULL) {
+        create_expiring_page(&new_value, r);
+        apr_hash_set(cache_req_obj->expiring_page_hash, new_value->uri, APR_HASH_KEY_STRING,
new_value);
+        cache_pq_insert(cache_req_obj->cache_request_pq, new_value);
+
+        // if this is the first entry added in the queue, then lets wake up one of the threads.
+        if(queue_size == 0) {
+            apr_thread_cond_signal(cache_req_obj->cond);
+        }
+        return new_value;
+    }
+    /* if its already there, then increment the priority by one.*/
+    else {
+        value->priority++;
+        cache_pq_change_priority(cache_req_obj->cache_request_pq, (value->priority
- 1), value->priority, value);
+        return value;
+    }
+}
+
+/* delete the entry from the queue. */
+static int delete_from_queue(cache_req_expiring_page *expiring_page) {
+    cache_req_expiring_page *value;
+
+    if(!expiring_page || !(expiring_page->uri)) {
+        return DECLINED;
+    }
+
+    value = apr_hash_get(cache_req_obj->expiring_page_hash, expiring_page->uri, APR_HASH_KEY_STRING);
+    if(value == NULL) {
+        return DECLINED;
+    }
+    else {
+        apr_hash_set(cache_req_obj->expiring_page_hash, expiring_page->uri, APR_HASH_KEY_STRING,
NULL);
+        delete_expiring_page(value);
+        return OK;
+    }
+}
+
+/*
+ * insert the soon-to-expire page into the queue.
+ */
+int notify_cache_requester(request_rec *r, cache_handle_t *handle) {
+    const char *is_from_cache_requester;
+    cache_req_expiring_page *value;
+
+    is_from_cache_requester = apr_table_get(r->headers_in, "Cache-Requester");
+    if(is_from_cache_requester) {
+
+        /* verify the request is actually made by mod-cache-requester. */
+        if(!strcmp(is_from_cache_requester, sconf->secret_code)) {
+            return DECLINED;
+        }
+        return OK;
+    }
+
+     /*
+    See the expiry time of the page. If its soon to expire, 
+    then notify add this page in the queue (if already there then increment the counter)
to re-request
+    */
+    if(!is_soon_to_expire(handle)) {
+        apr_thread_mutex_lock(cache_req_obj->lock);
+        value = insert_into_queue(r);
+        apr_thread_mutex_unlock(cache_req_obj->lock);
+
+    }
+    return OK;
+}
+
+/*
+ * Checks whether this page is soon-to-expire. returns 0 if its the case
+ */
+int is_soon_to_expire(cache_handle_t *handle) {
+    cache_info *info;
+    if(!handle) {
+        return -1;
+    }
+
+    info = &(handle->cache_obj->info);
+    if((apr_time_sec(info->expire) - apr_time_sec(apr_time_now())) < sconf->expiring_signal_time_in_seconds)
{
+        return 0;
+    }
+    return -1;
+}
+
+/*
+ *  Process the most popular page, remove it from the queue (in request_page) 
+ *  - re request it using libcurl - make_curl_request
+ */
+static size_t process_received_data(void *buffer, size_t size, size_t nmemb, void *userp);
+
+void make_curl_request(char *request_uri, char *secret_code_header) {
+    CURL *curl;
+    struct curl_slist *headers=NULL;
+    int returnValue;
+
+    curl = curl_easy_init();
+    headers = curl_slist_append(headers, secret_code_header);
+
+    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+    curl_easy_setopt(curl, CURLOPT_URL, request_uri);
+    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, process_received_data);
+
+    returnValue = curl_easy_perform(curl);
+
+    ap_log_error(APLOG_MARK, APLOG_INFO, 0, NULL, "cache_requester: URI: %s re-requested,
with return value: %d", request_uri, returnValue);
+
+}
+
+void *APR_THREAD_FUNC request_page(apr_thread_t *thd, void * dummy) {
+    cache_req_expiring_page *curr_obj;
+    request_rec *r;
+    request_rec *rnew;
+    char *secret_code_header;
+
+    secret_code_header = (char *) malloc (sizeof("Cache-Requester: ") + sizeof(sconf->secret_code)
+ 1);
+    strcpy(secret_code_header, "Cache-Requester: ");
+    strcat(secret_code_header, sconf->secret_code);
+
+    for(;;) {
+        apr_thread_mutex_lock(cache_req_obj->lock);
+
+        /* If no entry in the queue => sleep */
+        while(cache_pq_size(cache_req_obj->cache_request_pq) == 0) {
+            apr_thread_cond_wait(cache_req_obj->cond, cache_req_obj->lock);
+        }
+
+        /*Pop the most popular page from the queue*/
+        curr_obj = cache_pq_pop(cache_req_obj->cache_request_pq);
+
+        if(!curr_obj) {
+            apr_thread_mutex_unlock(cache_req_obj->lock);
+            continue;
+        }
+
+        /*Remove entry from the expiring_page and insert it into expiring_page hashmap.*/
+        apr_hash_set(cache_req_obj->expiring_page_hash, curr_obj->uri, APR_HASH_KEY_STRING,
NULL);
+        apr_thread_mutex_unlock(cache_req_obj->lock);
+
+        //Re-request the page.
+        make_curl_request(curr_obj->request_uri, secret_code_header);
+
+        //Delete the object.
+        delete_expiring_page(curr_obj);
+    }
+    free(secret_code_header);
+    return NULL;
+}
+
+static size_t process_received_data(void *buffer, size_t size, size_t nmemb, void *userp)
{
+    /*
+     * throw away data received as a part of curl request.
+     */
+}
+
+/**
+ * Directive Configurations.
+ */
+static void *create_cache_requester_config(apr_pool_t *p, server_rec *s)
+{
+    sconf = apr_pcalloc(p, sizeof(cache_req_conf_t));
+    sconf->number_of_threads = DEFAULT_NUMBER_OF_THREADS;
+    sconf->queue_size = DEFAULT_QUEUE_SIZE;
+    sconf->expiring_signal_time_in_seconds = DEFAULT_EXPIRING_SIGNAL_TIME_IN_SECONDS;
+    sconf->secret_code = DEFAULT_SECRET_CODE;
+    return sconf;
+}
+
+static const char *set_no_of_threads(cmd_parms *parms, void *in_struct_ptr, const char *arg)
{
+    apr_size_t val;
+
+    if (sscanf(arg, "%" APR_SIZE_T_FMT, &val) != 1) {
+        return "CacheRequesterThreads argument must be an integer representing number of
therads.";
+    }
+    if(val < 1) {
+        return "CacheRequesterThreads argument must not be less than 1.";
+    }
+    sconf->number_of_threads = val;
+    return NULL;
+}
+
+static const char *set_max_queue_size(cmd_parms *parms, void *in_struct_ptr, const char *arg)
{
+    apr_size_t val;
+
+    if (sscanf(arg, "%" APR_SIZE_T_FMT, &val) != 1) {
+        return "CacheRequesterMaxQueueSize argument must be an integer representing number
of therads.";
+    }
+
+    if(val < 1) {
+        return "CacheRequesterMaxQueueSize argument must be greater than 1.";
+    }
+    sconf->queue_size = val;
+    return NULL;
+}
+
+static const char *set_soon_to_expire_time(cmd_parms *parms, void *in_struct_ptr, const char
*arg) {
+    apr_size_t val;
+
+    if (sscanf(arg, "%" APR_SIZE_T_FMT, &val) != 1) {
+        return "CacheRequesterSoonToExpireTime value must be a positive integer";
+    }
+    if(val < 1) {
+        return "CacheRequesterSoonToExpireTime argument must be greater than 1.";
+    }
+
+    sconf->expiring_signal_time_in_seconds = val;
+    return NULL;
+}
+
+static const char *set_secret_code(cmd_parms *parms, void *name, const char *arg) {
+
+    sconf->secret_code = arg;
+    return NULL;
+}
+
+static const command_rec cache_req_cmds[] =
+{
+    AP_INIT_TAKE1("CacheRequesterThreads", set_no_of_threads, NULL, RSRC_CONF,
+                  "Number of cache requester threads"),
+    AP_INIT_TAKE1("CacheRequesterMaxQueueSize", set_max_queue_size, NULL, RSRC_CONF,
+                  "The maximum number of objects allowed to be placed in the cache requester
queue."),
+    AP_INIT_TAKE1("CacheRequesterSoonToExpireTime", set_soon_to_expire_time, NULL, RSRC_CONF,
+                  "The required time remaining to expire a page to be able to be re-requested
(in seconds)"),
+    AP_INIT_TAKE1("CacheRequesterSecretCode", set_secret_code, NULL, RSRC_CONF,
+                  "the secret code to authenticate requests made by mod-cache-requester"),
+
+    {NULL}
+};
+ 
+/**
+ * initialize and start the threads.
+ */
+static int init_cache_req_obj_threads(apr_pool_t *p, server_rec *s) {
+    int i;
+    cache_req_thread *current_thread;
+
+    current_thread = cache_req_obj->thread_pool->queue;
+    apr_thread_create(&(current_thread->handle), NULL, request_page, NULL, p);
+    current_thread->next = NULL;
+    for(i=0; i < sconf->number_of_threads-1; i++) {
+        current_thread->next = (cache_req_thread *) apr_palloc(p, sizeof(cache_req_thread));
+        apr_thread_create(&(current_thread->next->handle), NULL, request_page,
NULL, p);
+        current_thread->next->next = NULL;
+        current_thread = current_thread->next;
+    }
+}
+
+/**
+ * module configuration stuff.
+ */
+
+static int cache_requester_child_init(apr_pool_t *p, apr_pool_t *plog,
+                                       apr_pool_t *ptemp, server_rec *s) {
+    //Here Threads would be instantiated.
+    init_cache_req_obj_threads(p, s);
+}
+
+static int cache_requester_post_config(apr_pool_t *p, apr_pool_t *plog,
+                                 apr_pool_t *ptemp, server_rec *s) {
+    // Initialize the curl lib.
+    curl_global_init(CURL_GLOBAL_ALL);
+
+    //Create main object first.
+    cache_req_obj = (cache_requester *) apr_palloc(p, sizeof(cache_requester));
+
+    // instantiate the thread pool.
+    cache_req_obj->thread_pool = (cache_req_thread_pool *) apr_palloc(p, sizeof(cache_req_thread_pool));
+
+    // create the lock, condition variable and priority queue.
+    apr_thread_mutex_create(&(cache_req_obj->lock), APR_THREAD_MUTEX_DEFAULT, p);
+    apr_thread_cond_create(&(cache_req_obj->cond), p);
+    cache_req_obj->cache_request_pq = cache_pq_init(sconf->queue_size, 
+                                                    cache_req_get_priority, 
+                                                    cache_req_get_pos, 
+                                                    cache_req_set_pos);
+
+    // instantiate the hashmaps.
+    cache_req_obj->expiring_page_hash = apr_hash_make(p);
+
+    //Instantiate the thread queue.
+    cache_req_obj->thread_pool->queue = (cache_req_thread *) apr_palloc(p, sizeof(cache_req_thread));
+
+    return OK;
+}
+
+static const cache_req_provider cache_requester_provider =
+{
+    &notify_cache_requester,
+};
+
+static void register_hooks(apr_pool_t *p)
+{
+    ap_hook_post_config(cache_requester_post_config, NULL, NULL, APR_HOOK_MIDDLE);
+    ap_hook_child_init(cache_requester_child_init, NULL, NULL, APR_HOOK_MIDDLE);
+    ap_register_provider(p, CACHE_REQUESTER_PROVIDER_GROUP, "cache_req", "0",
+                         &cache_requester_provider);
+}
+
+module AP_MODULE_DECLARE_DATA cache_requester_module =
+{
+    STANDARD20_MODULE_STUFF,
+    NULL,                          /* create per-directory config structure */
+    NULL,                          /* merge per-directory config structures */
+    create_cache_requester_config, /* create per-server config structure */
+    NULL,                          /* merge per-server config structures */
+    cache_req_cmds,                /* command apr_table_t */
+    register_hooks
+};

Added: httpd/mod_cache_requester/trunk/mod_cache_requester.h
URL: http://svn.apache.org/viewcvs/httpd/mod_cache_requester/trunk/mod_cache_requester.h?rev=380618&view=auto
==============================================================================
--- httpd/mod_cache_requester/trunk/mod_cache_requester.h (added)
+++ httpd/mod_cache_requester/trunk/mod_cache_requester.h Fri Feb 24 00:21:50 2006
@@ -0,0 +1,78 @@
+/* Copyright 2001-2005 The Apache Software Foundation or its licensors, as
+ * applicable.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CACHE_REQUESTER_THREADPOOL_H
+#define CACHE_REQUESTER_THREADPOOL_H
+#endif
+
+#include "apr_thread_proc.h"
+#include "mod_cache.h"
+#include "cache_pqueue.h"
+#include "cache_cache.h"
+#include "ap_provider.h"
+#include "apr_pools.h"
+#include "ap_mpm.h"
+#include "apr_thread_mutex.h"
+#include "apr_thread_cond.h"
+#include "apr_strings.h"
+#include <curl/curl.h>
+
+
+
+
+// Structure to store the parameters. Currently we support 'number of threads' and 'queue_size
of soon-to-expire pages' parameters.
+typedef struct cache_req_conf {
+    long number_of_threads;
+    long queue_size;
+    long expiring_signal_time_in_seconds;
+    char *secret_code;
+} cache_req_conf_t;
+
+
+// Linked list structure of  threads. each instance contains pointer to the thread and pointer
to next instance of the linked list.
+typedef struct cache_req_thread {
+    apr_thread_t *handle;
+    struct cache_req_thread *next;
+}cache_req_thread;
+
+// request object to be stored in the queue for each soon-to-expire page.
+typedef struct cache_req_expiring_page {
+    char *uri;
+    char *request_uri;
+    long priority;
+    apr_uint32_t pos;
+    struct cache_req_expiring_page *next;
+}cache_req_expiring_page;
+
+// thread pool. 
+// contains thread queue, lock, condition variable.
+typedef struct cache_req_thread_pool {
+    cache_req_thread *queue;
+
+    //cache_requester_request *processing_requests;
+}cache_req_thread_pool;
+
+// main structure, which uses all the above structures and keeps them organized.
+typedef struct cache_requester {
+    cache_req_thread_pool *thread_pool;
+
+    cache_pqueue_t *cache_request_pq;
+    apr_hash_t *expiring_page_hash;
+    apr_thread_mutex_t *lock;
+    apr_thread_cond_t *cond;
+
+    apr_pool_t *pool;
+}cache_requester;



Mime
View raw message