Author: parin
Date: Fri Feb 24 00:21:50 2006
New Revision: 380618
URL: http://svn.apache.org/viewcvs?rev=380618&view=rev
Log:
mod-cache-requester source code
Added:
httpd/mod_cache_requester/trunk/LICENSE
httpd/mod_cache_requester/trunk/README
httpd/mod_cache_requester/trunk/mod_cache_requester.c
httpd/mod_cache_requester/trunk/mod_cache_requester.h
Added: httpd/mod_cache_requester/trunk/LICENSE
URL: http://svn.apache.org/viewcvs/httpd/mod_cache_requester/trunk/LICENSE?rev=380618&view=auto
==============================================================================
--- httpd/mod_cache_requester/trunk/LICENSE (added)
+++ httpd/mod_cache_requester/trunk/LICENSE Fri Feb 24 00:21:50 2006
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
Added: httpd/mod_cache_requester/trunk/README
URL: http://svn.apache.org/viewcvs/httpd/mod_cache_requester/trunk/README?rev=380618&view=auto
==============================================================================
--- httpd/mod_cache_requester/trunk/README (added)
+++ httpd/mod_cache_requester/trunk/README Fri Feb 24 00:21:50 2006
@@ -0,0 +1,105 @@
+/* Copyright 2001-2005 The Apache Software Foundation or its licensors, as
+ * applicable.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+Mod-Cache-Requester
+-------------------
+
+Contents
+--------
+This document covers following topics for mod-cache-requester.
+
+1. Concept
+2. Implementation
+3. Compilation
+4. Configuration.
+
+
+1. Concept
+----------
+a.) Every time a page expires, it is removed from the cache and thus next request would require
server to re-generate the cached page.
+
+b.) this module keeps track of all popular pages and re-requests them before they are removed
from the cache so that server 'refreshes' those pages.
+
+c.) Execution flow.
+ a. request is received by te mod-cache quick handler.
+ b. if page is 'soon-to-expire' then quick-handler lets mod-cache-requester know about this
page and it continues processing the request.
+ c. if page is already there in queue of mod-cache-requester, its priority is incremented
by one. if not, then this page is inserted in the queue.
+ d. mod-cache-requester pops the highest priority page from queue and re-requests (in a seperate
thread) that page,
+ with the note indicating that this page is coming from the mod-cache-requester so it
should not be served from cache.
+ rather quick handler should add save filter for this request.
+ e. mod-cache-requester receives data which it throws away as the objective of this request
- to reload page in cache - is already achieved.
+
+ f. if no pending request is available in the queue of mod-cache-requester then all threads
go to sleep.
+
+2. Implementation
+-----------------
+a.) a new provider is added in mod_cache.h (cache_req_provider) which provides an interface
(notify-cache-requester) to notify about soon-to-expire pages.
+ This provider would allow us to change/add implementation of mod-cache-requester without
affecting mod_cache.
+
+b.) mod-cache-requester implements a cache-req-provider.
+
+c.) mod_cache_requester.h implements key data structures. (again it would allow us to add/change
the implementation of re-requesting technique. keeping all these data structures in mod_cache_requester.c
would not let us have this flexibility)
+
+d.) mod-cache-requester uses libcurl(function make_curl_request) to make the re-request the
documents. and it adds a note in input headers indicating that this page is being requested
by mod-cache-requester. So server does return cached page and it reloads the page. for security
reasons,
+
+3. Compilation
+--------------
+a.) copy mod_cache_requester[.h/.c], config.m4 in the cache modules.
+
+b.) apply mod_cache.h.patch and mod_cache.c.patches.
+
+c.) config.m4 should detect the required libraries automatically, if now please follow below
instructions.
+ while compiling and linking this module, provide libcurl path. http://curl.haxx.se/libcurl/using/
+ a. curl-config --cflags command returns list of all files to be included in CFLAGS
+ b. curl-config --libs command returns list of all libraries to e incuded in LDFLAGS
+
+4. Configuration
+----------------
+a.) This is a sub-module of a mod_cache
+
+b.) it provides following directives.
+ a. CacheRequesterThreads: Number of threads those run in background and keep re-requesting
popular pages.
+ b. CacheRequesterMaxQueueSize: Maximum size of the expiring pages waiting page queue.
+ c. CacheRequesterSoonToExpireTime: (In Seconds) when expiring time of a page is equal or
less than CacheRequesterSoonToExpireTime,
+ then such page would be inserted into the queue whose elements are going to be re-requested.
+ d. CacheRequesterSecretCode: This string value is used by libcurl request as an input-header.
This would be used to authenticate requests created by mod-cache-requester.
+
+c.) Sample config file is shown below.
+# I have set CacheRequesterSoonToExpireTime too high just to make most of the pages in the
cache be considered as soon-to-be-expired!
+
+LoadModule cache_module modules/mod_cache.so
+LoadModule disk_cache_module modules/mod_disk_cache.so
+LoadModule mem_cache_module modules/mod_mem_cache.so
+LoadModule cache_requester_module modules/mod_cache_requester.so
+
+<IfModule mod_cache.c>
+ CacheIgnoreCacheControl On
+ <IfModule mod_mem_cache.c>
+ CacheEnable mem /
+ MCacheSize 4096
+ MCacheMaxObjectCount 100
+ MCacheMinObjectSize 1
+ MCacheMaxObjectSize 2048
+ </IfModule>
+
+ <IfModule mod_cache_requester.c>
+ CacheRequesterThreads 2
+ CacheRequesterMaxQueueSize 10
+ CacheRequesterSoonToExpireTime 1000000
+ CacheRequesterSecretCode ABCDEF
+ </IfModule>
+</IfModule>
+
Added: httpd/mod_cache_requester/trunk/mod_cache_requester.c
URL: http://svn.apache.org/viewcvs/httpd/mod_cache_requester/trunk/mod_cache_requester.c?rev=380618&view=auto
==============================================================================
--- httpd/mod_cache_requester/trunk/mod_cache_requester.c (added)
+++ httpd/mod_cache_requester/trunk/mod_cache_requester.c Fri Feb 24 00:21:50 2006
@@ -0,0 +1,428 @@
+/* Copyright 2001-2005 The Apache Software Foundation or its licensors, as
+ * applicable.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+This mod-cache-requester modules starts background threads which keeps monitoring soon to
expire pages from cache.
+and if such pages are 'popular' then those pages are re-requested using libcurl so that such
pages are not removed from the cache.
+ */
+
+#define CORE_PRIVATE
+#include "mod_cache_requester.h"
+#include <curl/curl.h>
+
+#if APR_HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#if !APR_HAS_THREADS
+#error This module does not currently compile unless you have a thread-capable APR. Sorry!
+#endif
+
+# define DEFAULT_NUMBER_OF_THREADS 2
+# define DEFAULT_QUEUE_SIZE 10
+# define DEFAULT_EXPIRING_SIGNAL_TIME_IN_SECONDS 1000
+# define DEFAULT_SECRET_CODE "SECRET_CODE"
+
+module AP_MODULE_DECLARE_DATA cache_requester_module;
+
+static cache_req_conf_t *sconf;
+static cache_requester *cache_req_obj;
+
+/**
+ * Priority queue management functions
+ */
+static long cache_req_get_priority(void *data) {
+ cache_req_expiring_page *expiring_page = (cache_req_expiring_page *) data;
+ return expiring_page->priority;
+}
+
+static void cache_req_set_pos(void *a, apr_ssize_t pos)
+{
+ cache_req_expiring_page *obj = (cache_req_expiring_page *)a;
+
+ apr_atomic_set32(&(obj->pos), pos);
+}
+
+static apr_ssize_t cache_req_get_pos(void *a)
+{
+ cache_req_expiring_page *obj = (cache_req_expiring_page *)a;
+ return apr_atomic_read32(&(obj->pos));
+}
+
+/**
+ * Queue Management operations. all those operations must be thread safe. use cache_req_obj->lock
to ensure mutual exclusion.
+ */
+static void create_expiring_page(cache_req_expiring_page **node, request_rec *r) {
+ char *request_uri;
+
+ *node = (cache_req_expiring_page *) malloc(sizeof(cache_req_expiring_page));
+ if(!node) {
+ return;
+ }
+ (*node)->priority = 0;
+
+ request_uri = apr_pstrcat(r->pool, r->server->server_hostname,
+ ":",
+ apr_itoa(r->pool, r->server->port),
+ r->uri, NULL);
+
+ (*node)->request_uri = malloc(strlen(request_uri) + 1);
+ strcpy((*node)->request_uri, request_uri);
+
+ (*node)->uri = malloc(strlen(r->uri) + 1);
+ strcpy((*node)->uri, r->uri);
+
+ (*node)->next = NULL;
+}
+
+static void delete_expiring_page(cache_req_expiring_page *node) {
+ if(node->request_uri) {
+ free(node->request_uri);
+ }
+ if(node->uri) {
+ free(node->uri);
+ }
+
+ if(node) {
+ free(node);
+ }
+}
+
+static cache_req_expiring_page *insert_into_queue(request_rec *r) {
+
+ cache_req_expiring_page *value;
+ cache_req_expiring_page *new_value;
+ int queue_size;
+
+ queue_size = cache_pq_size(cache_req_obj->cache_request_pq);
+
+ if(queue_size >= sconf->queue_size) {
+ ap_log_error(APLOG_MARK, APLOG_INFO, 0, r->server,
+ "cache_requester: cache size queue exceeded");
+ return NULL;
+ }
+
+ /*check if the entry for this page already exists or not.*/
+ value = apr_hash_get(cache_req_obj->expiring_page_hash, r->uri, APR_HASH_KEY_STRING);
+
+ /* if not, then insert it into the queue, initialize the priority*/
+ if(value == NULL) {
+ create_expiring_page(&new_value, r);
+ apr_hash_set(cache_req_obj->expiring_page_hash, new_value->uri, APR_HASH_KEY_STRING,
new_value);
+ cache_pq_insert(cache_req_obj->cache_request_pq, new_value);
+
+ // if this is the first entry added in the queue, then lets wake up one of the threads.
+ if(queue_size == 0) {
+ apr_thread_cond_signal(cache_req_obj->cond);
+ }
+ return new_value;
+ }
+ /* if its already there, then increment the priority by one.*/
+ else {
+ value->priority++;
+ cache_pq_change_priority(cache_req_obj->cache_request_pq, (value->priority
- 1), value->priority, value);
+ return value;
+ }
+}
+
+/* delete the entry from the queue. */
+static int delete_from_queue(cache_req_expiring_page *expiring_page) {
+ cache_req_expiring_page *value;
+
+ if(!expiring_page || !(expiring_page->uri)) {
+ return DECLINED;
+ }
+
+ value = apr_hash_get(cache_req_obj->expiring_page_hash, expiring_page->uri, APR_HASH_KEY_STRING);
+ if(value == NULL) {
+ return DECLINED;
+ }
+ else {
+ apr_hash_set(cache_req_obj->expiring_page_hash, expiring_page->uri, APR_HASH_KEY_STRING,
NULL);
+ delete_expiring_page(value);
+ return OK;
+ }
+}
+
+/*
+ * insert the soon-to-expire page into the queue.
+ */
+int notify_cache_requester(request_rec *r, cache_handle_t *handle) {
+ const char *is_from_cache_requester;
+ cache_req_expiring_page *value;
+
+ is_from_cache_requester = apr_table_get(r->headers_in, "Cache-Requester");
+ if(is_from_cache_requester) {
+
+ /* verify the request is actually made by mod-cache-requester. */
+ if(!strcmp(is_from_cache_requester, sconf->secret_code)) {
+ return DECLINED;
+ }
+ return OK;
+ }
+
+ /*
+ See the expiry time of the page. If its soon to expire,
+ then notify add this page in the queue (if already there then increment the counter)
to re-request
+ */
+ if(!is_soon_to_expire(handle)) {
+ apr_thread_mutex_lock(cache_req_obj->lock);
+ value = insert_into_queue(r);
+ apr_thread_mutex_unlock(cache_req_obj->lock);
+
+ }
+ return OK;
+}
+
+/*
+ * Checks whether this page is soon-to-expire. returns 0 if its the case
+ */
+int is_soon_to_expire(cache_handle_t *handle) {
+ cache_info *info;
+ if(!handle) {
+ return -1;
+ }
+
+ info = &(handle->cache_obj->info);
+ if((apr_time_sec(info->expire) - apr_time_sec(apr_time_now())) < sconf->expiring_signal_time_in_seconds)
{
+ return 0;
+ }
+ return -1;
+}
+
+/*
+ * Process the most popular page, remove it from the queue (in request_page)
+ * - re request it using libcurl - make_curl_request
+ */
+static size_t process_received_data(void *buffer, size_t size, size_t nmemb, void *userp);
+
+void make_curl_request(char *request_uri, char *secret_code_header) {
+ CURL *curl;
+ struct curl_slist *headers=NULL;
+ int returnValue;
+
+ curl = curl_easy_init();
+ headers = curl_slist_append(headers, secret_code_header);
+
+ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+ curl_easy_setopt(curl, CURLOPT_URL, request_uri);
+ curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, process_received_data);
+
+ returnValue = curl_easy_perform(curl);
+
+ ap_log_error(APLOG_MARK, APLOG_INFO, 0, NULL, "cache_requester: URI: %s re-requested,
with return value: %d", request_uri, returnValue);
+
+}
+
+void *APR_THREAD_FUNC request_page(apr_thread_t *thd, void * dummy) {
+ cache_req_expiring_page *curr_obj;
+ request_rec *r;
+ request_rec *rnew;
+ char *secret_code_header;
+
+ secret_code_header = (char *) malloc (sizeof("Cache-Requester: ") + sizeof(sconf->secret_code)
+ 1);
+ strcpy(secret_code_header, "Cache-Requester: ");
+ strcat(secret_code_header, sconf->secret_code);
+
+ for(;;) {
+ apr_thread_mutex_lock(cache_req_obj->lock);
+
+ /* If no entry in the queue => sleep */
+ while(cache_pq_size(cache_req_obj->cache_request_pq) == 0) {
+ apr_thread_cond_wait(cache_req_obj->cond, cache_req_obj->lock);
+ }
+
+ /*Pop the most popular page from the queue*/
+ curr_obj = cache_pq_pop(cache_req_obj->cache_request_pq);
+
+ if(!curr_obj) {
+ apr_thread_mutex_unlock(cache_req_obj->lock);
+ continue;
+ }
+
+ /*Remove entry from the expiring_page and insert it into expiring_page hashmap.*/
+ apr_hash_set(cache_req_obj->expiring_page_hash, curr_obj->uri, APR_HASH_KEY_STRING,
NULL);
+ apr_thread_mutex_unlock(cache_req_obj->lock);
+
+ //Re-request the page.
+ make_curl_request(curr_obj->request_uri, secret_code_header);
+
+ //Delete the object.
+ delete_expiring_page(curr_obj);
+ }
+ free(secret_code_header);
+ return NULL;
+}
+
+static size_t process_received_data(void *buffer, size_t size, size_t nmemb, void *userp)
{
+ /*
+ * throw away data received as a part of curl request.
+ */
+}
+
+/**
+ * Directive Configurations.
+ */
+static void *create_cache_requester_config(apr_pool_t *p, server_rec *s)
+{
+ sconf = apr_pcalloc(p, sizeof(cache_req_conf_t));
+ sconf->number_of_threads = DEFAULT_NUMBER_OF_THREADS;
+ sconf->queue_size = DEFAULT_QUEUE_SIZE;
+ sconf->expiring_signal_time_in_seconds = DEFAULT_EXPIRING_SIGNAL_TIME_IN_SECONDS;
+ sconf->secret_code = DEFAULT_SECRET_CODE;
+ return sconf;
+}
+
+static const char *set_no_of_threads(cmd_parms *parms, void *in_struct_ptr, const char *arg)
{
+ apr_size_t val;
+
+ if (sscanf(arg, "%" APR_SIZE_T_FMT, &val) != 1) {
+ return "CacheRequesterThreads argument must be an integer representing number of
therads.";
+ }
+ if(val < 1) {
+ return "CacheRequesterThreads argument must not be less than 1.";
+ }
+ sconf->number_of_threads = val;
+ return NULL;
+}
+
+static const char *set_max_queue_size(cmd_parms *parms, void *in_struct_ptr, const char *arg)
{
+ apr_size_t val;
+
+ if (sscanf(arg, "%" APR_SIZE_T_FMT, &val) != 1) {
+ return "CacheRequesterMaxQueueSize argument must be an integer representing number
of therads.";
+ }
+
+ if(val < 1) {
+ return "CacheRequesterMaxQueueSize argument must be greater than 1.";
+ }
+ sconf->queue_size = val;
+ return NULL;
+}
+
+static const char *set_soon_to_expire_time(cmd_parms *parms, void *in_struct_ptr, const char
*arg) {
+ apr_size_t val;
+
+ if (sscanf(arg, "%" APR_SIZE_T_FMT, &val) != 1) {
+ return "CacheRequesterSoonToExpireTime value must be a positive integer";
+ }
+ if(val < 1) {
+ return "CacheRequesterSoonToExpireTime argument must be greater than 1.";
+ }
+
+ sconf->expiring_signal_time_in_seconds = val;
+ return NULL;
+}
+
+static const char *set_secret_code(cmd_parms *parms, void *name, const char *arg) {
+
+ sconf->secret_code = arg;
+ return NULL;
+}
+
+static const command_rec cache_req_cmds[] =
+{
+ AP_INIT_TAKE1("CacheRequesterThreads", set_no_of_threads, NULL, RSRC_CONF,
+ "Number of cache requester threads"),
+ AP_INIT_TAKE1("CacheRequesterMaxQueueSize", set_max_queue_size, NULL, RSRC_CONF,
+ "The maximum number of objects allowed to be placed in the cache requester
queue."),
+ AP_INIT_TAKE1("CacheRequesterSoonToExpireTime", set_soon_to_expire_time, NULL, RSRC_CONF,
+ "The required time remaining to expire a page to be able to be re-requested
(in seconds)"),
+ AP_INIT_TAKE1("CacheRequesterSecretCode", set_secret_code, NULL, RSRC_CONF,
+ "the secret code to authenticate requests made by mod-cache-requester"),
+
+ {NULL}
+};
+
+/**
+ * initialize and start the threads.
+ */
+static int init_cache_req_obj_threads(apr_pool_t *p, server_rec *s) {
+ int i;
+ cache_req_thread *current_thread;
+
+ current_thread = cache_req_obj->thread_pool->queue;
+ apr_thread_create(&(current_thread->handle), NULL, request_page, NULL, p);
+ current_thread->next = NULL;
+ for(i=0; i < sconf->number_of_threads-1; i++) {
+ current_thread->next = (cache_req_thread *) apr_palloc(p, sizeof(cache_req_thread));
+ apr_thread_create(&(current_thread->next->handle), NULL, request_page,
NULL, p);
+ current_thread->next->next = NULL;
+ current_thread = current_thread->next;
+ }
+}
+
+/**
+ * module configuration stuff.
+ */
+
+static int cache_requester_child_init(apr_pool_t *p, apr_pool_t *plog,
+ apr_pool_t *ptemp, server_rec *s) {
+ //Here Threads would be instantiated.
+ init_cache_req_obj_threads(p, s);
+}
+
+static int cache_requester_post_config(apr_pool_t *p, apr_pool_t *plog,
+ apr_pool_t *ptemp, server_rec *s) {
+ // Initialize the curl lib.
+ curl_global_init(CURL_GLOBAL_ALL);
+
+ //Create main object first.
+ cache_req_obj = (cache_requester *) apr_palloc(p, sizeof(cache_requester));
+
+ // instantiate the thread pool.
+ cache_req_obj->thread_pool = (cache_req_thread_pool *) apr_palloc(p, sizeof(cache_req_thread_pool));
+
+ // create the lock, condition variable and priority queue.
+ apr_thread_mutex_create(&(cache_req_obj->lock), APR_THREAD_MUTEX_DEFAULT, p);
+ apr_thread_cond_create(&(cache_req_obj->cond), p);
+ cache_req_obj->cache_request_pq = cache_pq_init(sconf->queue_size,
+ cache_req_get_priority,
+ cache_req_get_pos,
+ cache_req_set_pos);
+
+ // instantiate the hashmaps.
+ cache_req_obj->expiring_page_hash = apr_hash_make(p);
+
+ //Instantiate the thread queue.
+ cache_req_obj->thread_pool->queue = (cache_req_thread *) apr_palloc(p, sizeof(cache_req_thread));
+
+ return OK;
+}
+
+static const cache_req_provider cache_requester_provider =
+{
+ ¬ify_cache_requester,
+};
+
+static void register_hooks(apr_pool_t *p)
+{
+ ap_hook_post_config(cache_requester_post_config, NULL, NULL, APR_HOOK_MIDDLE);
+ ap_hook_child_init(cache_requester_child_init, NULL, NULL, APR_HOOK_MIDDLE);
+ ap_register_provider(p, CACHE_REQUESTER_PROVIDER_GROUP, "cache_req", "0",
+ &cache_requester_provider);
+}
+
+module AP_MODULE_DECLARE_DATA cache_requester_module =
+{
+ STANDARD20_MODULE_STUFF,
+ NULL, /* create per-directory config structure */
+ NULL, /* merge per-directory config structures */
+ create_cache_requester_config, /* create per-server config structure */
+ NULL, /* merge per-server config structures */
+ cache_req_cmds, /* command apr_table_t */
+ register_hooks
+};
Added: httpd/mod_cache_requester/trunk/mod_cache_requester.h
URL: http://svn.apache.org/viewcvs/httpd/mod_cache_requester/trunk/mod_cache_requester.h?rev=380618&view=auto
==============================================================================
--- httpd/mod_cache_requester/trunk/mod_cache_requester.h (added)
+++ httpd/mod_cache_requester/trunk/mod_cache_requester.h Fri Feb 24 00:21:50 2006
@@ -0,0 +1,78 @@
+/* Copyright 2001-2005 The Apache Software Foundation or its licensors, as
+ * applicable.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CACHE_REQUESTER_THREADPOOL_H
+#define CACHE_REQUESTER_THREADPOOL_H
+#endif
+
+#include "apr_thread_proc.h"
+#include "mod_cache.h"
+#include "cache_pqueue.h"
+#include "cache_cache.h"
+#include "ap_provider.h"
+#include "apr_pools.h"
+#include "ap_mpm.h"
+#include "apr_thread_mutex.h"
+#include "apr_thread_cond.h"
+#include "apr_strings.h"
+#include <curl/curl.h>
+
+
+
+
+// Structure to store the parameters. Currently we support 'number of threads' and 'queue_size
of soon-to-expire pages' parameters.
+typedef struct cache_req_conf {
+ long number_of_threads;
+ long queue_size;
+ long expiring_signal_time_in_seconds;
+ char *secret_code;
+} cache_req_conf_t;
+
+
+// Linked list structure of threads. each instance contains pointer to the thread and pointer
to next instance of the linked list.
+typedef struct cache_req_thread {
+ apr_thread_t *handle;
+ struct cache_req_thread *next;
+}cache_req_thread;
+
+// request object to be stored in the queue for each soon-to-expire page.
+typedef struct cache_req_expiring_page {
+ char *uri;
+ char *request_uri;
+ long priority;
+ apr_uint32_t pos;
+ struct cache_req_expiring_page *next;
+}cache_req_expiring_page;
+
+// thread pool.
+// contains thread queue, lock, condition variable.
+typedef struct cache_req_thread_pool {
+ cache_req_thread *queue;
+
+ //cache_requester_request *processing_requests;
+}cache_req_thread_pool;
+
+// main structure, which uses all the above structures and keeps them organized.
+typedef struct cache_requester {
+ cache_req_thread_pool *thread_pool;
+
+ cache_pqueue_t *cache_request_pq;
+ apr_hash_t *expiring_page_hash;
+ apr_thread_mutex_t *lock;
+ apr_thread_cond_t *cond;
+
+ apr_pool_t *pool;
+}cache_requester;
|