Mailing-List: contact modules-dev-help@httpd.apache.org; run by ezmlm
Precedence: bulk
Reply-To: modules-dev@httpd.apache.org
Received-SPF: pass (herse.apache.org: local policy)
Content-class: urn:content-classes:message
MIME-Version: 1.0
Content-Type: text/plain;
	charset="us-ascii"
Content-Transfer-Encoding: quoted-printable
Subject: RE: help on gathering data from the response
Date: Wed, 10 Jan 2007 10:19:45 +0530
Message-ID: <318C84E390AAFD4E9BDADA9E3639067405438B@ex-ind-u0.vanenburg.com>
In-Reply-To: <318C84E390AAFD4E9BDADA9E3639067405437F@ex-ind-u0.vanenburg.com>
Thread-Topic: help on gathering data from the response
Thread-Index: Acczr1OhjNDSF0onR6mvkvWOf4J+ogAwxWmg
From: "Sai Jai Ganesh Gurubaran" <sgurubar@cordys.com>
To: <modules-dev@httpd.apache.org>

Can anyone just validate the code that is given below?


-----Original Message-----
From: Sai Jai Ganesh Gurubaran=20
Sent: Tuesday, January 09, 2007 11:01 AM
To: modules-dev@httpd.apache.org
Subject: help on gathering data from the response

Hi All,
	The requirement for us is to write a apache module that collects
printable data  from the response.
For this, we have used the idea from mod_streamav code.
The memory and CPU usage of the Apache has never crossed 40 MB per
process and works fine.
Recently we are getting Apache Outages (i.e. Apache is running but not
responding).
We are not in a position to remove this module and test for the same due
to the functionality required :-(
The traffic of the site is 25Mbps and Apache 59 on RHEL 3 keeps going
for days without any problems.
This outage is very random. There are no logs in the error log or access
log indicating problems.
Any help from the developers will be highly appreciated.

Here is the code we use to perform the above said operation.

Regards,
Ganesh


/* Structure to hold the context and other relevant data */

typedef struct ffifilter_ctx {

  char *buffer;         		/*buffer to store data to be
scanned.*/
  int curr_buffer_length;			/*length of content
currently stored in buffer*/
  int has_been_categorized_already;    	/* to ignore content more than
max_content_length_limit being processed again*/=20

  apr_bucket_brigade *bb;		/*bucket brigade to be passed to
next filter, content of the brigade will be created during scanning
process*/

} ffi_ctx;


static apr_status_t testFilter_run(ap_filter_t *f,
					apr_bucket_brigade *bb)
{
	 request_rec* r =3D f->r;

	// To take into account all the MIME Types related to text
	if ( (r->content_type !=3D NULL) &&
(strstr(r->content_type,text)=3D=3DNULL))
    	{
        		return ap_pass_brigade(f->next,bb);
    	}
	if(r->status !=3D 200)
	{
		ap_log_rerror(APLOG_MARK, APLOG_INFO, 0,f->r,"Got a %d
status for request : %s , quitting." , r->status, r->uri);
		return ap_pass_brigade(f->next,bb);
	}
=09
	ffi_ctx *ctx=3D (ffi_ctx *)f->ctx;      /*filter context*/
	apr_bucket *curr_bucket;=09
=09
	if(ctx =3D=3D NULL)
  	{
		ap_log_rerror(APLOG_MARK, APLOG_INFO, 0,f->r,"Got a new
request: %s  [File name]%s",  f->r->uri, f->r->filename);
  		if(testFilter_init(f) =3D=3D APR_ENOMEM)
		{
			/* No Memory availble*/
			return OK;
		}
		ctx =3D (ffi_ctx *)f->ctx;
  	}
=09
	if(testFilter_save_brigade(f,bb)=3D=3DEOS_NOT_FOUND)
	{
		ap_log_rerror(APLOG_MARK, APLOG_INFO, 0,f->r,"Returning
OK, to fetch more content.%s", f->r->uri);
		return OK;
	}

	/* If there is no content then just pass on*/
	if(ctx->curr_buffer_length =3D=3D 0)
	{
		ap_log_rerror(APLOG_MARK, APLOG_INFO, 0,f->r,"Got a null
content for %s", f->r->uri);
		testFilter_send(f);
	        return testFilter_cleanup(f,bb);
	}
=09
	testFilter_send(f);
	return testFilter_cleanup(f,bb);
}


/*************************  TRIAL ***********************************/
/*initialize context when a new request invoke the dynamic filter
  1.allocate space for buffer
  2.init bb for next filter
  if fail, return -1; else return 0;
*/
static int testFilter_init(ap_filter_t *f )
{
	ffi_ctx *ctx =3D (ffi_ctx *)f->ctx;

	/*no need to do initialization here*/
	if(ctx !=3D NULL)
	{
	    return 0;
  	}

	/* allocate a new context structure*/
	f->ctx =3D apr_pcalloc(f->r->pool,sizeof(ffi_ctx));
	ctx =3D (ffi_ctx *)f->ctx;

	/*create a new bucket brigade*/
	ctx->bb =3D apr_brigade_create(f->r->pool, f->c->bucket_alloc);
=09
	//initialize to the "max_content_length_limit"
      	ctx->buffer =3D (char *)malloc(max_content_length_limit + 1);
	if(ctx->buffer =3D=3D NULL)
        {
               ap_log_rerror(APLOG_MARK, APLOG_ERR, 0,
f->r,"[testFilter_init] Could not allocate memory for ctx->buffer.
Quit");
               return APR_ENOMEM;
        }


	ctx->curr_buffer_length =3D 0;
	ctx->has_been_categorized_already =3D 0;=20
	return 0;
}


/*
Aim is to collect only printable data of the response
1. Define a maximum limit on length beyond which data gathering is not
done.
2. Pass on the brigade to the next filter once we are done.

*/


//*****************************************************************
int testFilter_save_brigade(ap_filter_t *f, apr_bucket_brigade *bb)
{
	ffi_ctx  *ctx =3D (ffi_ctx *)f->ctx;
	const char *buffer;
	apr_size_t len, loop;
	apr_bucket *curr_bucket, *tmp_bucket;
	apr_size_t stopGatheringData =3D 0;
	apr_status_t return_value;
=20
	for ( 	curr_bucket =3D APR_BRIGADE_FIRST(bb);
          	curr_bucket !=3D APR_BRIGADE_SENTINEL(bb);
		curr_bucket =3D APR_BUCKET_NEXT(curr_bucket))=20
	  {
		if(APR_BUCKET_IS_EOS(curr_bucket))
                {
                        ctx->buffer[ctx->curr_buffer_length] =3D '\0';
=20
APR_BRIGADE_INSERT_TAIL(ctx->bb,apr_bucket_eos_create(f->c->bucket_alloc
));
                        /*delete curret bucket from bb*/
                        apr_bucket_delete(curr_bucket);
                        return EOS_FOUND;
                }
                else if(APR_BUCKET_IS_FLUSH(curr_bucket))
                {
                     /* Do we need to delete this bucket???*/
                     apr_bucket_delete(curr_bucket);
		     continue;
                }
		else
		{
			/*Gather the data from the bucket */
			if(stopGatheringData !=3D 1)
			{
				return_value =3D
apr_bucket_read(curr_bucket,&buffer,&len,APR_BLOCK_READ);
				if(return_value !=3D APR_SUCCESS)
        			{
					ap_log_rerror(APLOG_MARK,
APLOG_ERR, 0,f->r,
=09
"[testFilter_save_brigade] apr_bucket_read() call failed.");
					return EOS_FOUND;
				}
			=09
				for(loop=3D0 ; loop < len ; ++loop)
        			{
		                	if(ctx->curr_buffer_length <
max_content_length_limit)
                			{
						/* Gather only the
printable characters*/
=20
if(isprint(buffer[loop]))
                        			{
=09
ctx->buffer[ctx->curr_buffer_length++]=3Dbuffer[loop];
        	                		}
                			}=09
                			else
		                	{       /*reached the max limit,
break out of the loop*/
                		        	stopGatheringData =3D 1;
=09
ctx->buffer[ctx->curr_buffer_length] =3D '\0';=09
=09
ctx->has_been_categorized_already =3D 1;
						break;
                			}
        			}
				f->ctx =3D ctx;
			}

	      		/*insert current bucket into ctx->bb, pending to
be passed to next filter     */
		      	return_value =3D
apr_bucket_copy(curr_bucket,&tmp_bucket);
			if(return_value !=3D APR_SUCCESS)
                        {
                                ap_log_rerror(APLOG_MARK, APLOG_ERR,
0,f->r,
=20
"[testFilter_save_brigade] apr_bucket_copy() call failed.");
				return EOS_FOUND;
                        }

      			APR_BRIGADE_INSERT_TAIL(ctx->bb,tmp_bucket);

		      	/*delete curret bucket from bb*/
      			apr_bucket_delete(curr_bucket);
    		}
    	}// For Loop=20

	/* We have exceeded the quota, let's stop right here. Else we
will take up all the RAM :( */
	if(stopGatheringData =3D=3D 1)
  	{
		ap_log_rerror(APLOG_MARK, APLOG_INFO, 0,f->r,
                       "Faking EOS - Reached the size specified in the
max_content_length_limit directive");
        	return EOS_FOUND;
  	}

	return EOS_NOT_FOUND;
}

//*****************************************************************
/*send ctx->bb to next filter */
static void testFilter_send(ap_filter_t *f)
{
	ffi_ctx *ctx =3D (ffi_ctx *)f->ctx;
=09
APR_BRIGADE_INSERT_TAIL(ctx->bb,apr_bucket_flush_create(f->c->bucket_all
oc));
	ap_pass_brigade(f->next,ctx->bb);
}

//*****************************************************************
/*remove all buckets in f->ctx->bb*/
static apr_status_t testFilter_cleanbrigade(ap_filter_t *f)
{
	apr_bucket *curr_bucket;
	ffi_ctx *ctx =3D (ffi_ctx *)f->ctx;

	while (!APR_BRIGADE_EMPTY(ctx->bb)) {
          curr_bucket =3D APR_BRIGADE_FIRST(ctx->bb);
          apr_bucket_delete(curr_bucket);
        }
	return APR_SUCCESS;
}
//*****************************************************************
/*cleanup everything. */
static apr_status_t testFilter_cleanup(ap_filter_t *f,apr_bucket_brigade
*bb)
{
	ffi_ctx *ctx =3D (ffi_ctx *)f->ctx;
  	if(ctx->buffer)
  	{
		free(ctx->buffer);
  		ctx->buffer =3D NULL;
	}
	apr_brigade_destroy(bb);
	return APR_SUCCESS;
}
/*************************  TRIAL END
***********************************/

***********************************************************************
The information in this message is confidential and may be legally
privileged. It is intended solely for the addressee. Access to this=20
message by anyone else is unauthorized. If you are not the=20
intended recipient, any disclosure, copying, or distribution of the=20
message, or any action or omission taken by you in reliance on=20
it is prohibited and may be unlawful. Please immediately contact=20
the sender if you have received this message in error. This email=20
does not constitute any commitment from Cordys Holding BV or=20
any of its subsidiaries except when expressly agreed in a written=20
agreement between the intended recipient and=20
Cordys Holding BV or its subsidiaries.
***********************************************************************