Mailing-List: contact dev-help@hc.apache.org; run by ezmlm
Precedence: bulk
Reply-To: "HttpComponents Project" <dev@hc.apache.org>
Date: Fri, 23 Nov 2012 08:12:58 +0000 (UTC)
From: "Bryant Harris (JIRA)" <jira@apache.org>
To: dev@hc.apache.org
Message-ID: <1480120369.18777.1353658378153.JavaMail.jiratomcat@arcas>
In-Reply-To: <1091689132.5727.1353400859509.JavaMail.jiratomcat@arcas>
Subject: [jira] [Commented] (HTTPCLIENT-1263) CachingHttpClient not
 consuming backend HttpResponse entity causing
 PoolingClientConnectionManager to become unresponsive
MIME-Version: 1.0
Content-Type: text/plain; charset=utf-8
Content-Transfer-Encoding: 7bit


    [ https://issues.apache.org/jira/browse/HTTPCLIENT-1263?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13503103#comment-13503103 ] 

Bryant Harris commented on HTTPCLIENT-1263:
-------------------------------------------

Sure, Here's a simple example to demonstrate, complied using 4.2.1 jars


package sample;

import java.util.Random;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.TimeUnit;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DecompressingHttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.cache.CacheConfig;
import org.apache.http.impl.client.cache.CachingHttpClient;
import org.apache.http.impl.conn.PoolingClientConnectionManager;
import org.apache.http.util.EntityUtils;

/**
 * Simple test to demonstrate hanging issue when using a pooling http client.
 * 
 * Test creates 3 reader threads sharing a single HttpClient. performing reads
 * across 3 different domains.
 * 
 * Each thread properly consumes its content.
 * 
 * When using a caching client (useCachingClient = true) it will hang.  When
 * using standard client (useCachingClient = false) it works as expected
 * @author bryant_harris
 *
 */
public class HttpClient1263 {
	/** Set this value true demonstrates the bug, setting it to false shows that bypassing caching works as expected */
	static boolean useCachingClient = false;
	
	static HttpClient standardClient = null;
	static HttpClient cachedClient = null;
	static PoolingClientConnectionManager connectionManager = null;
	
	/**
	 * Standard client, small number of connections, wrapped in a DecompressingHttpClient.
	 * @return
	 */
	protected static synchronized HttpClient getStandardClient() {
		if ( standardClient == null ) {
			connectionManager = new PoolingClientConnectionManager();
			connectionManager.setMaxTotal(2);
			connectionManager.closeIdleConnections(120, TimeUnit.SECONDS);
			standardClient = new DecompressingHttpClient( new DefaultHttpClient (connectionManager));
	     }
		return standardClient;
	}
	
	/**
	 * Wrapping standard client in pool, to expose the issue, using very small
	 * cache to force network loads.
	 * @return
	 */
	protected static synchronized HttpClient getCachedClient() {
		if ( cachedClient == null ) {
			CacheConfig cacheConfig = new CacheConfig();
			cacheConfig.setMaxObjectSize( 512*1024 );
			cacheConfig.setMaxCacheEntries( 1 );
			cachedClient = new CachingHttpClient(getStandardClient(),
												 cacheConfig);
		}
		
		return cachedClient;
	}
	
	static Timer timer;

	/**
	 * @param args
	 */
	public static void main(String[] args) throws Exception {
		timer = new Timer("Hang detecting Timer",  true);

		// create some concurrent readers that will hit different domains.
		HttpReaderThread t1 = new HttpReaderThread("http://www.apple.com");
		HttpReaderThread t2 = new HttpReaderThread("http://www.google.com");
		HttpReaderThread t3 = new HttpReaderThread("http://www.yahoo.com");
		
		t1.start();
		t2.start();
		t3.start();
	}

	static Random random = new Random(System.currentTimeMillis());
	static class HttpReaderThread extends Thread {
		String url;
		TimerTask hungTimer;
		
		public HttpReaderThread(String theUrl) {
			url = theUrl;
		}
		
		public void run() {
			for ( int i=0; i<20; i++ ) {
				try {
					HttpGet method = new HttpGet(url);
					
					// Use client to read a request
					HttpClient client = useCachingClient ? getCachedClient() : getStandardClient();
					HttpResponse response = client.execute(method);
					HttpEntity entity = response.getEntity();
					EntityUtils.consume(entity);
					
					Thread.sleep(Math.abs(random.nextLong() % 100));
					
					// resetting a hung timer that will fire if we dont' complete
					// a read in a reasonable amount of time.
					if ( hungTimer != null )
						hungTimer.cancel();
					hungTimer = new TimerTask() {
						public void run() {
							System.err.println(url + " Thread is hung");
						}
					};
					timer.schedule(hungTimer, 5000);
				}
				catch (Exception e) {
					e.printStackTrace();
				}
			}
			
			if ( hungTimer != null )
				hungTimer.cancel();
			System.out.println(url + " Thread completed.");
		}
	}
}

                
> CachingHttpClient not consuming backend HttpResponse entity causing PoolingClientConnectionManager to become unresponsive
> -------------------------------------------------------------------------------------------------------------------------
>
>                 Key: HTTPCLIENT-1263
>                 URL: https://issues.apache.org/jira/browse/HTTPCLIENT-1263
>             Project: HttpComponents HttpClient
>          Issue Type: Bug
>          Components: HttpClient
>    Affects Versions: 4.2.1
>            Reporter: Bryant Harris
>              Labels: cache, connection-pooling, consumed
>
> I've noticed that when issuing requests via a pooled and cached HttpClient that the client eventually becomes unresponsive (which appears to be because an HttpEntity is not getting consumed properly).
> Steps to reproduce.
> 1. Here is how I've configured all the relevant classes.
>     HttpClient standardClient = null;
>     HttpClient cachedClient = null;
>     PoolingClientConnectionManager connectionManager = null;
>     protected synchronized HttpClient getStandardClient() {
>       if ( standardClient == null ) {
>         connectionManager = new PoolingClientConnectionManager();
>         connectionManager.setMaxTotal(2);
>         connectionManager.closeIdleConnections(120, TimeUnit.SECONDS);
>         standardClient = new DecompressingHttpClient( new DefaultHttpClient (connectionManager));
>       }
>       return standardClient;
>     }
> protected synchronized HttpClient getCachedClient() {
>   if ( cachedClient == null ) {
>     CacheConfig cacheConfig = new CacheConfig();
>     cacheConfig.setMaxObjectSize( 512*1024 );
>     cacheConfig.setMaxCacheEntries( 10 );
>     cachedClient = new CachingHttpClient(getStandardClient(),
>                                          getCacheStorage(),
>                                          cacheConfig);
>   }
>   return cachedClient;
> }
> As you can see I have two http clients. A caching http client that wraps the standard client.
> Now what I've found is that if I remove cachedClient and only use standardClient, I don't have any issues with the pool hanging and orphaned connections.
> 2.  Here is my code for how I issue and consume requests
>     HttpClient httpClient = cacheOkay ? getCachedClient() : getStandardClient();  
>     HttpResponse response = httpClient.execute(request, localContext);
>     HttpEntity resEntity = response.getEntity();  
>     int responseStatus = response.getStatusLine().getStatusCode();
>     byte[] responseBody = EntityUtils.toByteArray(resEntity);
>     EntityUtils.consume(resEntity);
> If you set up a test like this and use the cached client, it will hang fairly quickly.
> I've been able to work around this by creating the CachingHttpClient as follows:
>     protected synchronized HttpClient getCachedClient() {
>     	if ( cachedClient == null ) {
>     		WhosHereApplication application = WhosHereApplication.getInstance();
>     		cachedClient = new CachingHttpClient(getStandardClient(),
>     											 new HeapResourceFactory() {
> 													@Override
> 													public Resource generate(
> 															String requestId,
> 															InputStream instream,
> 															InputLimit limit)
> 															throws IOException {
> 														try {
> 															return super.generate(requestId, instream, limit);
> 														}
> 														finally {
> 															instream.close();
> 														}
> 													}
>     			
>     											 },
>     				                             application.getCacheStorage(),
>     				                             application.getCacheConfig());
>     		Log.i(tag, "Creating CachingHttpClient");
>     	}
>     		
>     	return cachedClient;
>     }
> Notice the inline subclass of HeapResourceFactory where I add the stream close call.  Once I add this the caching client no longer freezes up.
> I'm not familiar enough with the source code to pinpoint the issue, but appears the back end entity is not getting consumed properly, forcing this work around.

--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@hc.apache.org
For additional commands, e-mail: dev-help@hc.apache.org