hc-httpclient-users mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "bit1129@163.com" <bit1...@163.com>
Subject Re: httpclient4 is extremely slow than correpsoning code of HttpClient3
Date Tue, 07 Jul 2015 14:05:28 GMT

Can someone kindly help me on this? Thanks a lot!




bit1129@163.com
 
From: bit1129@163.com
Date: 2015-07-07 20:51
To: httpclient-users
Subject: httpclient4 is extremely slow than correpsoning code of HttpClient3

I have following http client code(HttpUtils and HttpClientManager)with HttpClient 4.3.6, and
a httpclient code in 3.1, and a test case as well. I observed that HttpClient 4.3.6 is very
much slowly than HttpClient 3.1 code. In HttpClient 4.3.6 version code, there are a lot
socket time out error(the socket time is 10 seconds).

I pasted the related classes and code below. It is kind of long code, but I still would ask
you do me a favor to review the http client configuration that may cause the problem. Many
Thanks in advance.

Following codes include:
1. HttpClient 4.3.6 code to issue HttpGet request
2. HttpClient 3.1 code to issue HttpGet request
3. Test Case that demontrate the problem.


############################################Http Client 4.3.6 Code goes here############################################
///////////////////////////HttpUtils//////////////////////////////////////////
import java.util.ArrayList; 
import java.util.List; 
import java.util.Map; 

import org.apache.http.Consts; 
import org.apache.http.NameValuePair; 
import org.apache.http.client.config.RequestConfig; 
import org.apache.http.client.entity.UrlEncodedFormEntity; 
import org.apache.http.client.methods.HttpGet; 
import org.apache.http.client.methods.HttpPost; 
import org.apache.http.message.BasicNameValuePair; 
import org.slf4j.Logger; 
import org.slf4j.LoggerFactory; 



public class HttpUtils { 

///This is the util method that will be used to issue http request.
public static String httpInvoke(String httpUrl, Map<String, Object> parameters, RequestConfig
config){ 
    HttpClientManager httpClientConnectionManager = HttpClientManager.getHttpClientConnectionManagerInstance();

    HttpGet httpGet = new HttpGet(httpUrl); 
    if(config != null ){ 
           httpGet.setConfig(config); 
} 
String result = httpClientConnectionManager.execute(httpGet); 
return result; 
} 
}

//////////////////////////////////HttpClientManager/////////////////////////////////

import java.nio.charset.CodingErrorAction; 
import java.util.ArrayList; 
import java.util.Collection; 

import org.apache.commons.httpclient.Header; 
import org.apache.commons.httpclient.HttpStatus; 
import org.apache.commons.lang3.StringUtils; 
import org.apache.http.Consts; 
import org.apache.http.HttpEntity; 
import org.apache.http.HttpHost; 
import org.apache.http.client.config.RequestConfig; 
import org.apache.http.client.entity.GzipDecompressingEntity; 
import org.apache.http.client.methods.CloseableHttpResponse; 
import org.apache.http.client.methods.HttpGet; 
import org.apache.http.client.methods.HttpPost; 
import org.apache.http.config.ConnectionConfig; 
import org.apache.http.config.MessageConstraints; 
import org.apache.http.config.SocketConfig; 
import org.apache.http.impl.client.CloseableHttpClient; 
import org.apache.http.impl.client.HttpClientBuilder; 
import org.apache.http.impl.client.HttpClients; 
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; 
import org.apache.http.message.BasicHeader; 
import org.apache.http.params.CoreProtocolPNames; 
import org.apache.http.util.EntityUtils; 
import org.slf4j.Logger; 
import org.slf4j.LoggerFactory; 


///Main class that encapsulate the Http Client
public class HttpClientManager { 
private static Logger logger = LoggerFactory.getLogger(HttpClientManager.class); 

private static int defaultConnectionTimeout = 10*1000;  //connection timeout
private static int defaultSocketTimeout = 10*1000;  //socket time out
private static int connectionRequestTimeout = 10*1000;  //connection request timeout

private static int defaultMaxRouteConnections = 128; 
private static int defaultMaxTotalConnections = 1024; 

private static int defaultMaxHeaderCount = 200; 
private static int defaultMaxLineLength = 2000; 

private static String Charset = "utf-8"; 

private String proxyHost = null; 
private String proxyPort =null; 

private PoolingHttpClientConnectionManager cm; 

private final static HttpClientManager httpClientConnectionManager = new HttpClientManager();


private HttpClientManager() { 
logger.info("HttpClientManager initial!"); 
cm = new PoolingHttpClientConnectionManager(); 
cm.setMaxTotal(defaultMaxTotalConnections); 
cm.setDefaultMaxPerRoute(defaultMaxRouteConnections); 
} 


private CloseableHttpClient getHttpclient(){ 
HttpClientBuilder httpClientBuilder = HttpClients.custom(); 
httpClientBuilder.setConnectionManager(cm); 

RequestConfig.Builder requestConfigBuilder = RequestConfig.custom() 
.setConnectTimeout(defaultConnectionTimeout) 
.setSocketTimeout(defaultSocketTimeout) 
.setConnectionRequestTimeout(connectionRequestTimeout) 
.setExpectContinueEnabled(false) 
.setStaleConnectionCheckEnabled(true); 

if(StringUtils.isNotBlank(proxyHost) && StringUtils.isNotBlank(proxyPort)){ 
try{ 
logger.info("using proxy, proxyHost:{}, proxyPort:{}", proxyHost, proxyPort); 
int proxyPortInt = Integer.parseInt(proxyPort); 
requestConfigBuilder.setProxy(new HttpHost(proxyHost, proxyPortInt)); 
} catch(Exception e){ 
logger.error("parseInt proxyPort:{}", proxyPort, e); 
} 
} 

SocketConfig socketConfig = SocketConfig.custom().setTcpNoDelay(true).build(); 

MessageConstraints messageConstraints = MessageConstraints.custom().setMaxHeaderCount(defaultMaxHeaderCount).setMaxLineLength(defaultMaxLineLength).build();


ConnectionConfig connectionConfig = ConnectionConfig.custom() 
.setMalformedInputAction(CodingErrorAction.IGNORE) 
.setUnmappableInputAction(CodingErrorAction.IGNORE) 
.setCharset(Consts.UTF_8) 
.setMessageConstraints(messageConstraints).build(); 

Collection<BasicHeader> collection = new ArrayList<BasicHeader>(); 
collection.add(new BasicHeader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN;
rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3")); 
collection.add(new BasicHeader("Accept-Language", "zh-cn,zh,en-US,en;q=0.5")); 
collection.add(new BasicHeader("Accept-Charset", Charset)); 
collection.add(new BasicHeader("Accept-Encoding", "gzip")); 

httpClientBuilder.setDefaultRequestConfig(requestConfigBuilder.build()); 
httpClientBuilder.setDefaultSocketConfig(socketConfig); 
httpClientBuilder.setDefaultConnectionConfig(connectionConfig); 
httpClientBuilder.setDefaultHeaders(collection); 

return httpClientBuilder.build(); 
} 

///This is the method that will be call the execute the HttpGet request
public String execute(HttpGet httpGet) { 
String body = ""; 
try{ 
CloseableHttpClient httpclient = this.getHttpclient(); 
CloseableHttpResponse response = httpclient.execute(httpGet); 
int status = response.getStatusLine().getStatusCode(); 
try { 
if (status == HttpStatus.SC_OK) { 
HttpEntity entity = response.getEntity(); 
if (entity != null) { 
Header header = (Header) entity.getContentEncoding(); 
if(header != null && "gzip".equals(header.getValue())){ 
body = EntityUtils.toString(new GzipDecompressingEntity(entity), Charset); 
} else { 
body = EntityUtils.toString(entity, Charset); 
} 
} 
} else { 
logger.error("[httpClientManager] [fail] [httpGet:{}] [status:{}]", httpGet, status); 
} 
} finally { 
response.close(); 
} 
} catch(Exception e) { 
logger.error("[module:httpClientManager] [action:execute] [httpGet:{}] [error:{}] ", httpGet,
e.getMessage(), e); 
} 
return body; 
} 


////Singleton object that will be used to access httpClientConnectionManager
public static HttpClientManager getHttpClientConnectionManagerInstance(){ 
return httpClientConnectionManager; 
} 

} 


##################################################My HttpClient 1.3 code goes here###########################################

import org.apache.commons.httpclient.HttpClient; 
import org.apache.commons.httpclient.HttpMethod; 
import org.apache.commons.httpclient.methods.GetMethod; 

import java.io.BufferedReader; 
import java.io.IOException; 
import java.io.InputStream; 
import java.io.InputStreamReader; 

public class HttpClientUtil { 

private static final HttpClientUtil INSTANCE = new HttpClientUtil(); 

private static final int TIMEOUT = 10 * 1000; //10s 

private HttpClientUtil() { 
} 

///Actually, this class is more like a static class than Singleton 
public static HttpClientUtil getInstance() { 
return INSTANCE; 
} 

//brand new http client per request 
private HttpClient newHttpClient() { 
HttpClient client = new HttpClient(); 
client.getHttpConnectionManager().getParams().setConnectionTimeout(TIMEOUT); 
client.getHttpConnectionManager().getParams().setSoTimeout(TIMEOUT); 
return client; 
} 

//FIXME The encoding should be provided when convert the response binary stream into string

public static String responseBodyAsString(HttpMethod method) throws IOException { 
BufferedReader br = null; 
String lsr = System.getProperty("line.separator"); 
try { 
InputStream in = method.getResponseBodyAsStream(); 
br = new BufferedReader(new InputStreamReader(in, "UTF-8")); 
StringBuffer sb = new StringBuffer(); 
String line; 
while ((line = br.readLine()) != null) { 
sb.append(line).append(lsr); 
} 
return sb.toString(); 
} finally { 
if (br != null) { 
br.close(); 
} 
} 
} 

/////This is the method that will HttpGet to the url
public String get(String url) throws IOException { 
GetMethod pm = new GetMethod(url); 
pm.setRequestHeader("Connection", "close"); 
HttpClient client = newHttpClient(); 
try { 
client.executeMethod(pm); 
String response = responseBodyAsString(pm); 
return response; 
} finally { 
pm.releaseConnection(); 
} 
} 
}

##########################################Test Case##########################################
   
The following code runs in the one thread in the junit test
        while (i++ < 5000) { 
long start = System.currentTimeMillis(); 
HttpUtils.httpInvoke(TEST_URL, null, null);
long a = System.currentTimeMillis() - start; 
String timeSpent = "" + (a >= 1000 ? a + "(>1000)" : a); 
writeToFile(timeSpent + "\n", logFile); 
Thread.sleep(loopInterval); 
}





bit1129@163.com
Mime
  • Unnamed multipart/alternative (inline, None, 0 bytes)
View raw message