hc-httpclient-users mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Stefan Magnus Landrø <stefan.lan...@gmail.com>
Subject Re: httpclient4 is extremely slow than correpsoning code of HttpClient3
Date Tue, 07 Jul 2015 16:33:46 GMT
>> try{ 
>> CloseableHttpClient httpclient = 

Try with resources will close client

Sendt fra min iPhone

> Den 7. jul. 2015 kl. 16.59 skrev Todd <bit1129@163.com>:
> 
> Thanks Stefan for the reply. Could you please point to me which code you are referring
to?
> 
> 
> 
> At 2015-07-07 22:49:28, "Stefan Magnus Landrø" <stefan.landro@gmail.com> wrote:
>> You shouldn't close http client on every request, only response
>> 
>> Sendt fra min iPhone
>> 
>>> Den 7. jul. 2015 kl. 16.05 skrev "bit1129@163.com" <bit1129@163.com>:
>>> 
>>> 
>>> Can someone kindly help me on this? Thanks a lot!
>>> 
>>> 
>>> 
>>> 
>>> bit1129@163.com
>>> 
>>> From: bit1129@163.com
>>> Date: 2015-07-07 20:51
>>> To: httpclient-users
>>> Subject: httpclient4 is extremely slow than correpsoning code of HttpClient3
>>> 
>>> I have following http client code(HttpUtils and HttpClientManager)with HttpClient
4.3.6, and a httpclient code in 3.1, and a test case as well. I observed that HttpClient 4.3.6
is very much slowly than HttpClient 3.1 code. In HttpClient 4.3.6 version code, there are
a lot
>>> socket time out error(the socket time is 10 seconds).
>>> 
>>> I pasted the related classes and code below. It is kind of long code, but I still
would ask you do me a favor to review the http client configuration that may cause the problem.
Many Thanks in advance.
>>> 
>>> Following codes include:
>>> 1. HttpClient 4.3.6 code to issue HttpGet request
>>> 2. HttpClient 3.1 code to issue HttpGet request
>>> 3. Test Case that demontrate the problem.
>>> 
>>> 
>>> ############################################Http Client 4.3.6 Code goes here############################################
>>> ///////////////////////////HttpUtils//////////////////////////////////////////
>>> import java.util.ArrayList; 
>>> import java.util.List; 
>>> import java.util.Map; 
>>> 
>>> import org.apache.http.Consts; 
>>> import org.apache.http.NameValuePair; 
>>> import org.apache.http.client.config.RequestConfig; 
>>> import org.apache.http.client.entity.UrlEncodedFormEntity; 
>>> import org.apache.http.client.methods.HttpGet; 
>>> import org.apache.http.client.methods.HttpPost; 
>>> import org.apache.http.message.BasicNameValuePair; 
>>> import org.slf4j.Logger; 
>>> import org.slf4j.LoggerFactory; 
>>> 
>>> 
>>> 
>>> public class HttpUtils { 
>>> 
>>> ///This is the util method that will be used to issue http request.
>>> public static String httpInvoke(String httpUrl, Map<String, Object> parameters,
RequestConfig config){ 
>>>   HttpClientManager httpClientConnectionManager = HttpClientManager.getHttpClientConnectionManagerInstance();

>>>   HttpGet httpGet = new HttpGet(httpUrl); 
>>>   if(config != null ){ 
>>>          httpGet.setConfig(config); 
>>> } 
>>> String result = httpClientConnectionManager.execute(httpGet); 
>>> return result; 
>>> } 
>>> }
>>> 
>>> //////////////////////////////////HttpClientManager/////////////////////////////////
>>> 
>>> import java.nio.charset.CodingErrorAction; 
>>> import java.util.ArrayList; 
>>> import java.util.Collection; 
>>> 
>>> import org.apache.commons.httpclient.Header;
>>> import org.apache.commons.httpclient.HttpStatus; 
>>> import org.apache.commons.lang3.StringUtils; 
>>> import org.apache.http.Consts; 
>>> import org.apache.http.HttpEntity; 
>>> import org.apache.http.HttpHost; 
>>> import org.apache.http.client.config.RequestConfig; 
>>> import org.apache.http.client.entity.GzipDecompressingEntity; 
>>> import org.apache.http.client.methods.CloseableHttpResponse; 
>>> import org.apache.http.client.methods.HttpGet; 
>>> import org.apache.http.client.methods.HttpPost; 
>>> import org.apache.http.config.ConnectionConfig; 
>>> import org.apache.http.config.MessageConstraints; 
>>> import org.apache.http.config.SocketConfig; 
>>> import org.apache.http.impl.client.CloseableHttpClient; 
>>> import org.apache.http.impl.client.HttpClientBuilder; 
>>> import org.apache.http.impl.client.HttpClients; 
>>> import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; 
>>> import org.apache.http.message.BasicHeader; 
>>> import org.apache.http.params.CoreProtocolPNames; 
>>> import org.apache.http.util.EntityUtils; 
>>> import org.slf4j.Logger; 
>>> import org.slf4j.LoggerFactory; 
>>> 
>>> 
>>> ///Main class that encapsulate the Http Client
>>> public class HttpClientManager { 
>>> private static Logger logger = LoggerFactory.getLogger(HttpClientManager.class);

>>> 
>>> private static int defaultConnectionTimeout = 10*1000;  //connection timeout
>>> private static int defaultSocketTimeout = 10*1000;  //socket time out
>>> private static int connectionRequestTimeout = 10*1000;  //connection request
timeout
>>> 
>>> private static int defaultMaxRouteConnections = 128; 
>>> private static int defaultMaxTotalConnections = 1024; 
>>> 
>>> private static int defaultMaxHeaderCount = 200; 
>>> private static int defaultMaxLineLength = 2000; 
>>> 
>>> private static String Charset = "utf-8"; 
>>> 
>>> private String proxyHost = null; 
>>> private String proxyPort =null; 
>>> 
>>> private PoolingHttpClientConnectionManager cm; 
>>> 
>>> private final static HttpClientManager httpClientConnectionManager = new HttpClientManager();

>>> 
>>> private HttpClientManager() { 
>>> logger.info("HttpClientManager initial!"); 
>>> cm = new PoolingHttpClientConnectionManager(); 
>>> cm.setMaxTotal(defaultMaxTotalConnections); 
>>> cm.setDefaultMaxPerRoute(defaultMaxRouteConnections); 
>>> } 
>>> 
>>> 
>>> private CloseableHttpClient getHttpclient(){ 
>>> HttpClientBuilder httpClientBuilder = HttpClients.custom(); 
>>> httpClientBuilder.setConnectionManager(cm); 
>>> 
>>> RequestConfig.Builder requestConfigBuilder = RequestConfig.custom() 
>>> .setConnectTimeout(defaultConnectionTimeout) 
>>> .setSocketTimeout(defaultSocketTimeout) 
>>> .setConnectionRequestTimeout(connectionRequestTimeout) 
>>> .setExpectContinueEnabled(false) 
>>> .setStaleConnectionCheckEnabled(true); 
>>> 
>>> if(StringUtils.isNotBlank(proxyHost) && StringUtils.isNotBlank(proxyPort)){

>>> try{ 
>>> logger.info("using proxy, proxyHost:{}, proxyPort:{}", proxyHost, proxyPort);

>>> int proxyPortInt = Integer.parseInt(proxyPort); 
>>> requestConfigBuilder.setProxy(new HttpHost(proxyHost, proxyPortInt)); 
>>> } catch(Exception e){ 
>>> logger.error("parseInt proxyPort:{}", proxyPort, e); 
>>> } 
>>> } 
>>> 
>>> SocketConfig socketConfig = SocketConfig.custom().setTcpNoDelay(true).build();

>>> 
>>> MessageConstraints messageConstraints = MessageConstraints.custom().setMaxHeaderCount(defaultMaxHeaderCount).setMaxLineLength(defaultMaxLineLength).build();

>>> 
>>> ConnectionConfig connectionConfig = ConnectionConfig.custom() 
>>> .setMalformedInputAction(CodingErrorAction.IGNORE) 
>>> .setUnmappableInputAction(CodingErrorAction.IGNORE) 
>>> .setCharset(Consts.UTF_8) 
>>> .setMessageConstraints(messageConstraints).build(); 
>>> 
>>> Collection<BasicHeader> collection = new ArrayList<BasicHeader>();

>>> collection.add(new BasicHeader("User-Agent", "Mozilla/5.0 (Windows; U; Windows
NT 5.1; zh-CN; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3")); 
>>> collection.add(new BasicHeader("Accept-Language", "zh-cn,zh,en-US,en;q=0.5"));

>>> collection.add(new BasicHeader("Accept-Charset", Charset)); 
>>> collection.add(new BasicHeader("Accept-Encoding", "gzip")); 
>>> 
>>> httpClientBuilder.setDefaultRequestConfig(requestConfigBuilder.build()); 
>>> httpClientBuilder.setDefaultSocketConfig(socketConfig); 
>>> httpClientBuilder.setDefaultConnectionConfig(connectionConfig); 
>>> httpClientBuilder.setDefaultHeaders(collection); 
>>> 
>>> return httpClientBuilder.build(); 
>>> } 
>>> 
>>> ///This is the method that will be call the execute the HttpGet request
>>> public String execute(HttpGet httpGet) { 
>>> String body = ""; 
>>> try{ 
>>> CloseableHttpClient httpclient = this.getHttpclient(); 
>>> CloseableHttpResponse response = httpclient.execute(httpGet); 
>>> int status = response.getStatusLine().getStatusCode(); 
>>> try { 
>>> if (status == HttpStatus.SC_OK) { 
>>> HttpEntity entity = response.getEntity(); 
>>> if (entity != null) { 
>>> Header header = (Header) entity.getContentEncoding(); 
>>> if(header != null && "gzip".equals(header.getValue())){ 
>>> body = EntityUtils.toString(new GzipDecompressingEntity(entity), Charset); 
>>> } else { 
>>> body = EntityUtils.toString(entity, Charset); 
>>> } 
>>> } 
>>> } else { 
>>> logger.error("[httpClientManager] [fail] [httpGet:{}] [status:{}]", httpGet,
status); 
>>> } 
>>> } finally { 
>>> response.close(); 
>>> } 
>>> } catch(Exception e) { 
>>> logger.error("[module:httpClientManager] [action:execute] [httpGet:{}] [error:{}]
", httpGet, e.getMessage(), e); 
>>> } 
>>> return body; 
>>> } 
>>> 
>>> 
>>> ////Singleton object that will be used to access httpClientConnectionManager
>>> public static HttpClientManager getHttpClientConnectionManagerInstance(){ 
>>> return httpClientConnectionManager; 
>>> } 
>>> 
>>> } 
>>> 
>>> 
>>> ##################################################My HttpClient 1.3 code goes
here###########################################
>>> 
>>> import org.apache.commons.httpclient.HttpClient;
>>> import org.apache.commons.httpclient.HttpMethod; 
>>> import org.apache.commons.httpclient.methods.GetMethod; 
>>> 
>>> import java.io.BufferedReader; 
>>> import java.io.IOException; 
>>> import java.io.InputStream; 
>>> import java.io.InputStreamReader; 
>>> 
>>> public class HttpClientUtil { 
>>> 
>>> private static final HttpClientUtil INSTANCE = new HttpClientUtil(); 
>>> 
>>> private static final int TIMEOUT = 10 * 1000; //10s 
>>> 
>>> private HttpClientUtil() { 
>>> } 
>>> 
>>> ///Actually, this class is more like a static class than Singleton 
>>> public static HttpClientUtil getInstance() { 
>>> return INSTANCE; 
>>> } 
>>> 
>>> //brand new http client per request 
>>> private HttpClient newHttpClient() { 
>>> HttpClient client = new HttpClient(); 
>>> client.getHttpConnectionManager().getParams().setConnectionTimeout(TIMEOUT);

>>> client.getHttpConnectionManager().getParams().setSoTimeout(TIMEOUT); 
>>> return client; 
>>> } 
>>> 
>>> //FIXME The encoding should be provided when convert the response binary stream
into string 
>>> public static String responseBodyAsString(HttpMethod method) throws IOException
{ 
>>> BufferedReader br = null; 
>>> String lsr = System.getProperty("line.separator"); 
>>> try { 
>>> InputStream in = method.getResponseBodyAsStream(); 
>>> br = new BufferedReader(new InputStreamReader(in, "UTF-8")); 
>>> StringBuffer sb = new StringBuffer(); 
>>> String line; 
>>> while ((line = br.readLine()) != null) { 
>>> sb.append(line).append(lsr); 
>>> } 
>>> return sb.toString(); 
>>> } finally { 
>>> if (br != null) { 
>>> br.close(); 
>>> } 
>>> } 
>>> } 
>>> 
>>> /////This is the method that will HttpGet to the url
>>> public String get(String url) throws IOException { 
>>> GetMethod pm = new GetMethod(url); 
>>> pm.setRequestHeader("Connection", "close"); 
>>> HttpClient client = newHttpClient(); 
>>> try { 
>>> client.executeMethod(pm); 
>>> String response = responseBodyAsString(pm); 
>>> return response; 
>>> } finally { 
>>> pm.releaseConnection(); 
>>> } 
>>> } 
>>> }
>>> 
>>> ##########################################Test Case##########################################
   
>>> The following code runs in the one thread in the junit test
>>>       while (i++ < 5000) { 
>>> long start = System.currentTimeMillis(); 
>>> HttpUtils.httpInvoke(TEST_URL, null, null);
>>> long a = System.currentTimeMillis() - start; 
>>> String timeSpent = "" + (a >= 1000 ? a + "(>1000)" : a); 
>>> writeToFile(timeSpent + "\n", logFile); 
>>> Thread.sleep(loopInterval); 
>>> }
>>> 
>>> 
>>> 
>>> 
>>> 
>>> bit1129@163.com
>> 
>> ---------------------------------------------------------------------
>> To unsubscribe, e-mail: httpclient-users-unsubscribe@hc.apache.org
>> For additional commands, e-mail: httpclient-users-help@hc.apache.org
>> 

Mime
  • Unnamed multipart/alternative (inline, 7-Bit, 0 bytes)
View raw message