hc-httpclient-users mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Todd <bit1...@163.com>
Subject Re:Re: httpclient4 is extremely slow than correpsoning code of HttpClient3
Date Tue, 07 Jul 2015 22:25:38 GMT
Hi,
Can someone help review the code? Thanks very much!




bit1129@163.com" <bit1129@163.com> wrote:
>


>Can someone kindly help me on this? Thanks a lot!
>
>
>
>
>bit1129@163.com
> 
>From: bit1129@163.com
>Date: 2015-07-07 20:51
>To: httpclient-users
>Subject: httpclient4 is extremely slow than correpsoning code of HttpClient3
>
>I have following http client code(HttpUtils and HttpClientManager)with HttpClient 4.3.6,
and a httpclient code in 3.1, and a test case as well. I observed that HttpClient 4.3.6 is
very much slowly than HttpClient 3.1 code. In HttpClient 4.3.6 version code, there are a lot
>socket time out error(the socket time is 10 seconds).
>
>I pasted the related classes and code below. It is kind of long code, but I still would
ask you do me a favor to review the http client configuration that may cause the problem.
Many Thanks in advance.
>
>Following codes include:
>1. HttpClient 4.3.6 code to issue HttpGet request
>2. HttpClient 3.1 code to issue HttpGet request
>3. Test Case that demontrate the problem.
>
>
>############################################Http Client 4.3.6 Code goes here############################################
>///////////////////////////HttpUtils//////////////////////////////////////////
>import java.util.ArrayList; 
>import java.util.List; 
>import java.util.Map; 
>
>import org.apache.http.Consts; 
>import org.apache.http.NameValuePair; 
>import org.apache.http.client.config.RequestConfig; 
>import org.apache.http.client.entity.UrlEncodedFormEntity; 
>import org.apache.http.client.methods.HttpGet; 
>import org.apache.http.client.methods.HttpPost; 
>import org.apache.http.message.BasicNameValuePair; 
>import org.slf4j.Logger; 
>import org.slf4j.LoggerFactory; 
>
>
>
>public class HttpUtils { 
>
>///This is the util method that will be used to issue http request.
>public static String httpInvoke(String httpUrl, Map<String, Object> parameters,
RequestConfig config){ 
>    HttpClientManager httpClientConnectionManager = HttpClientManager.getHttpClientConnectionManagerInstance();

>    HttpGet httpGet = new HttpGet(httpUrl); 
>    if(config != null ){ 
>           httpGet.setConfig(config); 
>} 
>String result = httpClientConnectionManager.execute(httpGet); 
>return result; 
>} 
>}
>
>//////////////////////////////////HttpClientManager/////////////////////////////////
>
>import java.nio.charset.CodingErrorAction; 
>import java.util.ArrayList; 
>import java.util.Collection; 
>
>import org.apache.commons.httpclient.Header; 
>import org.apache.commons.httpclient.HttpStatus; 
>import org.apache.commons.lang3.StringUtils; 
>import org.apache.http.Consts; 
>import org.apache.http.HttpEntity; 
>import org.apache.http.HttpHost; 
>import org.apache.http.client.config.RequestConfig; 
>import org.apache.http.client.entity.GzipDecompressingEntity; 
>import org.apache.http.client.methods.CloseableHttpResponse; 
>import org.apache.http.client.methods.HttpGet; 
>import org.apache.http.client.methods.HttpPost; 
>import org.apache.http.config.ConnectionConfig; 
>import org.apache.http.config.MessageConstraints; 
>import org.apache.http.config.SocketConfig; 
>import org.apache.http.impl.client.CloseableHttpClient; 
>import org.apache.http.impl.client.HttpClientBuilder; 
>import org.apache.http.impl.client.HttpClients; 
>import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; 
>import org.apache.http.message.BasicHeader; 
>import org.apache.http.params.CoreProtocolPNames; 
>import org.apache.http.util.EntityUtils; 
>import org.slf4j.Logger; 
>import org.slf4j.LoggerFactory; 
>
>
>///Main class that encapsulate the Http Client
>public class HttpClientManager { 
>private static Logger logger = LoggerFactory.getLogger(HttpClientManager.class); 
>
>private static int defaultConnectionTimeout = 10*1000;  //connection timeout
>private static int defaultSocketTimeout = 10*1000;  //socket time out
>private static int connectionRequestTimeout = 10*1000;  //connection request timeout
>
>private static int defaultMaxRouteConnections = 128; 
>private static int defaultMaxTotalConnections = 1024; 
>
>private static int defaultMaxHeaderCount = 200; 
>private static int defaultMaxLineLength = 2000; 
>
>private static String Charset = "utf-8"; 
>
>private String proxyHost = null; 
>private String proxyPort =null; 
>
>private PoolingHttpClientConnectionManager cm; 
>
>private final static HttpClientManager httpClientConnectionManager = new HttpClientManager();

>
>private HttpClientManager() { 
>logger.info("HttpClientManager initial!"); 
>cm = new PoolingHttpClientConnectionManager(); 
>cm.setMaxTotal(defaultMaxTotalConnections); 
>cm.setDefaultMaxPerRoute(defaultMaxRouteConnections); 
>} 
>
>
>private CloseableHttpClient getHttpclient(){ 
>HttpClientBuilder httpClientBuilder = HttpClients.custom(); 
>httpClientBuilder.setConnectionManager(cm); 
>
>RequestConfig.Builder requestConfigBuilder = RequestConfig.custom() 
>.setConnectTimeout(defaultConnectionTimeout) 
>.setSocketTimeout(defaultSocketTimeout) 
>.setConnectionRequestTimeout(connectionRequestTimeout) 
>.setExpectContinueEnabled(false) 
>.setStaleConnectionCheckEnabled(true); 
>
>if(StringUtils.isNotBlank(proxyHost) && StringUtils.isNotBlank(proxyPort)){ 
>try{ 
>logger.info("using proxy, proxyHost:{}, proxyPort:{}", proxyHost, proxyPort); 
>int proxyPortInt = Integer.parseInt(proxyPort); 
>requestConfigBuilder.setProxy(new HttpHost(proxyHost, proxyPortInt)); 
>} catch(Exception e){ 
>logger.error("parseInt proxyPort:{}", proxyPort, e); 
>} 
>} 
>
>SocketConfig socketConfig = SocketConfig.custom().setTcpNoDelay(true).build(); 
>
>MessageConstraints messageConstraints = MessageConstraints.custom().setMaxHeaderCount(defaultMaxHeaderCount).setMaxLineLength(defaultMaxLineLength).build();

>
>ConnectionConfig connectionConfig = ConnectionConfig.custom() 
>.setMalformedInputAction(CodingErrorAction.IGNORE) 
>.setUnmappableInputAction(CodingErrorAction.IGNORE) 
>.setCharset(Consts.UTF_8) 
>.setMessageConstraints(messageConstraints).build(); 
>
>Collection<BasicHeader> collection = new ArrayList<BasicHeader>(); 
>collection.add(new BasicHeader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1;
zh-CN; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3")); 
>collection.add(new BasicHeader("Accept-Language", "zh-cn,zh,en-US,en;q=0.5")); 
>collection.add(new BasicHeader("Accept-Charset", Charset)); 
>collection.add(new BasicHeader("Accept-Encoding", "gzip")); 
>
>httpClientBuilder.setDefaultRequestConfig(requestConfigBuilder.build()); 
>httpClientBuilder.setDefaultSocketConfig(socketConfig); 
>httpClientBuilder.setDefaultConnectionConfig(connectionConfig); 
>httpClientBuilder.setDefaultHeaders(collection); 
>
>return httpClientBuilder.build(); 
>} 
>
>///This is the method that will be call the execute the HttpGet request
>public String execute(HttpGet httpGet) { 
>String body = ""; 
>try{ 
>CloseableHttpClient httpclient = this.getHttpclient(); 
>CloseableHttpResponse response = httpclient.execute(httpGet); 
>int status = response.getStatusLine().getStatusCode(); 
>try { 
>if (status == HttpStatus.SC_OK) { 
>HttpEntity entity = response.getEntity(); 
>if (entity != null) { 
>Header header = (Header) entity.getContentEncoding(); 
>if(header != null && "gzip".equals(header.getValue())){ 
>body = EntityUtils.toString(new GzipDecompressingEntity(entity), Charset); 
>} else { 
>body = EntityUtils.toString(entity, Charset); 
>} 
>} 
>} else { 
>logger.error("[httpClientManager] [fail] [httpGet:{}] [status:{}]", httpGet, status);

>} 
>} finally { 
>response.close(); 
>} 
>} catch(Exception e) { 
>logger.error("[module:httpClientManager] [action:execute] [httpGet:{}] [error:{}] ", httpGet,
e.getMessage(), e); 
>} 
>return body; 
>} 
>
>
>////Singleton object that will be used to access httpClientConnectionManager
>public static HttpClientManager getHttpClientConnectionManagerInstance(){ 
>return httpClientConnectionManager; 
>} 
>
>} 
>
>
>##################################################My HttpClient 1.3 code goes here###########################################
>
>import org.apache.commons.httpclient.HttpClient; 
>import org.apache.commons.httpclient.HttpMethod; 
>import org.apache.commons.httpclient.methods.GetMethod; 
>
>import java.io.BufferedReader; 
>import java.io.IOException; 
>import java.io.InputStream; 
>import java.io.InputStreamReader; 
>
>public class HttpClientUtil { 
>
>private static final HttpClientUtil INSTANCE = new HttpClientUtil(); 
>
>private static final int TIMEOUT = 10 * 1000; //10s 
>
>private HttpClientUtil() { 
>} 
>
>///Actually, this class is more like a static class than Singleton 
>public static HttpClientUtil getInstance() { 
>return INSTANCE; 
>} 
>
>//brand new http client per request 
>private HttpClient newHttpClient() { 
>HttpClient client = new HttpClient(); 
>client.getHttpConnectionManager().getParams().setConnectionTimeout(TIMEOUT); 
>client.getHttpConnectionManager().getParams().setSoTimeout(TIMEOUT); 
>return client; 
>} 
>
>//FIXME The encoding should be provided when convert the response binary stream into string

>public static String responseBodyAsString(HttpMethod method) throws IOException { 
>BufferedReader br = null; 
>String lsr = System.getProperty("line.separator"); 
>try { 
>InputStream in = method.getResponseBodyAsStream(); 
>br = new BufferedReader(new InputStreamReader(in, "UTF-8")); 
>StringBuffer sb = new StringBuffer(); 
>String line; 
>while ((line = br.readLine()) != null) { 
>sb.append(line).append(lsr); 
>} 
>return sb.toString(); 
>} finally { 
>if (br != null) { 
>br.close(); 
>} 
>} 
>} 
>
>/////This is the method that will HttpGet to the url
>public String get(String url) throws IOException { 
>GetMethod pm = new GetMethod(url); 
>pm.setRequestHeader("Connection", "close"); 
>HttpClient client = newHttpClient(); 
>try { 
>client.executeMethod(pm); 
>String response = responseBodyAsString(pm); 
>return response; 
>} finally { 
>pm.releaseConnection(); 
>} 
>} 
>}
>
>##########################################Test Case##########################################
   
>The following code runs in the one thread in the junit test
>        while (i++ < 5000) { 
>long start = System.currentTimeMillis(); 
>HttpUtils.httpInvoke(TEST_URL, null, null);
>long a = System.currentTimeMillis() - start; 
>String timeSpent = "" + (a >= 1000 ? a + "(>1000)" : a); 
>writeToFile(timeSpent + "\n", logFile); 
>Thread.sleep(loopInterval); 
>}
>
>
>
>
>
>bit1129@163.com
Mime
  • Unnamed multipart/alternative (inline, None, 0 bytes)
View raw message