hc-httpclient-users mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zxt...@gmail.com
Subject Performance problem with multiple requests to the same server simultaneously
Date Mon, 05 Apr 2010 04:20:58 GMT
Used multiple threads to connect to the same server (httpcomponents-client 
4.0.1). Each thread represents an account on the server and has different 
username/password combination, so different cookies. Normally 1000 threads are 
used. The problem is, even though I configured the ThreadSafeClientConnManager 
to use 2000 connections, it seems that it only used 2 to 3 connections at the 
same time (I used fiddler to check it). 1000 requests (one request per thread) 
takes about 500 seconds to complete. Unacceptable performance. Most probable 
reason is incorrect configuration. I'm new to network programming and 
httpcomponents, and I can not figure out and solve the problem.  Your help is 
much appreciated. 

The following is a class I use to communicate with the server. Each thread has 
an instance of this class.

public class ServerCommunicator {

	ServerCommunicator(String baseUrl) {
		if (!httpClientInitialized)
			initializeHttpClient();
		this.baseUrl = baseUrl;
		this.context = new BasicHttpContext();
		this.cookieStore = new BasicCookieStore();
		this.context.setAttribute(ClientContext.COOKIE_STORE, this.cookieStore);
	}

	public void setBaseUrl(String baseUrl) {
		this.baseUrl = baseUrl;
	}

	public void readCookies(String cookieFile) throws NumberFormatException,
			IOException {
		String name = null;
		String value = null;
		String comment = null;
		// String commentUrl = null;
		Date expiryDate = null;
		// boolean isPersistent = false;
		String domain = null;
		String path = null;
		int[] ports = null;
		boolean isSecure = false;
		int version = 0;

		BufferedReader reader = new BufferedReader(new FileReader(cookieFile));
		String line;
		while ((line = reader.readLine()) != null) {
			String[] cols = line.split(",");
			name = cols[0].length() == 0 ? null : cols[0];
			value = cols[1].length() == 0 ? null : cols[1];
			comment = cols[2].length() == 0 ? null : cols[2];
			// commentUrl = cols[3].length() == 0 ? null : cols[3];
			expiryDate = cols[4].length() == 0 ? null : new Date(Long
					.parseLong(cols[4]));
			// isPersistent = Boolean.parseBoolean(cols[5]);
			domain = cols[6].length() == 0 ? null : cols[6];
			path = cols[7].length() == 0 ? null : cols[7];
			ports = null;
			if (cols[8].length() != 0) {
				String[] strPorts = cols[8].split(" ");
				ports = new int[strPorts.length];
				for (int i = 0; i < strPorts.length; i++)
					ports[i] = Integer.parseInt(strPorts[i]);
			}
			isSecure = Boolean.parseBoolean(cols[9]);
			version = Integer.parseInt(cols[10]);

			BasicClientCookie cookie = new BasicClientCookie(name, value);
			cookie.setComment(comment);
			cookie.setDomain(domain);
			cookie.setExpiryDate(expiryDate);
			cookie.setPath(path);
			cookie.setSecure(isSecure);
			cookie.setVersion(version);

			this.cookieStore.addCookie(cookie);
		}

		reader.close();
	}

	public void writeCookies(String cookieFile) throws IOException {
		int[] ports = null;

		BufferedWriter writer = new BufferedWriter(new FileWriter(cookieFile));
		StringBuilder line = new StringBuilder(256);

		List<Cookie> cookies = this.cookieStore.getCookies();
		for (Cookie cookie : cookies) {
			line.append(cookie.getName());
			line.append(',');
			if (cookie.getValue() != null)
				line.append(cookie.getValue());
			line.append(',');
			if (cookie.getComment() != null)
				line.append(cookie.getComment());
			line.append(',');
			if (cookie.getCommentURL() != null)
				line.append(cookie.getCommentURL());
			line.append(',');
			if (cookie.getExpiryDate() != null)
				line.append(cookie.getExpiryDate().getTime());
			line.append(',');
			line.append(cookie.isPersistent());
			line.append(',');
			if (cookie.getDomain() != null)
				line.append(cookie.getDomain());
			line.append(',');
			if (cookie.getPath() != null)
				line.append(cookie.getPath());
			line.append(',');
			ports = cookie.getPorts();
			if (ports != null) {
				line.append(ports[0]);
				for (int i = 1; i < ports.length; i++) {
					line.append(' ');
					line.append(ports[i]);
				}
			}
			line.append(',');
			line.append(cookie.isSecure());
			line.append(',');
			line.append(cookie.getVersion());

			writer.write(line.toString());
			writer.newLine();
			line.delete(0, line.length());
		}

		writer.close();
	}

	public synchronized byte[] getData(String path)
			throws ClientProtocolException, IOException {
		String url = getUrl(path);
		HttpGet getReq = new HttpGet(url);
		HttpResponse resp = httpClient.execute(getReq, this.context);
		statusCode = resp.getStatusLine().getStatusCode();
		HttpEntity entity = resp.getEntity();
		if (entity != null) {
			return EntityUtils.toByteArray(entity);
		} else {
			return null;
		}
	}

	public synchronized String get(String path) throws ClientProtocolException,
			IOException {
		String url = getUrl(path);
		HttpGet getReq = new HttpGet(url);
		HttpResponse resp = httpClient.execute(getReq, this.context);
		statusCode = resp.getStatusLine().getStatusCode();
		HttpEntity entity = resp.getEntity();
		if (entity != null) {
			return EntityUtils.toString(entity);
		} else {
			return null;
		}
	}

	public synchronized String post(String path, String content)
			throws ClientProtocolException, IOException {
		String url = getUrl(path);
		HttpPost postReq = new HttpPost(url);
		StringEntity postEnt = new StringEntity(content);
		postEnt.setContentType("application/x-www-form-urlencoded");
		postReq.setEntity(postEnt);
		HttpResponse resp = httpClient.execute(postReq, this.context);
		statusCode = resp.getStatusLine().getStatusCode();
		HttpEntity entity = resp.getEntity();
		if (entity != null) {
			return EntityUtils.toString(entity);
		} else {
			return null;
		}
	}

	public synchronized String post(String path, List<NameValuePair> params)
			throws ClientProtocolException, IOException {
		String url = getUrl(path);
		HttpPost postReq = new HttpPost(url);
		UrlEncodedFormEntity postEnt = new UrlEncodedFormEntity(params,
				HTTP.UTF_8);
		postEnt.setContentType("application/x-www-form-urlencoded");
		postReq.setEntity(postEnt);
		HttpResponse resp = httpClient.execute(postReq, this.context);
		statusCode = resp.getStatusLine().getStatusCode();
		HttpEntity entity = resp.getEntity();
		if (entity != null) {
			return EntityUtils.toString(entity);
		} else {
			return null;
		}
	}

	public static void main(String[] args) {
	}

	private String getUrl(String path) {
		return path.startsWith("http://") ? path : baseUrl + path;
	}

	private int statusCode = 0;
	public int getStatusCode() {
		return statusCode;
	}
	
	private String baseUrl = null;
	private HttpContext context = null;
	private CookieStore cookieStore = null;
	private static HttpClient httpClient = null;
	private static String proxyHost = null;
	private static int proxyPort = 80;
	private static boolean httpClientInitialized = false;

	public static void setProxy(String host, int port) {
		proxyHost = host;
		proxyPort = port;
	}
	
	private static void initializeHttpClient() {
		if (!httpClientInitialized) {
			// Create and initialize HTTP parameters
			HttpParams params = new BasicHttpParams();
			ConnManagerParams.setMaxTotalConnections(params, 2000);
			// ConnManagerParams.setTimeout(params, 10000); // in millisecs
			HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1);

			// Create and initialize scheme registry
			SchemeRegistry schemeRegistry = new SchemeRegistry();
			schemeRegistry.register(new Scheme("http", PlainSocketFactory
					.getSocketFactory(), 80));

			ClientConnectionManager cm = new ThreadSafeClientConnManager(
					params, schemeRegistry);

			httpClient = new DefaultHttpClient(cm, params);

			if (proxyHost != null) {
				httpClient.getParams().setParameter(
						ConnRoutePNames.DEFAULT_PROXY,
						new HttpHost(proxyHost, proxyPort));
			}
			httpClient.getParams().setParameter("http.useragent",
					"Mozilla/4.0 (compatible; MSIE 6.1; Windows XP)");
			httpClient.getParams().setParameter("http.protocol.allow-circular-
redirects", true);
			httpClientInitialized = true;
		}
	}
	
	public static void shutdown() {
		httpClient.getConnectionManager().shutdown();
	}
}


---------------------------------------------------------------------
To unsubscribe, e-mail: httpclient-users-unsubscribe@hc.apache.org
For additional commands, e-mail: httpclient-users-help@hc.apache.org


Mime
View raw message