Return-Path:
Delivered-To: apmail-jakarta-commons-httpclient-dev-archive@www.apache.org
Received: (qmail 53360 invoked from network); 11 Oct 2003 13:34:46 -0000
Received: from daedalus.apache.org (HELO mail.apache.org) (208.185.179.12)
by minotaur-2.apache.org with SMTP; 11 Oct 2003 13:34:46 -0000
Received: (qmail 42071 invoked by uid 500); 11 Oct 2003 13:34:40 -0000
Delivered-To: apmail-jakarta-commons-httpclient-dev-archive@jakarta.apache.org
Received: (qmail 42057 invoked by uid 500); 11 Oct 2003 13:34:40 -0000
Mailing-List: contact commons-httpclient-dev-help@jakarta.apache.org;
run by ezmlm
Precedence: bulk
List-Unsubscribe:
List-Subscribe:
List-Help:
List-Post:
List-Id: "Commons HttpClient Project"
Reply-To: "Commons HttpClient Project"
Delivered-To: mailing list commons-httpclient-dev@jakarta.apache.org
Received: (qmail 41993 invoked from network); 11 Oct 2003 13:34:39 -0000
Received: from unknown (HELO mail.newsclub.de) (213.167.167.144)
by daedalus.apache.org with SMTP; 11 Oct 2003 13:34:39 -0000
Received: (qmail 17482 invoked from network); 11 Oct 2003 13:41:21 -0000
Received: from unknown (HELO blue.localnet) (nobody@127.0.0.1)
by localhost with SMTP; 11 Oct 2003 13:41:21 -0000
From: Christian Kohlschuetter
To: "Commons HttpClient Project"
Subject: Re: Proposal: Configurable HTTP Response length limit
Date: Sat, 11 Oct 2003 15:34:50 +0200
User-Agent: KMail/1.5.3
References:
<825BF35A92B3F0479CC164ECBBE9376E0D49BB@kccxoex06.corp.kpmgconsulting.com>
<3F86E4B9.2080203@tibco.com> <200310111214.14038.ck@rrzn.uni-hannover.de>
In-Reply-To: <200310111214.14038.ck@rrzn.uni-hannover.de>
MIME-Version: 1.0
Content-Type: Multipart/Mixed;
boundary="Boundary-00=_6bAi/9mDMSRpZL2"
Message-Id: <200310111534.50704.ck@rrzn.uni-hannover.de>
X-Spam-Rating: daedalus.apache.org 1.6.2 0/1000/N
X-Spam-Rating: minotaur-2.apache.org 1.6.2 0/1000/N
--Boundary-00=_6bAi/9mDMSRpZL2
Content-Type: text/plain;
charset="iso-8859-1"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline
Here is the new patch (attached to this mail).
--Boundary-00=_6bAi/9mDMSRpZL2
Content-Type: text/x-diff;
charset="iso-8859-1";
name="maxBytes.patch"
Content-Transfer-Encoding: 8bit
Content-Disposition: attachment;
filename="maxBytes.patch"
Index: org/apache/commons/httpclient/AutoCloseInputStream.java
===================================================================
RCS file: /home/cvspublic/jakarta-commons/httpclient/src/java/org/apache/commons/httpclient/AutoCloseInputStream.java,v
retrieving revision 1.7
diff -u -r1.7 AutoCloseInputStream.java
--- org/apache/commons/httpclient/AutoCloseInputStream.java 28 Jan 2003 04:40:20 -0000 1.7
+++ org/apache/commons/httpclient/AutoCloseInputStream.java 11 Oct 2003 13:24:41 -0000
@@ -71,7 +71,7 @@
* Closes an underlying stream as soon as the end of the stream is reached, and
* notifies a client when it has done so.
*
- * @author Ortwin Gl�ck
+ * @author Ortwin Glück
* @author Eric Johnson
* @author Mike Bowler
*
@@ -94,6 +94,11 @@
* been exhausted
*/
private ResponseConsumedWatcher watcher = null;
+
+ /**
+ * True if errors (IOExceptions) occured while reading
+ */
+ private boolean readErrors = false;
/**
* Create a new auto closing stream for the provided connection
@@ -119,8 +124,13 @@
if (isReadAllowed()) {
// underlying stream not closed, go ahead and read.
- l = super.read();
- checkClose(l);
+ try {
+ l = super.read();
+ checkClose(l);
+ } catch(IOException e) {
+ readErrors = true;
+ throw e;
+ }
}
return l;
@@ -139,8 +149,13 @@
int l = -1;
if (isReadAllowed()) {
- l = super.read(b, off, len);
- checkClose(l);
+ try {
+ l = super.read(b, off, len);
+ checkClose(l);
+ } catch(IOException e) {
+ readErrors = true;
+ throw e;
+ }
}
return l;
@@ -158,8 +173,13 @@
int l = -1;
if (isReadAllowed()) {
- l = super.read(b);
- checkClose(l);
+ try {
+ l = super.read(b);
+ checkClose(l);
+ } catch(IOException e) {
+ readErrors = true;
+ throw e;
+ }
}
return l;
}
@@ -208,11 +228,17 @@
*/
private void notifyWatcher() throws IOException {
if (streamOpen) {
- super.close();
- streamOpen = false;
-
- if (watcher != null) {
- watcher.responseConsumed();
+ try {
+ super.close();
+ } catch(IOException e) {
+ readErrors = true;
+ throw e;
+ } finally {
+ streamOpen = false;
+
+ if (watcher != null) {
+ watcher.responseConsumed(readErrors);
+ }
}
}
}
Index: org/apache/commons/httpclient/ChunkedInputStream.java
===================================================================
RCS file: /home/cvspublic/jakarta-commons/httpclient/src/java/org/apache/commons/httpclient/ChunkedInputStream.java,v
retrieving revision 1.19
diff -u -r1.19 ChunkedInputStream.java
--- org/apache/commons/httpclient/ChunkedInputStream.java 5 Aug 2003 19:29:30 -0000 1.19
+++ org/apache/commons/httpclient/ChunkedInputStream.java 11 Oct 2003 13:24:42 -0000
@@ -82,7 +82,7 @@
* not requiring the client to remember to read the entire contents of the
* response.
*
- * @author Ortwin Gl�ck
+ * @author Ortwin Glück
* @author Sean C. Sullivan
* @author Martin Elwin
* @author Eric Johnson
@@ -238,7 +238,12 @@
if (!bof) {
readCRLF();
}
- chunkSize = getChunkSizeFromInputStream(in);
+ try {
+ chunkSize = getChunkSizeFromInputStream(in);
+ } catch(IOException e) {
+ LOG.debug("Cannot get chunk size - assuming EOF reached", e);
+ chunkSize = 0;
+ }
bof = false;
pos = 0;
if (chunkSize == 0) {
Index: org/apache/commons/httpclient/HttpConnection.java
===================================================================
RCS file: /home/cvspublic/jakarta-commons/httpclient/src/java/org/apache/commons/httpclient/HttpConnection.java,v
retrieving revision 1.75
diff -u -r1.75 HttpConnection.java
--- org/apache/commons/httpclient/HttpConnection.java 10 Sep 2003 21:37:48 -0000 1.75
+++ org/apache/commons/httpclient/HttpConnection.java 11 Oct 2003 13:24:50 -0000
@@ -572,6 +572,26 @@
public void setConnectionTimeout(int timeout) {
this.connectTimeout = timeout;
}
+
+ /**
+ * Sets the maximum number of bytes readable from the HTTP Response.
+ *
+ * @param limit The bytes limit. 0 means no limit.
+ */
+ public void setResponseMaxBytes(long limit) {
+ this.responseMaxBytes = limit;
+ }
+
+ /**
+ * Resets the number of received bytes from the HTTP Response.
+ * This is useful for Connection reuse.
+ *
+ */
+ public void resetResponseByteCount() {
+ if(limitedSizeInputStream != null) {
+ limitedSizeInputStream.resetByteCounter();
+ }
+ }
/**
* Open this connection to the current host and port
@@ -637,7 +657,7 @@
if (sendBufferSize != -1) {
socket.setSendBufferSize(sendBufferSize);
}
- inputStream = new PushbackInputStream(socket.getInputStream());
+ inputStream = new PushbackInputStream(prepareResponseInputStream(socket.getInputStream()));
outputStream = new BufferedOutputStream(
new WrappedOutputStream(socket.getOutputStream()),
socket.getSendBufferSize()
@@ -690,7 +710,7 @@
if (sendBufferSize != -1) {
socket.setSendBufferSize(sendBufferSize);
}
- inputStream = new PushbackInputStream(socket.getInputStream());
+ inputStream = new PushbackInputStream(prepareResponseInputStream(socket.getInputStream()));
outputStream = new BufferedOutputStream(
new WrappedOutputStream(socket.getOutputStream()),
socket.getSendBufferSize()
@@ -1153,6 +1173,17 @@
socket.setSendBufferSize(sendBufferSize);
}
}
+
+ protected InputStream prepareResponseInputStream(InputStream in) {
+ if(this.responseMaxBytes != 0) {
+ limitedSizeInputStream = new LimitedSizeInputStream(in, this.responseMaxBytes);
+ limitedSizeInputStream.setThrowException(true);
+ in = limitedSizeInputStream;
+ } else {
+ limitedSizeInputStream = null;
+ }
+ return in;
+ }
// -- Timeout Exception
/**
@@ -1319,6 +1350,9 @@
/** My InputStream. */
private PushbackInputStream inputStream = null;
+
+ /** The proably underlying limited-size InputStream */
+ private LimitedSizeInputStream limitedSizeInputStream = null;
/** My OutputStream. */
private OutputStream outputStream = null;
@@ -1362,4 +1396,7 @@
/** The local interface on which the connection is created, or null for the default */
private InetAddress localAddress;
+
+ /** The maximum number of bytes to be read from response */
+ private long responseMaxBytes;
}
Index: org/apache/commons/httpclient/HttpMethodBase.java
===================================================================
RCS file: /home/cvspublic/jakarta-commons/httpclient/src/java/org/apache/commons/httpclient/HttpMethodBase.java,v
retrieving revision 1.181
diff -u -r1.181 HttpMethodBase.java
--- org/apache/commons/httpclient/HttpMethodBase.java 3 Oct 2003 20:57:35 -0000 1.181
+++ org/apache/commons/httpclient/HttpMethodBase.java 11 Oct 2003 13:24:56 -0000
@@ -117,7 +117,7 @@
* @author dIon Gillard
* @author Jeff Dever
* @author Davanum Srinivas
- * @author Ortwin Gl�ck
+ * @author Ortwin Glück
* @author Eric Johnson
* @author Michael Becke
* @author Oleg Kalnichevski
@@ -991,7 +991,18 @@
boolean requestSent = false;
writeRequest(state, conn);
requestSent = true;
- readResponse(state, conn);
+ try {
+ readResponse(state, conn);
+ } catch(HttpRecoverableException e) {
+ throw e;
+ } catch(HttpException e) {
+ responseConnection.close();
+ throw e;
+ } catch(IOException e) {
+ responseConnection.close();
+ throw e;
+ }
+
// the method has successfully executed
used = true;
@@ -1743,8 +1754,8 @@
result = new AutoCloseInputStream(
result,
new ResponseConsumedWatcher() {
- public void responseConsumed() {
- responseBodyConsumed();
+ public void responseConsumed(boolean withErrors) {
+ responseBodyConsumed(withErrors);
}
}
);
@@ -2222,13 +2233,17 @@
*
*/
protected void responseBodyConsumed() {
+ responseBodyConsumed(false);
+ }
+ protected void responseBodyConsumed(boolean withErrors) {
// make sure this is the initial invocation of the notification,
// ignore subsequent ones.
responseStream = null;
responseConnection.setLastResponseInputStream(null);
- if (shouldCloseConnection(responseConnection)) {
+ if (withErrors || shouldCloseConnection(responseConnection)) {
+ System.out.println("CLOSING CONNECTION");
responseConnection.close();
}
Index: org/apache/commons/httpclient/HttpMethodDirector.java
===================================================================
RCS file: /home/cvspublic/jakarta-commons/httpclient/src/java/org/apache/commons/httpclient/HttpMethodDirector.java,v
retrieving revision 1.5
diff -u -r1.5 HttpMethodDirector.java
--- org/apache/commons/httpclient/HttpMethodDirector.java 8 Oct 2003 23:34:16 -0000 1.5
+++ org/apache/commons/httpclient/HttpMethodDirector.java 11 Oct 2003 13:24:57 -0000
@@ -228,6 +228,8 @@
this.params.getConnectionManagerTimeout()
);
connection.setLocked(true);
+ connection.setResponseMaxBytes(this.params.getLongParameter(HttpMethodParams.RESPONSE_MAX_BYTES, 0));
+ connection.resetResponseByteCount(); // if we are reusing an existing HttpConnection object
realms = new HashSet();
proxyRealms = new HashSet();
Index: org/apache/commons/httpclient/LimitedSizeInputStream.java
===================================================================
RCS file: org/apache/commons/httpclient/LimitedSizeInputStream.java
diff -N org/apache/commons/httpclient/LimitedSizeInputStream.java
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ org/apache/commons/httpclient/LimitedSizeInputStream.java 11 Oct 2003 13:24:57 -0000
@@ -0,0 +1,210 @@
+package org.apache.commons.httpclient;
+
+import java.io.EOFException;
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+* An InputStream
which reads only up to a specific maximum number of bytes.
+*
+* @author Christian Kohlschuetter
+* @version $Id$
+*/
+public class LimitedSizeInputStream extends FilterInputStream {
+ private long max;
+ private long count = 0;
+ private boolean pastEOF = false;
+ private long mark = 0;
+ private boolean throwException = false;
+
+ /**
+ * Creates a new LimitedSizeInputStream
using the given
+ * underlying InputStream
, reading a maximum of maxBytes
+ * bytes from that stream.
+ */
+ public LimitedSizeInputStream(InputStream in, long maxBytes) {
+ super(in);
+ setLimit(maxBytes);
+ resetByteCounter();
+ }
+
+ /**
+ * Returns the current limit.
+ *
+ * @return Maximum number of bytes to be read.
+ */
+ public long getLimit() {
+ return max;
+ }
+ /**
+ * Sets the new limit.
+ *
+ * @param maxBytes Maximum number of bytes to be read.
+ */
+ public void setLimit(long maxBytes) {
+ long oldMax = max;
+ this.max = Math.max(0, maxBytes);
+ if (max < oldMax) {
+ pastEOF = true;
+ }
+ }
+ /**
+ * Returns the number of bytes read/skipped.
+ *
+ * @return Number of consumed bytes.
+ */
+ public long getByteCounter() {
+ return count;
+ }
+ /**
+ * Resets the read-bytes counter to 0.
+ */
+ public void resetByteCounter() {
+ count = 0;
+ mark = 0;
+ pastEOF = false;
+ }
+
+ /* (non-Javadoc)
+ * @see java.io.InputStream#read()
+ */
+ public int read() throws IOException {
+ if (pastEOF) {
+ return returnEOF();
+ }
+ if (max != 0 && count >= max) {
+ pastEOF = true;
+ return returnEOF();
+ }
+ int b = in.read();
+ if (b >= 0) {
+ count++;
+ } else {
+ pastEOF = true;
+ }
+ return b;
+ }
+
+ /* (non-Javadoc)
+ * @see java.io.InputStream#available()
+ */
+ public int available() throws IOException {
+ if (max != 0 && count >= max) {
+ return 0;
+ }
+ return Math.min(in.available(), (int) (max - count));
+ }
+
+ /* (non-Javadoc)
+ * @see java.io.InputStream#close()
+ */
+ public void close() throws IOException {
+ in.close();
+ }
+
+ /* (non-Javadoc)
+ * @see java.io.InputStream#mark(int)
+ */
+ public synchronized void mark(int readlimit) {
+ in.mark(readlimit);
+ mark = count;
+ }
+
+ /* (non-Javadoc)
+ * @see java.io.InputStream#read(byte[], int, int)
+ */
+ public int read(byte[] b, int off, int len) throws IOException {
+ if (pastEOF) {
+ return returnEOF();
+ }
+ if (max != 0) {
+ if (count >= max) {
+ return returnEOF();
+ }
+ len = Math.min(len, (int) (max - count));
+ }
+ if (len <= 0) {
+ return returnEOF();
+ }
+
+ int r = in.read(b, off, len);
+ if (r == -1) {
+ pastEOF = true;
+ } else {
+ count += r;
+ if (count >= max) {
+ pastEOF = true;
+ }
+ }
+ return r;
+ }
+
+ /* (non-Javadoc)
+ * @see java.io.InputStream#read(byte[])
+ */
+ public int read(byte[] b) throws IOException {
+ return read(b, 0, b.length);
+ }
+
+ /* (non-Javadoc)
+ * @see java.io.InputStream#reset()
+ */
+ public synchronized void reset() throws IOException {
+ in.reset();
+ count = mark;
+ pastEOF = (count > max);
+ }
+
+ /* (non-Javadoc)
+ * @see java.io.InputStream#skip(long)
+ */
+ public long skip(long n) throws IOException {
+ if (pastEOF) {
+ return 0;
+ }
+ if (max != 0) {
+ n = Math.min(n, max - count);
+ }
+ long skipped = in.skip(n);
+ count += skipped;
+ if (count >= max) {
+ pastEOF = true;
+ }
+ return skipped;
+ }
+
+ /**
+ * Checks if we thrown an EOFException when the limit is reached
+ *
+ * @return
+ */
+ public boolean isThrowException() {
+ return throwException;
+ }
+
+ /**
+ * Sets the state of throwing an EOFException instead of returning -1
+ * when the limit is reached.
+ *
+ * @param b
+ */
+ public void setThrowException(boolean b) {
+ throwException = b;
+ }
+
+ /**
+ * Returns -1 (EOF) or throws an EOFException if this behaviour was set
+ * using setThrowException
+ *
+ * @return -1
+ * @throws EOFException instead of -1, if setThrowException was set to true.
+ */
+ private int returnEOF() throws EOFException {
+ if (throwException && max != 0 && count >= max) {
+ throw new EOFException("Maximum permitted number of bytes read; forcing unexpected EOF");
+ }
+ return -1;
+ }
+
+}
Index: org/apache/commons/httpclient/ResponseConsumedWatcher.java
===================================================================
RCS file: /home/cvspublic/jakarta-commons/httpclient/src/java/org/apache/commons/httpclient/ResponseConsumedWatcher.java,v
retrieving revision 1.3
diff -u -r1.3 ResponseConsumedWatcher.java
--- org/apache/commons/httpclient/ResponseConsumedWatcher.java 28 Jan 2003 04:40:21 -0000 1.3
+++ org/apache/commons/httpclient/ResponseConsumedWatcher.java 11 Oct 2003 13:24:58 -0000
@@ -81,6 +81,8 @@
/**
* A response has been consumed.
+ *
+ * @param withErrors true if errors occured while consuming
*/
- void responseConsumed();
+ void responseConsumed(boolean withErrors);
}
Index: org/apache/commons/httpclient/params/HttpMethodParams.java
===================================================================
RCS file: /home/cvspublic/jakarta-commons/httpclient/src/java/org/apache/commons/httpclient/params/HttpMethodParams.java,v
retrieving revision 1.3
diff -u -r1.3 HttpMethodParams.java
--- org/apache/commons/httpclient/params/HttpMethodParams.java 3 Oct 2003 20:57:36 -0000 1.3
+++ org/apache/commons/httpclient/params/HttpMethodParams.java 11 Oct 2003 13:24:59 -0000
@@ -164,6 +164,13 @@
public static final String USE_EXPECT_CONTINUE = "http.protocol.expect-continue";
/**
+ * Sets the maximum number of bytes that will be read per HTTP method response.
+ * A value of zero means there is no limit. The default value is zero.
+ * This parameter expects a value of type {@link Long}.
+ */
+ public static final String RESPONSE_MAX_BYTES = "http.response.maxBytes";
+
+ /**
* Creates a new collection of parameters with the collection returned
* by {@link #getDefaultParams()} as a parent. The collection will defer
* to its parent for a default value if a particular parameter is not
--Boundary-00=_6bAi/9mDMSRpZL2
Content-Type: text/plain; charset=us-ascii
---------------------------------------------------------------------
To unsubscribe, e-mail: commons-httpclient-dev-unsubscribe@jakarta.apache.org
For additional commands, e-mail: commons-httpclient-dev-help@jakarta.apache.org
--Boundary-00=_6bAi/9mDMSRpZL2--