ant-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maart...@apache.org
Subject svn commit: r951772 - in /ant/ivy/core/trunk: ./ src/java/org/apache/ivy/util/url/ test/java/org/apache/ivy/util/url/
Date Sat, 05 Jun 2010 21:25:27 GMT
Author: maartenc
Date: Sat Jun  5 21:25:26 2010
New Revision: 951772

URL: http://svn.apache.org/viewvc?rev=951772&view=rev
Log:
IMPROVEMENT: added support for 'deflate' HTTP Content-Encoding (IVY-1194)

Modified:
    ant/ivy/core/trunk/CHANGES.txt
    ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/AbstractURLHandler.java
    ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/BasicURLHandler.java
    ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/HttpClientHandler.java
    ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/BasicURLHandlerTest.java
    ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/HttpclientURLHandlerTest.java

Modified: ant/ivy/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/CHANGES.txt?rev=951772&r1=951771&r2=951772&view=diff
==============================================================================
--- ant/ivy/core/trunk/CHANGES.txt (original)
+++ ant/ivy/core/trunk/CHANGES.txt Sat Jun  5 21:25:26 2010
@@ -115,7 +115,7 @@ for detailed view of each issue, please 
 - NEW: Support ivy.xml parent mechanism (IVY-742) (thanks to Jason Trump and Jean-Louis Boudart)
 - NEW: Make ivy.xml <conf description> available (IVY-1158)
 
-- IMPROVEMENT: added support for 'gzip' HTTP Content-Encoding (IVY-1194)
+- IMPROVEMENT: added support for 'gzip' and 'deflate' HTTP Content-Encoding (IVY-1194)
 - IMPROVEMENT: retrieve doesn't retrive files if the current one is more recent (IVY-1044)
 - IMPROVEMENT: better diagnostics when reporting bad ivy file in cache (IVY-1190)
 - IMPROVEMENT: Support changing="true" for inline retrieve (IVY-1180)

Modified: ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/AbstractURLHandler.java
URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/AbstractURLHandler.java?rev=951772&r1=951771&r2=951772&view=diff
==============================================================================
--- ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/AbstractURLHandler.java (original)
+++ ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/AbstractURLHandler.java Sat Jun  5
21:25:26 2010
@@ -17,13 +17,19 @@
  */
 package org.apache.ivy.util.url;
 
+import java.io.BufferedInputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.regex.Pattern;
+import java.util.zip.DataFormatException;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.Inflater;
+import java.util.zip.InflaterInputStream;
 
 public abstract class AbstractURLHandler implements URLHandler {
     
@@ -118,4 +124,44 @@ public abstract class AbstractURLHandler
         
         return new URL(normalizeToString(url));
     }
+    
+    protected InputStream getDecodingInputStream(String encoding, InputStream in) 
+            throws IOException {
+        InputStream result = null;
+        
+        if ("gzip".equals(encoding)) {
+            result = new GZIPInputStream(in);
+        } else if ("deflate".equals(encoding)) {
+            // There seems to be 2 variants of the "deflate"-encoding.
+            // I couldn't find a way to auto-detect which variant was
+            // used, so as (a not really good) work-around we try do
+            // decompress the first 100 bytes using the "zlib"-variant.
+            BufferedInputStream bStream = new BufferedInputStream(in);
+            bStream.mark(100);
+            byte[] bytes = new byte[100];
+            int nbBytes = bStream.read(bytes);
+            bStream.reset();
+            
+            Inflater inflater = new Inflater();
+            inflater.setInput(bytes, 0, nbBytes);
+            try {
+                inflater.inflate(new byte[1000]);
+                
+                // no error decompressing the first 100 bytes, so we
+                // assume the "zlib"-variant was used.
+                result = new InflaterInputStream(bStream);
+            } catch (DataFormatException e) {
+                // there was an error decompressing the first 100 bytes,
+                // so we assume the "gzip/raw"-variant was used.
+                result = new InflaterInputStream(bStream, new Inflater(true));
+            } finally {
+                inflater.end();
+            }
+        } else {
+            result = in;
+        }
+        
+        return result;
+    }
+
 }

Modified: ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/BasicURLHandler.java
URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/BasicURLHandler.java?rev=951772&r1=951771&r2=951772&view=diff
==============================================================================
--- ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/BasicURLHandler.java (original)
+++ ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/BasicURLHandler.java Sat Jun  5 21:25:26
2010
@@ -28,7 +28,6 @@ import java.net.HttpURLConnection;
 import java.net.URL;
 import java.net.URLConnection;
 import java.net.UnknownHostException;
-import java.util.zip.GZIPInputStream;
 
 import org.apache.ivy.Ivy;
 import org.apache.ivy.util.CopyProgressListener;
@@ -117,7 +116,7 @@ public class BasicURLHandler extends Abs
             url = normalizeToURL(url);
             conn = url.openConnection();
             conn.setRequestProperty("User-Agent", "Apache Ivy/" + Ivy.getIvyVersion());
-            conn.setRequestProperty("Accept-Encoding", "gzip");
+            conn.setRequestProperty("Accept-Encoding", "gzip,deflate");
             if (conn instanceof HttpURLConnection) {
                 HttpURLConnection httpCon = (HttpURLConnection) conn;
                 if (!checkStatusCode(url, httpCon)) {
@@ -126,12 +125,8 @@ public class BasicURLHandler extends Abs
                                 + " See log for more detail.");
                 }
             }
-            InputStream inStream;
-            if ("gzip".equals(conn.getContentEncoding())) {
-                inStream = new GZIPInputStream(conn.getInputStream());
-            } else {
-                inStream = conn.getInputStream();
-            }
+            InputStream inStream = getDecodingInputStream(conn.getContentEncoding(),
+                                                          conn.getInputStream());
             ByteArrayOutputStream outStream = new ByteArrayOutputStream();
 
             byte[] buffer = new byte[BUFFER_SIZE];
@@ -154,7 +149,7 @@ public class BasicURLHandler extends Abs
             src = normalizeToURL(src);
             srcConn = src.openConnection();
             srcConn.setRequestProperty("User-Agent", "Apache Ivy/" + Ivy.getIvyVersion());
-            srcConn.setRequestProperty("Accept-Encoding", "gzip");
+            srcConn.setRequestProperty("Accept-Encoding", "gzip,deflate");
             if (srcConn instanceof HttpURLConnection) {
                 HttpURLConnection httpCon = (HttpURLConnection) srcConn;
                 if (!checkStatusCode(src, httpCon)) {
@@ -163,23 +158,24 @@ public class BasicURLHandler extends Abs
                                 + " See log for more detail.");
                 }
             }
-            int contentLength = srcConn.getContentLength();
-
-            InputStream inStream;
-            if ("gzip".equals(srcConn.getContentEncoding())) {
-                inStream = new GZIPInputStream(srcConn.getInputStream());
-                contentLength = -1;
-            } else {
-                inStream = srcConn.getInputStream();
-            }
 
+            // do the download
+            InputStream inStream = getDecodingInputStream(srcConn.getContentEncoding(),
+                                                          srcConn.getInputStream());
             FileUtil.copy(inStream, dest, l);
-            if (dest.length() != contentLength && contentLength != -1) {
-                dest.delete();
-                throw new IOException(
-                        "Downloaded file size doesn't match expected Content Length for "
+ src
-                                + ". Please retry.");
+
+            // check content length only if content was not encoded
+            if (srcConn.getContentEncoding() == null) {
+                int contentLength = srcConn.getContentLength();
+                if (contentLength != -1 && dest.length() != contentLength) {
+                    dest.delete();
+                    throw new IOException(
+                            "Downloaded file size doesn't match expected Content Length for
" + src
+                                    + ". Please retry.");
+                }
             }
+            
+            // update modification date
             long lastModified = srcConn.getLastModified();
             if (lastModified > 0) {
                 dest.setLastModified(lastModified);

Modified: ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/HttpClientHandler.java
URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/HttpClientHandler.java?rev=951772&r1=951771&r2=951772&view=diff
==============================================================================
--- ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/HttpClientHandler.java (original)
+++ ant/ivy/core/trunk/src/java/org/apache/ivy/util/url/HttpClientHandler.java Sat Jun  5
21:25:26 2010
@@ -28,7 +28,6 @@ import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
-import java.util.zip.GZIPInputStream;
 
 import org.apache.commons.httpclient.Credentials;
 import org.apache.commons.httpclient.Header;
@@ -101,7 +100,10 @@ public class HttpClientHandler extends A
             throw new IOException("The HTTP response code for " + url
                     + " did not indicate a success." + " See log for more detail.");
         }
-        return new GETInputStream(get);
+        
+        Header encoding = get.getResponseHeader("Content-Encoding");
+        return getDecodingInputStream(encoding == null ? null : encoding.getValue(),
+                                      get.getResponseBodyAsStream());
     }
 
     public void download(URL src, File dest, CopyProgressListener l) throws IOException {
@@ -113,14 +115,9 @@ public class HttpClientHandler extends A
                         + " did not indicate a success." + " See log for more detail.");
             }
             
-            InputStream is;
-            Header[] contentEncodings = get.getResponseHeaders("Content-Encoding");
-            if ((contentEncodings.length > 0) && "gzip".equals(contentEncodings[0].getValue()))
{
-                is = new GZIPInputStream(get.getResponseBodyAsStream());
-            } else {
-                is = get.getResponseBodyAsStream();
-            }
-
+            Header encoding = get.getResponseHeader("Content-Encoding");
+            InputStream is = getDecodingInputStream(encoding == null ? null : encoding.getValue(),
+                                                    get.getResponseBodyAsStream());
             FileUtil.copy(is, dest, l);
             dest.setLastModified(getLastModified(get));
         } finally {
@@ -255,7 +252,7 @@ public class HttpClientHandler extends A
 
         GetMethod get = new GetMethod(normalizeToString(url));
         get.setDoAuthentication(useAuthentication(url) || useProxyAuthentication());
-        get.setRequestHeader("Accept-Encoding", "gzip");
+        get.setRequestHeader("Accept-Encoding", "gzip,deflate");
         client.executeMethod(get);
         return get;
     }
@@ -321,72 +318,6 @@ public class HttpClientHandler extends A
         return (proxyUserName != null && proxyUserName.trim().length() > 0);
     }
 
-    private static final class GETInputStream extends InputStream {
-        private InputStream is;
-
-        private GetMethod get;
-
-        private GETInputStream(GetMethod get) throws IOException {
-            this.get = get;
-            
-            Header[] contentEncodings = get.getResponseHeaders("Content-Encoding");
-            if ((contentEncodings.length > 0) && "gzip".equals(contentEncodings[0].getValue()))
{
-                is = new GZIPInputStream(get.getResponseBodyAsStream());
-            } else {
-                is = get.getResponseBodyAsStream();
-            }
-        }
-
-        public int available() throws IOException {
-            return is.available();
-        }
-
-        public void close() throws IOException {
-            is.close();
-            get.releaseConnection();
-        }
-
-        public boolean equals(Object obj) {
-            return is.equals(obj);
-        }
-
-        public int hashCode() {
-            return is.hashCode();
-        }
-
-        public void mark(int readlimit) {
-            is.mark(readlimit);
-        }
-
-        public boolean markSupported() {
-            return is.markSupported();
-        }
-
-        public int read() throws IOException {
-            return is.read();
-        }
-
-        public int read(byte[] b, int off, int len) throws IOException {
-            return is.read(b, off, len);
-        }
-
-        public int read(byte[] b) throws IOException {
-            return is.read(b);
-        }
-
-        public void reset() throws IOException {
-            is.reset();
-        }
-
-        public long skip(long n) throws IOException {
-            return is.skip(n);
-        }
-
-        public String toString() {
-            return is.toString();
-        }
-    }
-
     private static final class HttpClientHelper3x implements HttpClientHelper {
         private static final int VERSION = 3;
 

Modified: ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/BasicURLHandlerTest.java
URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/BasicURLHandlerTest.java?rev=951772&r1=951771&r2=951772&view=diff
==============================================================================
--- ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/BasicURLHandlerTest.java (original)
+++ ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/BasicURLHandlerTest.java Sat Jun
 5 21:25:26 2010
@@ -20,6 +20,8 @@ package org.apache.ivy.util.url;
 import java.io.File;
 import java.net.URL;
 
+import org.apache.ivy.util.FileUtil;
+
 import junit.framework.TestCase;
 
 /**
@@ -27,8 +29,21 @@ import junit.framework.TestCase;
  */
 public class BasicURLHandlerTest extends TestCase {
     // remote.test
+    private File testDir;
+    private BasicURLHandler handler;
+    
+    protected void setUp() throws Exception {
+        testDir = new File("build/BasicURLHandlerTest");
+        testDir.mkdirs();
+        
+        handler = new BasicURLHandler();
+    }
+    
+    protected void tearDown() throws Exception {
+        FileUtil.forceDelete(testDir);
+    }
+    
     public void testIsReachable() throws Exception {
-        URLHandler handler = new BasicURLHandler();
         assertTrue(handler.isReachable(new URL("http://www.google.fr/")));
         assertFalse(handler.isReachable(new URL("http://www.google.fr/unknownpage.html")));
 
@@ -39,4 +54,19 @@ public class BasicURLHandlerTest extends
         // assertTrue(handler.isReachable(new URL("ftp://ftp.mozilla.org/pub/dir.sizes")));
         assertFalse(handler.isReachable(new URL("ftp://ftp.mozilla.org/unknown.file")));
     }
+    
+    public void testContentEncoding() throws Exception {
+        assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/daniels.html"), new
File(testDir, "gzip.txt"));
+        assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/daniels.html?deflate=on&zlib=on"),
new File(testDir, "deflate-zlib.txt"));
+        assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/daniels.html?deflate=on"),
new File(testDir, "deflate.txt"));
+        assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/a5.ps"), new File(testDir,
"a5-gzip.ps"));
+        assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/a5.ps?deflate=on"),
new File(testDir, "a5-deflate.ps"));
+        assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/nh80.pdf"), new File(testDir,
"nh80-gzip.pdf"));
+        assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/nh80.pdf?deflate=on"),
new File(testDir, "nh80-deflate.pdf"));
+    }
+    
+    private void assertDownloadOK(URL url, File file) throws Exception {
+        handler.download(url, file, null);
+        assertTrue(file.exists());        
+    }
 }

Modified: ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/HttpclientURLHandlerTest.java
URL: http://svn.apache.org/viewvc/ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/HttpclientURLHandlerTest.java?rev=951772&r1=951771&r2=951772&view=diff
==============================================================================
--- ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/HttpclientURLHandlerTest.java (original)
+++ ant/ivy/core/trunk/test/java/org/apache/ivy/util/url/HttpclientURLHandlerTest.java Sat
Jun  5 21:25:26 2010
@@ -17,8 +17,10 @@
  */
 package org.apache.ivy.util.url;
 
+import java.io.File;
 import java.net.URL;
 
+import org.apache.ivy.util.FileUtil;
 import org.apache.ivy.util.url.URLHandler.URLInfo;
 
 import junit.framework.TestCase;
@@ -28,8 +30,21 @@ import junit.framework.TestCase;
  */
 public class HttpclientURLHandlerTest extends TestCase {
     // remote.test
+    private File testDir;
+    private HttpClientHandler handler;
+    
+    protected void setUp() throws Exception {
+        testDir = new File("build/HttpclientURLHandlerTest");
+        testDir.mkdirs();
+        
+        handler = new HttpClientHandler();
+    }
+    
+    protected void tearDown() throws Exception {
+        FileUtil.forceDelete(testDir);
+    }
+    
     public void testIsReachable() throws Exception {
-        URLHandler handler = new HttpClientHandler();
         assertTrue(handler.isReachable(new URL("http://www.google.fr/")));
         assertFalse(handler.isReachable(new URL("http://www.google.fr/unknownpage.html")));
     }
@@ -43,4 +58,19 @@ public class HttpclientURLHandlerTest ex
         
         assertEquals(URLHandler.UNAVAILABLE, info);
     }
+    
+    public void testContentEncoding() throws Exception {
+        assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/daniels.html"), new
File(testDir, "gzip.txt"));
+        assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/daniels.html?deflate=on&zlib=on"),
new File(testDir, "deflate-zlib.txt"));
+        assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/daniels.html?deflate=on"),
new File(testDir, "deflate.txt"));
+        assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/a5.ps"), new File(testDir,
"a5-gzip.ps"));
+        assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/a5.ps?deflate=on"),
new File(testDir, "a5-deflate.ps"));
+        assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/nh80.pdf"), new File(testDir,
"nh80-gzip.pdf"));
+        assertDownloadOK(new URL("http://carsten.codimi.de/gzip.yaws/nh80.pdf?deflate=on"),
new File(testDir, "nh80-deflate.pdf"));
+    }
+    
+    private void assertDownloadOK(URL url, File file) throws Exception {
+        handler.download(url, file, null);
+        assertTrue(file.exists());        
+    }
 }



Mime
View raw message