Return-Path: Delivered-To: apmail-hc-commits-archive@www.apache.org Received: (qmail 28086 invoked from network); 6 Sep 2010 22:01:21 -0000 Received: from unknown (HELO mail.apache.org) (140.211.11.3) by 140.211.11.9 with SMTP; 6 Sep 2010 22:01:21 -0000 Received: (qmail 11202 invoked by uid 500); 6 Sep 2010 22:01:21 -0000 Delivered-To: apmail-hc-commits-archive@hc.apache.org Received: (qmail 11160 invoked by uid 500); 6 Sep 2010 22:01:20 -0000 Mailing-List: contact commits-help@hc.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: "HttpComponents Project" Delivered-To: mailing list commits@hc.apache.org Received: (qmail 11152 invoked by uid 99); 6 Sep 2010 22:01:20 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 06 Sep 2010 22:01:20 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 06 Sep 2010 22:01:19 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 3AA112388A38; Mon, 6 Sep 2010 22:00:59 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r993161 - in /httpcomponents/httpclient/trunk/httpclient-cache/src: main/java/org/apache/http/impl/client/cache/ test/java/org/apache/http/impl/client/cache/ Date: Mon, 06 Sep 2010 22:00:59 -0000 To: commits@hc.apache.org From: olegk@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20100906220059.3AA112388A38@eris.apache.org> Author: olegk Date: Mon Sep 6 22:00:58 2010 New Revision: 993161 URL: http://svn.apache.org/viewvc?rev=993161&view=rev Log: HTTPCLIENT-987: cache module does not recognize equivalent URIs Contributed by Jonathan Moore Modified: httpcomponents/httpclient/trunk/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheInvalidator.java httpcomponents/httpclient/trunk/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/URIExtractor.java httpcomponents/httpclient/trunk/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCacheInvalidator.java httpcomponents/httpclient/trunk/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestURIExtractor.java Modified: httpcomponents/httpclient/trunk/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheInvalidator.java URL: http://svn.apache.org/viewvc/httpcomponents/httpclient/trunk/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheInvalidator.java?rev=993161&r1=993160&r2=993161&view=diff ============================================================================== --- httpcomponents/httpclient/trunk/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheInvalidator.java (original) +++ httpcomponents/httpclient/trunk/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheInvalidator.java Mon Sep 6 22:00:58 2010 @@ -115,8 +115,9 @@ class CacheInvalidator { } protected void flushUriIfSameHost(URL requestURL, URL targetURL) throws IOException { - if (targetURL.getAuthority().equalsIgnoreCase(requestURL.getAuthority())) { - storage.removeEntry(targetURL.toString()); + URL canonicalTarget = new URL(uriExtractor.canonicalizeUri(targetURL.toString())); + if (canonicalTarget.getAuthority().equalsIgnoreCase(requestURL.getAuthority())) { + storage.removeEntry(canonicalTarget.toString()); } } Modified: httpcomponents/httpclient/trunk/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/URIExtractor.java URL: http://svn.apache.org/viewvc/httpcomponents/httpclient/trunk/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/URIExtractor.java?rev=993161&r1=993160&r2=993161&view=diff ============================================================================== --- httpcomponents/httpclient/trunk/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/URIExtractor.java (original) +++ httpcomponents/httpclient/trunk/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/URIExtractor.java Mon Sep 6 22:00:58 2010 @@ -27,6 +27,11 @@ package org.apache.http.impl.client.cache; import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.URLDecoder; import java.net.URLEncoder; import java.util.ArrayList; import java.util.Collections; @@ -56,7 +61,51 @@ class URIExtractor { * @return String the extracted URI */ public String getURI(HttpHost host, HttpRequest req) { - return String.format("%s%s", host.toString(), req.getRequestLine().getUri()); + if (isRelativeRequest(req)) { + return canonicalizeUri(String.format("%s%s", host.toString(), req.getRequestLine().getUri())); + } + return canonicalizeUri(req.getRequestLine().getUri()); + } + + public String canonicalizeUri(String uri) { + try { + URL u = new URL(uri); + String protocol = u.getProtocol().toLowerCase(); + String hostname = u.getHost().toLowerCase(); + int port = canonicalizePort(u.getPort(), protocol); + String path = canonicalizePath(u.getPath()); + if ("".equals(path)) path = "/"; + String query = u.getQuery(); + String file = (query != null) ? (path + "?" + query) : path; + URL out = new URL(protocol, hostname, port, file); + return out.toString(); + } catch (MalformedURLException e) { + return uri; + } + } + + private String canonicalizePath(String path) { + try { + String decoded = URLDecoder.decode(path, "UTF-8"); + return (new URI(decoded)).getPath(); + } catch (UnsupportedEncodingException e) { + } catch (URISyntaxException e) { + } + return path; + } + + private int canonicalizePort(int port, String protocol) { + if (port == -1 && "http".equalsIgnoreCase(protocol)) { + return 80; + } else if (port == -1 && "https".equalsIgnoreCase(protocol)) { + return 443; + } + return port; + } + + private boolean isRelativeRequest(HttpRequest req) { + String requestUri = req.getRequestLine().getUri(); + return ("*".equals(requestUri) || requestUri.startsWith("/")); } protected String getFullHeaderValue(Header[] headers) { Modified: httpcomponents/httpclient/trunk/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCacheInvalidator.java URL: http://svn.apache.org/viewvc/httpcomponents/httpclient/trunk/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCacheInvalidator.java?rev=993161&r1=993160&r2=993161&view=diff ============================================================================== --- httpcomponents/httpclient/trunk/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCacheInvalidator.java (original) +++ httpcomponents/httpclient/trunk/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCacheInvalidator.java Mon Sep 6 22:00:58 2010 @@ -76,7 +76,7 @@ public class TestCacheInvalidator { public void testInvalidatesRequestsThatArentGETorHEAD() throws Exception { HttpRequest request = new BasicHttpRequest("POST","/path", HTTP_1_1); - final String theUri = "http://foo.example.com/path"; + final String theUri = "http://foo.example.com:80/path"; Set variantURIs = new HashSet(); cacheEntryHasVariantURIs(variantURIs); @@ -96,15 +96,15 @@ public class TestCacheInvalidator { request.setHeader("Content-Length","128"); String contentLocation = "http://foo.example.com/content"; - request.setHeader("Content-Location",contentLocation); + request.setHeader("Content-Location", contentLocation); - final String theUri = "http://foo.example.com/"; + final String theUri = "http://foo.example.com:80/"; Set variantURIs = new HashSet(); cacheEntryHasVariantURIs(variantURIs); cacheReturnsEntryForUri(theUri); entryIsRemoved(theUri); - entryIsRemoved(contentLocation); + entryIsRemoved("http://foo.example.com:80/content"); replayMocks(); @@ -122,13 +122,13 @@ public class TestCacheInvalidator { String contentLocation = "http://foo.example.com/content"; request.setHeader("Location",contentLocation); - final String theUri = "http://foo.example.com/"; + final String theUri = "http://foo.example.com:80/"; Set variantURIs = new HashSet(); cacheEntryHasVariantURIs(variantURIs); cacheReturnsEntryForUri(theUri); entryIsRemoved(theUri); - entryIsRemoved(contentLocation); + entryIsRemoved(extractor.canonicalizeUri(contentLocation)); replayMocks(); @@ -143,17 +143,16 @@ public class TestCacheInvalidator { request.setEntity(HttpTestUtils.makeBody(128)); request.setHeader("Content-Length","128"); - String contentLocation = "http://foo.example.com/content"; String relativePath = "/content"; request.setHeader("Content-Location",relativePath); - final String theUri = "http://foo.example.com/"; + final String theUri = "http://foo.example.com:80/"; Set variantURIs = new HashSet(); cacheEntryHasVariantURIs(variantURIs); cacheReturnsEntryForUri(theUri); entryIsRemoved(theUri); - entryIsRemoved(contentLocation); + entryIsRemoved("http://foo.example.com:80/content"); replayMocks(); @@ -171,7 +170,7 @@ public class TestCacheInvalidator { String contentLocation = "http://bar.example.com/content"; request.setHeader("Content-Location",contentLocation); - final String theUri = "http://foo.example.com/"; + final String theUri = "http://foo.example.com:80/"; Set variantURIs = new HashSet(); cacheEntryHasVariantURIs(variantURIs); @@ -212,7 +211,7 @@ public class TestCacheInvalidator { HttpRequest request = new BasicHttpRequest("GET","/",HTTP_1_1); request.setHeader("Cache-Control","no-cache"); - final String theUri = "http://foo.example.com/"; + final String theUri = "http://foo.example.com:80/"; cacheReturnsEntryForUri(theUri); Set variantURIs = new HashSet(); cacheEntryHasVariantURIs(variantURIs); @@ -230,7 +229,7 @@ public class TestCacheInvalidator { HttpRequest request = new BasicHttpRequest("GET","/",HTTP_1_1); request.setHeader("Pragma","no-cache"); - final String theUri = "http://foo.example.com/"; + final String theUri = "http://foo.example.com:80/"; cacheReturnsEntryForUri(theUri); Set variantURIs = new HashSet(); cacheEntryHasVariantURIs(variantURIs); @@ -247,7 +246,7 @@ public class TestCacheInvalidator { public void testVariantURIsAreFlushedAlso() throws Exception { HttpRequest request = new BasicHttpRequest("POST","/",HTTP_1_1); - final String theUri = "http://foo.example.com/"; + final String theUri = "http://foo.example.com:80/"; final String variantUri = "theVariantURI"; Set listOfURIs = new HashSet(); @@ -267,7 +266,7 @@ public class TestCacheInvalidator { @Test(expected=IOException.class) public void testCacheFlushException() throws Exception { HttpRequest request = new BasicHttpRequest("POST","/",HTTP_1_1); - String theURI = "http://foo.example.com/"; + String theURI = "http://foo.example.com:80/"; cacheReturnsExceptionForUri(theURI); Modified: httpcomponents/httpclient/trunk/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestURIExtractor.java URL: http://svn.apache.org/viewvc/httpcomponents/httpclient/trunk/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestURIExtractor.java?rev=993161&r1=993160&r2=993161&view=diff ============================================================================== --- httpcomponents/httpclient/trunk/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestURIExtractor.java (original) +++ httpcomponents/httpclient/trunk/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestURIExtractor.java Mon Sep 6 22:00:58 2010 @@ -29,6 +29,8 @@ package org.apache.http.impl.client.cach import org.apache.http.Header; import org.apache.http.HttpHost; import org.apache.http.HttpRequest; +import org.apache.http.HttpVersion; +import org.apache.http.client.methods.HttpGet; import org.apache.http.message.BasicHeader; import org.apache.http.message.BasicHttpRequest; import org.easymock.classextension.EasyMock; @@ -66,17 +68,24 @@ public class TestURIExtractor { } @Test + public void testExtractsUriFromAbsoluteUriInRequest() { + HttpHost host = new HttpHost("bar.example.com"); + HttpRequest req = new HttpGet("http://foo.example.com/"); + Assert.assertEquals("http://foo.example.com:80/", extractor.getURI(host, req)); + } + + @Test public void testGetURIWithDefaultPortAndScheme() { - Assert.assertEquals("http://www.comcast.net/", extractor.getURI(new HttpHost( + Assert.assertEquals("http://www.comcast.net:80/", extractor.getURI(new HttpHost( "www.comcast.net"), REQUEST_ROOT)); - Assert.assertEquals("http://www.fancast.com/full_episodes", extractor.getURI(new HttpHost( + Assert.assertEquals("http://www.fancast.com:80/full_episodes", extractor.getURI(new HttpHost( "www.fancast.com"), REQUEST_FULL_EPISODES)); } @Test public void testGetURIWithDifferentScheme() { - Assert.assertEquals("https://www.comcast.net/", extractor.getURI(new HttpHost( + Assert.assertEquals("https://www.comcast.net:443/", extractor.getURI(new HttpHost( "www.comcast.net", -1, "https"), REQUEST_ROOT)); Assert.assertEquals("myhttp://www.fancast.com/full_episodes", extractor.getURI( @@ -103,9 +112,9 @@ public class TestURIExtractor { @Test public void testGetURIWithQueryParameters() { - Assert.assertEquals("http://www.comcast.net/?foo=bar", extractor.getURI(new HttpHost( + Assert.assertEquals("http://www.comcast.net:80/?foo=bar", extractor.getURI(new HttpHost( "www.comcast.net", -1, "http"), new BasicHttpRequest("GET", "/?foo=bar"))); - Assert.assertEquals("http://www.fancast.com/full_episodes?foo=bar", extractor.getURI( + Assert.assertEquals("http://www.fancast.com:80/full_episodes?foo=bar", extractor.getURI( new HttpHost("www.fancast.com", -1, "http"), new BasicHttpRequest("GET", "/full_episodes?foo=bar"))); } @@ -258,4 +267,104 @@ public class TestURIExtractor { .assertEquals("{Accept-Encoding=gzip%2C+deflate&User-Agent=browser}" + theURI, result); } + + /* + * "When comparing two URIs to decide if they match or not, a client + * SHOULD use a case-sensitive octet-by-octet comparison of the entire + * URIs, with these exceptions: + * - A port that is empty or not given is equivalent to the default + * port for that URI-reference; + * - Comparisons of host names MUST be case-insensitive; + * - Comparisons of scheme names MUST be case-insensitive; + * - An empty abs_path is equivalent to an abs_path of "/". + * Characters other than those in the 'reserved' and 'unsafe' sets + * (see RFC 2396 [42]) are equivalent to their '"%" HEX HEX' encoding." + * + * http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.2.3 + */ + @Test + public void testEmptyPortEquivalentToDefaultPortForHttp() { + HttpHost host1 = new HttpHost("foo.example.com:"); + HttpHost host2 = new HttpHost("foo.example.com:80"); + HttpRequest req = new BasicHttpRequest("GET", "/", HttpVersion.HTTP_1_1); + Assert.assertEquals(extractor.getURI(host1, req), extractor.getURI(host2, req)); + } + + @Test + public void testEmptyPortEquivalentToDefaultPortForHttps() { + HttpHost host1 = new HttpHost("foo.example.com", -1, "https"); + HttpHost host2 = new HttpHost("foo.example.com", 443, "https"); + HttpRequest req = new BasicHttpRequest("GET", "/", HttpVersion.HTTP_1_1); + final String uri1 = extractor.getURI(host1, req); + final String uri2 = extractor.getURI(host2, req); + Assert.assertEquals(uri1, uri2); + } + + @Test + public void testEmptyPortEquivalentToDefaultPortForHttpsAbsoluteURI() { + HttpHost host = new HttpHost("foo.example.com", -1, "https"); + HttpGet get1 = new HttpGet("https://bar.example.com:/"); + HttpGet get2 = new HttpGet("https://bar.example.com:443/"); + final String uri1 = extractor.getURI(host, get1); + final String uri2 = extractor.getURI(host, get2); + Assert.assertEquals(uri1, uri2); + } + + @Test + public void testNotProvidedPortEquivalentToDefaultPortForHttpsAbsoluteURI() { + HttpHost host = new HttpHost("foo.example.com", -1, "https"); + HttpGet get1 = new HttpGet("https://bar.example.com/"); + HttpGet get2 = new HttpGet("https://bar.example.com:443/"); + final String uri1 = extractor.getURI(host, get1); + final String uri2 = extractor.getURI(host, get2); + Assert.assertEquals(uri1, uri2); + } + + @Test + public void testNotProvidedPortEquivalentToDefaultPortForHttp() { + HttpHost host1 = new HttpHost("foo.example.com"); + HttpHost host2 = new HttpHost("foo.example.com:80"); + HttpRequest req = new BasicHttpRequest("GET", "/", HttpVersion.HTTP_1_1); + Assert.assertEquals(extractor.getURI(host1, req), extractor.getURI(host2, req)); + } + + @Test + public void testHostNameComparisonsAreCaseInsensitive() { + HttpHost host1 = new HttpHost("foo.example.com"); + HttpHost host2 = new HttpHost("FOO.EXAMPLE.COM"); + HttpRequest req = new BasicHttpRequest("GET", "/", HttpVersion.HTTP_1_1); + Assert.assertEquals(extractor.getURI(host1, req), extractor.getURI(host2, req)); + } + + @Test + public void testSchemeNameComparisonsAreCaseInsensitive() { + HttpHost host1 = new HttpHost("foo.example.com", -1, "http"); + HttpHost host2 = new HttpHost("foo.example.com", -1, "HTTP"); + HttpRequest req = new BasicHttpRequest("GET", "/", HttpVersion.HTTP_1_1); + Assert.assertEquals(extractor.getURI(host1, req), extractor.getURI(host2, req)); + } + + @Test + public void testEmptyAbsPathIsEquivalentToSlash() { + HttpHost host = new HttpHost("foo.example.com"); + HttpRequest req1 = new BasicHttpRequest("GET", "/", HttpVersion.HTTP_1_1); + HttpRequest req2 = new HttpGet("http://foo.example.com"); + Assert.assertEquals(extractor.getURI(host, req1), extractor.getURI(host, req2)); + } + + @Test + public void testEquivalentPathEncodingsAreEquivalent() { + HttpHost host = new HttpHost("foo.example.com"); + HttpRequest req1 = new BasicHttpRequest("GET", "/~smith/home.html", HttpVersion.HTTP_1_1); + HttpRequest req2 = new BasicHttpRequest("GET", "/%7Esmith/home.html", HttpVersion.HTTP_1_1); + Assert.assertEquals(extractor.getURI(host, req1), extractor.getURI(host, req2)); + } + + @Test + public void testEquivalentExtraPathEncodingsAreEquivalent() { + HttpHost host = new HttpHost("foo.example.com"); + HttpRequest req1 = new BasicHttpRequest("GET", "/~smith/home.html", HttpVersion.HTTP_1_1); + HttpRequest req2 = new BasicHttpRequest("GET", "/%7Esmith%2Fhome.html", HttpVersion.HTTP_1_1); + Assert.assertEquals(extractor.getURI(host, req1), extractor.getURI(host, req2)); + } }