brooklyn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From henev...@apache.org
Subject [1/2] git commit: Use HttpClient for HTTP URLs in ResourceUtils.getResourceFromUrl
Date Tue, 23 Sep 2014 17:30:48 GMT
Repository: incubator-brooklyn
Updated Branches:
  refs/heads/master c249034b1 -> 472c06f88


Use HttpClient for HTTP URLs in ResourceUtils.getResourceFromUrl

HttpUrlConnection doesn't support authenticating using credentials in the url (http://user:pass@host)
and redirecting between protocols.
Switch to HttpClient which supports multiple authentication protocols and manually get the
credentials from the URL, passing them to HttpClient.


Project: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/commit/9b98c664
Tree: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/tree/9b98c664
Diff: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/diff/9b98c664

Branch: refs/heads/master
Commit: 9b98c6642ed6755405fde6225dfaf3401f9c38ec
Parents: c249034
Author: Svetoslav Neykov <svetoslav.neykov@cloudsoftcorp.com>
Authored: Tue Sep 23 15:32:34 2014 +0300
Committer: Svetoslav Neykov <svetoslav.neykov@cloudsoftcorp.com>
Committed: Tue Sep 23 18:06:06 2014 +0300

----------------------------------------------------------------------
 core/pom.xml                                    |   6 +
 .../main/java/brooklyn/util/ResourceUtils.java  |  72 +++++-
 .../brooklyn/util/ResourceUtilsHttpTest.java    | 244 +++++++++++++++++++
 .../java/brooklyn/util/ResourceUtilsTest.java   |   8 -
 pom.xml                                         |   6 +
 5 files changed, 327 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/9b98c664/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 4d33e60..cc08096 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -213,6 +213,12 @@
             <artifactId>jcommander</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+            <classifier>tests</classifier>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
   <!-- add maxmind geo-ip library; exact copy of source included as required by LGPL2
-->

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/9b98c664/core/src/main/java/brooklyn/util/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/util/ResourceUtils.java b/core/src/main/java/brooklyn/util/ResourceUtils.java
index 91ca566..ba704e0 100644
--- a/core/src/main/java/brooklyn/util/ResourceUtils.java
+++ b/core/src/main/java/brooklyn/util/ResourceUtils.java
@@ -18,23 +18,33 @@
  */
 package brooklyn.util;
 
+import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.io.UnsupportedEncodingException;
 import java.net.InetAddress;
 import java.net.JarURLConnection;
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.net.URLDecoder;
 import java.util.List;
 import java.util.NoSuchElementException;
 import java.util.Properties;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.auth.Credentials;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.util.EntityUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -45,6 +55,8 @@ import brooklyn.management.classloading.BrooklynClassLoadingContext;
 import brooklyn.management.classloading.JavaBrooklynClassLoadingContext;
 import brooklyn.util.collections.MutableMap;
 import brooklyn.util.exceptions.Exceptions;
+import brooklyn.util.http.HttpTool;
+import brooklyn.util.http.HttpTool.HttpClientBuilder;
 import brooklyn.util.javalang.Threads;
 import brooklyn.util.net.Networking;
 import brooklyn.util.net.Urls;
@@ -224,7 +236,11 @@ public class ResourceUtils {
                 if ("data".equals(protocol)) {
                     return new DataUriSchemeParser(url).lax().parse().getDataAsInputStream();
                 }
-                
+
+                if ("http".equals(protocol) || "https".equals(protocol)) {
+                    return getResourceViaHttp(url);
+                }
+
                 return new URL(url).openStream();
             }
 
@@ -388,6 +404,60 @@ public class ResourceUtils {
         }
     }
     
+    //For HTTP(S) targets use HttpClient so
+    //we can do authentication
+    private InputStream getResourceViaHttp(String resource) throws IOException {
+        URI uri = URI.create(resource);
+        HttpClientBuilder builder = HttpTool.httpClientBuilder()
+                .laxRedirect(true)
+                .uri(uri);
+        Credentials credentials = getUrlCredentials(uri.getRawUserInfo());
+        if (credentials != null) {
+            builder.credentials(credentials);
+        }
+        HttpClient client = builder.build();
+        HttpResponse result = client.execute(new HttpGet(resource));
+        int statusCode = result.getStatusLine().getStatusCode();
+        if (HttpTool.isStatusCodeHealthy(statusCode)) {
+            HttpEntity entity = result.getEntity();
+            if (entity != null) {
+                return entity.getContent();
+            } else {
+                return new ByteArrayInputStream(new byte[0]);
+            }
+        } else {
+            EntityUtils.consume(result.getEntity());
+            throw new IllegalStateException("Invalid response invoking " + resource + ":
response code " + statusCode);
+        }
+    }
+
+    private Credentials getUrlCredentials(String userInfo) {
+        if (userInfo != null) {
+            String[] arr = userInfo.split(":");
+            String username;
+            String password = null;
+            if (arr.length == 1) {
+                username = urlDecode(arr[0]);
+            } else if (arr.length == 2) {
+                username = urlDecode(arr[0]);
+                password = urlDecode(arr[1]);
+            } else {
+                return null;
+            }
+            return new UsernamePasswordCredentials(username, password);
+        } else {
+            return null;
+        }
+    }
+
+    private String urlDecode(String str) {
+        try {
+            return URLDecoder.decode(str, "UTF-8");
+        } catch (UnsupportedEncodingException e) {
+            throw Exceptions.propagate(e);
+        }
+    }
+
     /** takes {@link #getResourceFromUrl(String)} and reads fully, into a string */
     public String getResourceAsString(String url) {
         try {

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/9b98c664/core/src/test/java/brooklyn/util/ResourceUtilsHttpTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/brooklyn/util/ResourceUtilsHttpTest.java b/core/src/test/java/brooklyn/util/ResourceUtilsHttpTest.java
new file mode 100644
index 0000000..56d8166
--- /dev/null
+++ b/core/src/test/java/brooklyn/util/ResourceUtilsHttpTest.java
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package brooklyn.util;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetSocketAddress;
+
+import org.apache.http.Header;
+import org.apache.http.HttpException;
+import org.apache.http.HttpRequest;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.localserver.LocalTestServer;
+import org.apache.http.localserver.RequestBasicAuth;
+import org.apache.http.localserver.ResponseBasicUnauthorized;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.protocol.BasicHttpProcessor;
+import org.apache.http.protocol.HttpContext;
+import org.apache.http.protocol.HttpRequestHandler;
+import org.apache.http.protocol.ResponseConnControl;
+import org.apache.http.protocol.ResponseContent;
+import org.apache.http.protocol.ResponseServer;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import brooklyn.util.stream.Streams;
+import brooklyn.util.text.Strings;
+
+public class ResourceUtilsHttpTest {
+    private ResourceUtils utils;
+    private LocalTestServer server;
+    private String baseUrl;
+
+    @BeforeClass(alwaysRun=true)
+    public void setUp() throws Exception {
+        utils = ResourceUtils.create(this, "mycontext");
+
+        BasicHttpProcessor httpProcessor = new BasicHttpProcessor();
+        httpProcessor.addInterceptor(new ResponseServer());
+        httpProcessor.addInterceptor(new ResponseContent());
+        httpProcessor.addInterceptor(new ResponseConnControl());
+        httpProcessor.addInterceptor(new RequestBasicAuth());
+        httpProcessor.addInterceptor(new ResponseBasicUnauthorized());
+
+        server = new LocalTestServer(httpProcessor, null);
+        server.register("/simple", new SimpleResponseHandler("OK"));
+        server.register("/empty", new SimpleResponseHandler(HttpStatus.SC_NO_CONTENT, ""));
+        server.register("/missing", new SimpleResponseHandler(HttpStatus.SC_NOT_FOUND, "Missing"));
+        server.register("/redirect", new SimpleResponseHandler(HttpStatus.SC_MOVED_TEMPORARILY,
"Redirect", new BasicHeader("Location", "/simple")));
+        server.register("/cycle", new SimpleResponseHandler(HttpStatus.SC_MOVED_TEMPORARILY,
"Redirect", new BasicHeader("Location", "/cycle")));
+        server.register("/secure", new SimpleResponseHandler(HttpStatus.SC_MOVED_TEMPORARILY,
"Redirect", new BasicHeader("Location", "https://0.0.0.0/")));
+        server.register("/auth", new AuthHandler("test", "test", "OK"));
+        server.register("/auth_escape", new AuthHandler("test@me:/", "test", "OK"));
+        server.register("/auth_escape2", new AuthHandler("test@me:test", "", "OK"));
+        server.register("/no_credentials", new CheckNoCredentials());
+        server.start();
+
+        InetSocketAddress addr = server.getServiceAddress();
+        baseUrl = "http://" + addr.getHostName() + ":" + addr.getPort();
+    }
+
+    @AfterClass(alwaysRun=true)
+    public void tearDown() throws Exception {
+        server.stop();
+    }
+
+    @Test
+    public void testGet() throws Exception {
+        InputStream stream = utils.getResourceFromUrl(baseUrl + "/simple");
+        assertEquals(Streams.readFullyString(stream), "OK");
+    }
+
+    @Test
+    public void testGetEmpty() throws Exception {
+        InputStream stream = utils.getResourceFromUrl(baseUrl + "/empty");
+        assertEquals(Streams.readFullyString(stream), "");
+    }
+
+    @Test
+    public void testGetProtected() throws Exception {
+        String url = baseUrl.replace("http://", "http://test:test@") + "/auth";
+        InputStream stream = utils.getResourceFromUrl(url);
+        assertEquals(Streams.readFullyString(stream), "OK");
+    }
+
+    @Test
+    public void testGetProtectedEscape() throws Exception {
+        String url = baseUrl.replace("http://", "http://test%40me%3A%2F:test@") + "/auth_escape";
+        InputStream stream = utils.getResourceFromUrl(url);
+        assertEquals(Streams.readFullyString(stream), "OK");
+    }
+
+    @Test
+    public void testGetProtectedEscape2() throws Exception {
+        String url = baseUrl.replace("http://", "http://test%40me%3Atest@") + "/auth_escape2";
+        InputStream stream = utils.getResourceFromUrl(url);
+        assertEquals(Streams.readFullyString(stream), "OK");
+    }
+
+    @Test(expectedExceptions = RuntimeException.class)
+    public void testProtectedFailsWithoutCredentials() throws Exception {
+        utils.getResourceFromUrl(baseUrl + "/auth");
+    }
+
+    @Test
+    public void testInvalidCredentialsNotPassed() throws Exception {
+        String url = baseUrl + "/no_credentials?no:auth@needed";
+        InputStream stream = utils.getResourceFromUrl(url);
+        assertEquals(Streams.readFullyString(stream), "OK");
+    }
+
+    @Test
+    public void testRedirect() throws Exception {
+        InputStream stream = utils.getResourceFromUrl(baseUrl + "/redirect");
+        assertEquals(Streams.readFullyString(stream), "OK");
+    }
+
+    @Test(expectedExceptions = RuntimeException.class)
+    public void testCycleRedirect() throws Exception {
+        InputStream stream = utils.getResourceFromUrl(baseUrl + "/cycle");
+        assertEquals(Streams.readFullyString(stream), "OK");
+    }
+
+    @Test(expectedExceptions = RuntimeException.class)
+    public void testGetMissing() throws Exception {
+        utils.getResourceFromUrl(baseUrl + "/missing");
+    }
+
+    @Test
+    public void testFollowsProtoChange() throws Exception {
+        try {
+            utils.getResourceFromUrl(baseUrl + "/secure");
+        } catch (Exception e) {
+            assertTrue(e.getMessage().contains("Connection to https://0.0.0.0 refused"));
+        }
+    }
+
+    // See https://github.com/brooklyncentral/brooklyn/issues/1338
+    @Test(groups={"Integration"})
+    public void testResourceFromUrlFollowsRedirect() throws Exception {
+        String contents = new ResourceUtils(this).getResourceAsString("http://bit.ly/brooklyn-visitors-creation-script");
+        assertFalse(contents.contains("bit.ly"), "contents="+contents);
+    }
+
+    private static class SimpleResponseHandler implements HttpRequestHandler {
+        private int statusCode = HttpStatus.SC_OK;
+        private String responseBody = "";
+        private Header[] headers;
+
+        protected SimpleResponseHandler(String response) {
+            this.responseBody = response;
+        }
+
+        protected SimpleResponseHandler(int status, String response) {
+            this.statusCode = status;
+            this.responseBody = response;
+        }
+
+        protected SimpleResponseHandler(int status, String response, Header... headers) {
+            this.statusCode = status;
+            this.responseBody = response;
+            this.headers = headers;
+        }
+
+        @Override
+        public void handle(HttpRequest request, HttpResponse response, HttpContext context)
throws HttpException, IOException {
+            response.setStatusCode(statusCode);
+            response.setEntity(new StringEntity(responseBody));
+            if (headers != null) {
+                for (Header header : headers) {
+                    response.setHeader(header);
+                }
+            }
+        }
+    }
+
+    private static class AuthHandler implements HttpRequestHandler {
+        private String username;
+        private String password;
+        private String responseBody;
+
+        public AuthHandler(String username, String password, String response) {
+            this.username = username;
+            this.password = password;
+            this.responseBody = response;
+        }
+
+        public void handle(HttpRequest request, HttpResponse response, HttpContext context)
throws HttpException, IOException {
+            String creds = (String) context.getAttribute("creds");
+            if (creds == null || !creds.equals(getExpectedCredentials())) {
+                response.setStatusCode(HttpStatus.SC_UNAUTHORIZED);
+            } else {
+                response.setEntity(new StringEntity(responseBody));
+            }
+        }
+
+        private String getExpectedCredentials() {
+            if (Strings.isEmpty(password)) {
+                return username;
+            } else {
+                return username + ":" + password;
+            }
+        }
+
+    }
+
+    private static class CheckNoCredentials implements HttpRequestHandler {
+
+        @Override
+        public void handle(HttpRequest request, HttpResponse response,
+                HttpContext context) throws HttpException, IOException {
+            String creds = (String) context.getAttribute("creds");
+            if (creds == null) {
+                response.setEntity(new StringEntity("OK"));
+            } else {
+                response.setStatusCode(HttpStatus.SC_BAD_REQUEST);
+            }
+        }
+
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/9b98c664/core/src/test/java/brooklyn/util/ResourceUtilsTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/brooklyn/util/ResourceUtilsTest.java b/core/src/test/java/brooklyn/util/ResourceUtilsTest.java
index fb59c92..56c51cc 100644
--- a/core/src/test/java/brooklyn/util/ResourceUtilsTest.java
+++ b/core/src/test/java/brooklyn/util/ResourceUtilsTest.java
@@ -19,7 +19,6 @@
 package brooklyn.util;
 
 import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertNotNull;
 import static org.testng.Assert.assertTrue;
 
@@ -170,11 +169,4 @@ public class ResourceUtilsTest {
         assertEquals(utils.getResourceAsString("data://hello"), "hello");
         assertEquals(utils.getResourceAsString("data:hello world"), "hello world");
     }
-    
-    // See https://github.com/brooklyncentral/brooklyn/issues/1338
-    @Test(groups={"Integration", "WIP"})
-    public void testResourceFromUrlFollowsRedirect() throws Exception {
-        String contents = new ResourceUtils(this).getResourceAsString("http://bit.ly/brooklyn-visitors-creation-script");
-        assertFalse(contents.contains("bit.ly"), "contents="+contents);
-    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/9b98c664/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1f94f89..7623444 100644
--- a/pom.xml
+++ b/pom.xml
@@ -408,6 +408,12 @@
                 <version>${httpclient.version}</version>
             </dependency>
             <dependency>
+                <groupId>org.apache.httpcomponents</groupId>
+                <artifactId>httpclient</artifactId>
+                <classifier>tests</classifier>
+                <version>${httpclient.version}</version>
+            </dependency>
+            <dependency>
                 <groupId>aopalliance</groupId>
                 <artifactId>aopalliance</artifactId>
                 <version>${aopalliance.version}</version>


Mime
View raw message