metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rmerri...@apache.org
Subject [metron] branch master updated: METRON-1993 Stellar REST_GET should handle responses when content length is less than zero (merrimanr) closes apache/metron#1331
Date Thu, 14 Feb 2019 16:43:05 GMT
This is an automated email from the ASF dual-hosted git repository.

rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git


The following commit(s) were added to refs/heads/master by this push:
     new 4d420f4  METRON-1993 Stellar REST_GET should handle responses when content length
is less than zero (merrimanr) closes apache/metron#1331
4d420f4 is described below

commit 4d420f4d1704b2da4e725de3fda4c08d97691611
Author: merrimanr <merrimanr@gmail.com>
AuthorDate: Thu Feb 14 10:42:50 2019 -0600

    METRON-1993 Stellar REST_GET should handle responses when content length is less than
zero (merrimanr) closes apache/metron#1331
---
 .../metron/stellar/dsl/functions/RestConfig.java   |  9 +++
 .../stellar/dsl/functions/RestFunctions.java       | 30 ++++++--
 .../stellar/dsl/functions/RestFunctionsTest.java   | 85 ++++++++++++++++++++--
 3 files changed, 111 insertions(+), 13 deletions(-)

diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestConfig.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestConfig.java
index fdb6935..610717e 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestConfig.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestConfig.java
@@ -98,10 +98,15 @@ public class RestConfig extends HashMap<String, Object> {
    */
   public final static String POOLING_DEFAULT_MAX_PER_RUOTE = "pooling.default.max.per.route";
 
+  /**
+   * Setting this to true will verify the actual body content length equals the content length
header
+   */
+  public final static String VERIFY_CONTENT_LENGTH = "verify.content.length";
 
   public RestConfig() {
     put(TIMEOUT, 1000);
     put(RESPONSE_CODES_ALLOWED, Collections.singletonList(200));
+    put(VERIFY_CONTENT_LENGTH, false);
   }
 
   public String getBasicAuthUser() {
@@ -164,4 +169,8 @@ public class RestConfig extends HashMap<String, Object> {
   public Integer getPoolingDefaultMaxPerRoute() {
     return (Integer) get(POOLING_DEFAULT_MAX_PER_RUOTE);
   }
+
+  public Boolean verifyContentLength() {
+    return (Boolean) get(VERIFY_CONTENT_LENGTH);
+  }
 }
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java
index f07d54e..d6b03ce 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java
@@ -244,15 +244,13 @@ public class RestFunctions {
         scheduledFuture.cancel(true);
       }
       int statusCode = response.getStatusLine().getStatusCode();
+      LOG.debug("request = {}; response = {}", httpGet, response);
       if (restConfig.getResponseCodesAllowed().contains(statusCode)) {
         HttpEntity httpEntity = response.getEntity();
 
-        // Parse the reponse if present, return the empty value override if not
-        if (httpEntity != null && httpEntity.getContentLength() > 0) {
-          String json = EntityUtils.toString(response.getEntity());
-          return JSONUtils.INSTANCE.load(json, JSONUtils.MAP_SUPPLIER);
-        }
-        return restConfig.getEmptyContentOverride();
+        // Parse the response if present, return the empty value override if not
+        Optional<Object> parsedResponse = parseResponse(restConfig, httpGet, httpEntity);
+        return parsedResponse.orElseGet(restConfig::getEmptyContentOverride);
       } else {
         throw new IOException(String.format("Stellar REST request to %s expected status code
to be one of %s but " +
                 "failed with http status code %d: %s",
@@ -374,6 +372,26 @@ public class RestFunctions {
       return httpClientContext;
     }
 
+    protected Optional<Object> parseResponse(RestConfig restConfig, HttpGet httpGet,
HttpEntity httpEntity) throws IOException {
+      Optional<Object> parsedResponse = Optional.empty();
+      if (httpEntity != null) {
+        int actualContentLength = 0;
+        String json = EntityUtils.toString(httpEntity);
+        if (json != null && !json.isEmpty()) {
+          actualContentLength = json.length();
+          parsedResponse = Optional.of(JSONUtils.INSTANCE.load(json, JSONUtils.MAP_SUPPLIER));
+        }
+        if (restConfig.verifyContentLength() && actualContentLength != httpEntity.getContentLength())
{
+          throw new IOException(String.format("Stellar REST request to %s returned incorrect
or missing content length. " +
+                          "Content length in the response was %d but the actual body content
length was %d.",
+                  httpGet.getURI().toString(),
+                  httpEntity.getContentLength(),
+                  actualContentLength));
+        }
+      }
+      return parsedResponse;
+    }
+
     /**
      * Read bytes from a HDFS path.
      * @param inPath
diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/RestFunctionsTest.java
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/RestFunctionsTest.java
index 70f7d2f..2008a95 100644
--- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/RestFunctionsTest.java
+++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/RestFunctionsTest.java
@@ -19,6 +19,7 @@ package org.apache.metron.stellar.dsl.functions;
 
 import org.adrianwalker.multilinestring.Multiline;
 import org.apache.commons.io.FileUtils;
+import org.apache.http.HttpEntity;
 import org.apache.http.HttpHost;
 import org.apache.http.auth.AuthScope;
 import org.apache.http.auth.UsernamePasswordCredentials;
@@ -40,8 +41,10 @@ import org.mockserver.client.server.MockServerClient;
 import org.mockserver.junit.MockServerRule;
 import org.mockserver.junit.ProxyRule;
 
+import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.IOException;
+import java.net.URI;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Collections;
@@ -63,8 +66,10 @@ import static org.apache.metron.stellar.dsl.functions.RestConfig.PROXY_PORT;
 import static org.apache.metron.stellar.dsl.functions.RestConfig.SOCKET_TIMEOUT;
 import static org.apache.metron.stellar.dsl.functions.RestConfig.STELLAR_REST_SETTINGS;
 import static org.apache.metron.stellar.dsl.functions.RestConfig.TIMEOUT;
+import static org.apache.metron.stellar.dsl.functions.RestConfig.VERIFY_CONTENT_LENGTH;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
@@ -72,6 +77,7 @@ import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
 import static org.mockserver.model.HttpRequest.request;
 import static org.mockserver.model.HttpResponse.response;
 
@@ -269,7 +275,7 @@ public class RestFunctionsTest {
       // Test for default timeout
       RestConfig restConfig = restGet.getRestConfig(Collections.singletonList("uri"), new
HashMap<>());
 
-      assertEquals(2, restConfig.size());
+      assertEquals(3, restConfig.size());
       assertEquals(1000, restConfig.getTimeout().intValue());
       assertEquals(Collections.singletonList(200), restConfig.getResponseCodesAllowed());
       assertNull(restConfig.getBasicAuthUser());
@@ -287,7 +293,7 @@ public class RestFunctionsTest {
     {
       RestConfig restConfig = restGet.getRestConfig(Collections.singletonList("uri"), globalRestConfig);
 
-      assertEquals(5, restConfig.size());
+      assertEquals(6, restConfig.size());
       assertEquals(1000, restConfig.getTimeout().intValue());
       assertEquals(Collections.singletonList(200), restConfig.getResponseCodesAllowed());
       assertEquals(2000, restConfig.getSocketTimeout().intValue());
@@ -306,7 +312,7 @@ public class RestFunctionsTest {
     {
       RestConfig restConfig = restGet.getRestConfig(Arrays.asList("uri", functionRestConfig),
globalRestConfig);
 
-      assertEquals(5, restConfig.size());
+      assertEquals(6, restConfig.size());
       assertEquals(Collections.singletonList(200), restConfig.getResponseCodesAllowed());
       assertEquals(100, restConfig.getTimeout().intValue());
       assertEquals(1, restConfig.getSocketTimeout().intValue());
@@ -323,7 +329,7 @@ public class RestFunctionsTest {
     {
       RestConfig restConfig = restGet.getRestConfig(Arrays.asList("uri", functionRestConfig),
globalRestConfig);
 
-      assertEquals(5, restConfig.size());
+      assertEquals(6, restConfig.size());
       assertEquals(Collections.singletonList(200), restConfig.getResponseCodesAllowed());
       assertEquals(100, restConfig.getTimeout().intValue());
       assertEquals(2000, restConfig.getSocketTimeout().intValue());
@@ -342,7 +348,7 @@ public class RestFunctionsTest {
     {
       RestConfig restConfig = restGet.getRestConfig(Arrays.asList("uri", functionRestConfig),
globalRestConfig);
 
-      assertEquals(4, restConfig.size());
+      assertEquals(5, restConfig.size());
       assertEquals(Collections.singletonList(200), restConfig.getResponseCodesAllowed());
       assertEquals(100, restConfig.getTimeout().intValue());
       assertEquals(2000, restConfig.getSocketTimeout().intValue());
@@ -353,7 +359,7 @@ public class RestFunctionsTest {
     {
       RestConfig restConfig = restGet.getRestConfig(Collections.singletonList("uri"), globalRestConfig);
 
-      assertEquals(4, restConfig.size());
+      assertEquals(5, restConfig.size());
       assertEquals(Collections.singletonList(200), restConfig.getResponseCodesAllowed());
       assertEquals(1000, restConfig.getTimeout().intValue());
       assertEquals(2000, restConfig.getSocketTimeout().intValue());
@@ -364,7 +370,7 @@ public class RestFunctionsTest {
     {
       RestConfig restConfig = restGet.getRestConfig(Collections.singletonList("uri"), new
HashMap<>());
 
-      assertEquals(2, restConfig.size());
+      assertEquals(3, restConfig.size());
       assertEquals(Collections.singletonList(200), restConfig.getResponseCodesAllowed());
       assertEquals(1000, restConfig.getTimeout().intValue());
     }
@@ -602,4 +608,69 @@ public class RestFunctionsTest {
     verifyNoMoreInteractions(httpClient);
   }
 
+  @Test
+  public void restGetShouldParseResponse() throws Exception {
+    RestFunctions.RestGet restGet = new RestFunctions.RestGet();
+    RestConfig restConfig = new RestConfig();
+    HttpGet httpGet = mock(HttpGet.class);
+    HttpEntity httpEntity = mock(HttpEntity.class);
+
+    // return successfully parsed response
+    when(httpEntity.getContent()).thenReturn(new ByteArrayInputStream("{\"get\":\"success\"}".getBytes()));
+    Optional<Object> actual = restGet.parseResponse(restConfig, httpGet, httpEntity);
+    assertTrue(actual.isPresent());
+    assertEquals("success", ((Map<String, Object>) actual.get()).get("get"));
+  }
+
+  @Test
+  public void restGetShouldParseResponseOnNullHttpEntity() throws Exception {
+    RestFunctions.RestGet restGet = new RestFunctions.RestGet();
+    RestConfig restConfig = new RestConfig();
+    HttpGet httpGet = mock(HttpGet.class);
+
+    // return empty on null httpEntity
+    assertEquals(Optional.empty(), restGet.parseResponse(restConfig, httpGet, null));
+  }
+
+  @Test
+  public void restGetShouldParseResponseOnNullContent() throws Exception {
+    RestFunctions.RestGet restGet = new RestFunctions.RestGet();
+    RestConfig restConfig = new RestConfig();
+    HttpGet httpGet = mock(HttpGet.class);
+    HttpEntity httpEntity = mock(HttpEntity.class);
+
+    // return empty on null content
+    when(httpEntity.getContent()).thenReturn(null);
+    assertEquals(Optional.empty(), restGet.parseResponse(restConfig, httpGet, httpEntity));
+  }
+
+  @Test
+  public void restGetShouldParseResponseOnEmptyInputStream() throws Exception {
+    RestFunctions.RestGet restGet = new RestFunctions.RestGet();
+    RestConfig restConfig = new RestConfig();
+    HttpGet httpGet = mock(HttpGet.class);
+    HttpEntity httpEntity = mock(HttpEntity.class);
+
+    // return empty on empty input stream
+    when(httpEntity.getContent()).thenReturn(new ByteArrayInputStream("".getBytes()));
+    assertEquals(Optional.empty(), restGet.parseResponse(restConfig, httpGet, httpEntity));
+  }
+
+  @Test
+  public void restGetShouldThrowExceptionOnContentLengthMismatch() throws Exception {
+    thrown.expect(IOException.class);
+    thrown.expectMessage("Stellar REST request to uri returned incorrect or missing content
length. Content length in the response was -1 but the actual body content length was 17.");
+
+    RestFunctions.RestGet restGet = new RestFunctions.RestGet();
+    RestConfig restConfig = new RestConfig();
+    HttpGet httpGet = mock(HttpGet.class);
+    HttpEntity httpEntity = mock(HttpEntity.class);
+
+    restConfig.put(VERIFY_CONTENT_LENGTH, true);
+    when(httpGet.getURI()).thenReturn(new URI("uri"));
+    when(httpEntity.getContent()).thenReturn(new ByteArrayInputStream("{\"get\":\"success\"}".getBytes()));
+    when(httpEntity.getContentLength()).thenReturn(-1L);
+    restGet.parseResponse(restConfig, httpGet, httpEntity);
+  }
+
 }


Mime
View raw message