asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ti...@apache.org
Subject asterixdb git commit: [NO ISSUE][HYR][*DB][HTTP][TEST] Minor refactoring / test functionality
Date Fri, 18 Aug 2017 05:59:02 GMT
Repository: asterixdb
Updated Branches:
  refs/heads/master 5170fb212 -> e0e2cfc2b


[NO ISSUE][HYR][*DB][HTTP][TEST] Minor refactoring / test functionality

Adds ability to indicate which status code(s) to accept for an HTTP
request, instead of just 200.

Change-Id: I1f134d7874cc7de712e60911e89dac446c5c2cfd
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1948
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <tillw@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/e0e2cfc2
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/e0e2cfc2
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/e0e2cfc2

Branch: refs/heads/master
Commit: e0e2cfc2bcc36fac48bf506838582dca245e8001
Parents: 5170fb2
Author: Michael Blow <mblow@apache.org>
Authored: Thu Aug 17 13:47:05 2017 -0400
Committer: Till Westmann <tillw@apache.org>
Committed: Thu Aug 17 22:58:45 2017 -0700

----------------------------------------------------------------------
 .../api/http/server/ClusterApiServlet.java      | 33 ++++-----
 .../asterix/test/common/TestExecutor.java       | 78 ++++++++++++--------
 .../hyracks/http/server/utils/HttpUtil.java     | 14 ++++
 3 files changed, 76 insertions(+), 49 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e0e2cfc2/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java
index 82e8f7a..b24a9a1 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java
@@ -26,7 +26,6 @@ import java.util.concurrent.ConcurrentMap;
 import java.util.function.Predicate;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-import java.util.regex.Pattern;
 
 import org.apache.asterix.runtime.utils.CcApplicationContext;
 import org.apache.asterix.runtime.utils.ClusterStateManager;
@@ -50,7 +49,6 @@ import io.netty.handler.codec.http.HttpResponseStatus;
 public class ClusterApiServlet extends AbstractServlet {
 
     private static final Logger LOGGER = Logger.getLogger(ClusterApiServlet.class.getName());
-    private static final Pattern PARENT_DIR = Pattern.compile("/[^./]+/\\.\\./");
     protected static final String NODE_ID_KEY = "node_id";
     protected static final String CONFIG_URI_KEY = "configUri";
     protected static final String STATS_URI_KEY = "statsUri";
@@ -104,15 +102,8 @@ public class ClusterApiServlet extends AbstractServlet {
                 Section.COMMON, getConfigSelector()));
 
         ArrayNode ncs = (ArrayNode) json.get("ncs");
-        final StringBuilder requestURL = new StringBuilder("http://");
-        requestURL.append(request.getHeader(HttpHeaderNames.HOST));
-        requestURL.append(request.getHttpRequest().uri());
-        if (requestURL.charAt(requestURL.length() - 1) != '/') {
-            requestURL.append('/');
-        }
-        requestURL.append(pathToNode);
-        String clusterURL = canonicalize(requestURL);
-        String adminURL = canonicalize(clusterURL + "../");
+        String clusterURL = resolveClusterUrl(request, pathToNode);
+        String adminURL = HttpUtil.canonicalize(clusterURL + "../");
         String nodeURL = clusterURL + "node/";
         for (int i = 0; i < ncs.size(); i++) {
             ObjectNode nc = (ObjectNode) ncs.get(i);
@@ -137,18 +128,20 @@ public class ClusterApiServlet extends AbstractServlet {
         return json;
     }
 
+    protected String resolveClusterUrl(IServletRequest request, String pathToNode) {
+        final StringBuilder requestURL = new StringBuilder("http://");
+        requestURL.append(request.getHeader(HttpHeaderNames.HOST));
+        requestURL.append(request.getHttpRequest().uri());
+        if (requestURL.charAt(requestURL.length() - 1) != '/') {
+            requestURL.append('/');
+        }
+        requestURL.append(pathToNode);
+        return HttpUtil.canonicalize(requestURL);
+    }
+
     protected Predicate<IOption> getConfigSelector() {
         return option -> !option.hidden() && option != ControllerConfig.Option.CONFIG_FILE
                 && option != ControllerConfig.Option.CONFIG_FILE_URL;
     }
 
-    private String canonicalize(CharSequence requestURL) {
-        String clusterURL = "";
-        String newClusterURL = requestURL.toString();
-        while (!clusterURL.equals(newClusterURL)) {
-            clusterURL = newClusterURL;
-            newClusterURL = PARENT_DIR.matcher(clusterURL).replaceAll("/");
-        }
-        return clusterURL;
-    }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e0e2cfc2/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index a96adc0..e07ec72 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -103,17 +103,18 @@ public class TestExecutor {
     // see
     // https://stackoverflow.com/questions/417142/what-is-the-maximum-length-of-a-url-in-different-browsers/417184
     private static final long MAX_URL_LENGTH = 2000l;
-    private static final Pattern JAVA_BLOCK_COMMENT_PATTERN =
-            Pattern.compile("/\\*.*\\*/", Pattern.MULTILINE | Pattern.DOTALL);
+    private static final Pattern JAVA_BLOCK_COMMENT_PATTERN = Pattern.compile("/\\*.*\\*/",
+            Pattern.MULTILINE | Pattern.DOTALL);
     private static final Pattern JAVA_LINE_COMMENT_PATTERN = Pattern.compile("^//.*$", Pattern.MULTILINE);
     private static final Pattern SHELL_LINE_COMMENT_PATTERN = Pattern.compile("^#.*$", Pattern.MULTILINE);
     private static final Pattern REGEX_LINES_PATTERN = Pattern.compile("^(-)?/(.*)/([im]*)$");
-    private static final Pattern POLL_TIMEOUT_PATTERN =
-            Pattern.compile("polltimeoutsecs=(\\d+)(\\D|$)", Pattern.MULTILINE);
+    private static final Pattern POLL_TIMEOUT_PATTERN = Pattern.compile("polltimeoutsecs=(\\d+)(\\D|$)",
+            Pattern.MULTILINE);
     private static final Pattern POLL_DELAY_PATTERN = Pattern.compile("polldelaysecs=(\\d+)(\\D|$)",
Pattern.MULTILINE);
     private static final Pattern HANDLE_VARIABLE_PATTERN = Pattern.compile("handlevariable=(\\w+)");
     private static final Pattern VARIABLE_REF_PATTERN = Pattern.compile("\\$(\\w+)");
     private static final Pattern HTTP_PARAM_PATTERN = Pattern.compile("param (\\w+)=(.*)",
Pattern.MULTILINE);
+    private static final Pattern HTTP_STATUSCODE_PATTERN = Pattern.compile("statuscode (.*)",
Pattern.MULTILINE);
 
     public static final int TRUNCATE_THRESHOLD = 16384;
 
@@ -168,10 +169,10 @@ public class TestExecutor {
     public void runScriptAndCompareWithResult(File scriptFile, PrintWriter print, File expectedFile,
File actualFile,
             ComparisonEnum compare) throws Exception {
         System.err.println("Expected results file: " + expectedFile.toString());
-        BufferedReader readerExpected =
-                new BufferedReader(new InputStreamReader(new FileInputStream(expectedFile),
"UTF-8"));
-        BufferedReader readerActual =
-                new BufferedReader(new InputStreamReader(new FileInputStream(actualFile),
"UTF-8"));
+        BufferedReader readerExpected = new BufferedReader(
+                new InputStreamReader(new FileInputStream(expectedFile), "UTF-8"));
+        BufferedReader readerActual = new BufferedReader(
+                new InputStreamReader(new FileInputStream(actualFile), "UTF-8"));
         boolean regex = false;
         try {
             if (ComparisonEnum.BINARY.equals(compare)) {
@@ -354,10 +355,10 @@ public class TestExecutor {
     public void runScriptAndCompareWithResultRegex(File scriptFile, File expectedFile, File
actualFile)
             throws Exception {
         String lineExpected, lineActual;
-        try (BufferedReader readerExpected =
-                new BufferedReader(new InputStreamReader(new FileInputStream(expectedFile),
"UTF-8"));
-                BufferedReader readerActual =
-                        new BufferedReader(new InputStreamReader(new FileInputStream(actualFile),
"UTF-8"))) {
+        try (BufferedReader readerExpected = new BufferedReader(
+                new InputStreamReader(new FileInputStream(expectedFile), "UTF-8"));
+                BufferedReader readerActual = new BufferedReader(
+                        new InputStreamReader(new FileInputStream(actualFile), "UTF-8")))
{
             StringBuilder actual = new StringBuilder();
             while ((lineActual = readerActual.readLine()) != null) {
                 actual.append(lineActual).append('\n');
@@ -697,8 +698,8 @@ public class TestExecutor {
     // Insert and Delete statements are executed here
     public void executeUpdate(String str, URI uri) throws Exception {
         // Create a method instance.
-        HttpUriRequest request =
-                RequestBuilder.post(uri).setEntity(new StringEntity(str, StandardCharsets.UTF_8)).build();
+        HttpUriRequest request = RequestBuilder.post(uri).setEntity(new StringEntity(str,
StandardCharsets.UTF_8))
+                .build();
 
         // Execute the method.
         executeAndCheckHttpRequest(request);
@@ -708,10 +709,10 @@ public class TestExecutor {
     public InputStream executeAnyAQLAsync(String statement, boolean defer, OutputFormat fmt,
URI uri,
             Map<String, Object> variableCtx) throws Exception {
         // Create a method instance.
-        HttpUriRequest request =
-                RequestBuilder.post(uri).addParameter("mode", defer ? "asynchronous-deferred"
: "asynchronous")
-                        .setEntity(new StringEntity(statement, StandardCharsets.UTF_8))
-                        .setHeader("Accept", fmt.mimeType()).build();
+        HttpUriRequest request = RequestBuilder.post(uri)
+                .addParameter("mode", defer ? "asynchronous-deferred" : "asynchronous")
+                .setEntity(new StringEntity(statement, StandardCharsets.UTF_8)).setHeader("Accept",
fmt.mimeType())
+                .build();
 
         String handleVar = getHandleVariable(statement);
 
@@ -737,8 +738,8 @@ public class TestExecutor {
     // create function statement
     public void executeDDL(String str, URI uri) throws Exception {
         // Create a method instance.
-        HttpUriRequest request =
-                RequestBuilder.post(uri).setEntity(new StringEntity(str, StandardCharsets.UTF_8)).build();
+        HttpUriRequest request = RequestBuilder.post(uri).setEntity(new StringEntity(str,
StandardCharsets.UTF_8))
+                .build();
 
         // Execute the method.
         executeAndCheckHttpRequest(request);
@@ -748,8 +749,8 @@ public class TestExecutor {
     // and returns the contents as a string
     // This string is later passed to REST API for execution.
     public String readTestFile(File testFile) throws Exception {
-        BufferedReader reader =
-                new BufferedReader(new InputStreamReader(new FileInputStream(testFile), StandardCharsets.UTF_8));
+        BufferedReader reader = new BufferedReader(
+                new InputStreamReader(new FileInputStream(testFile), StandardCharsets.UTF_8));
         String line;
         StringBuilder stringBuilder = new StringBuilder();
         String ls = System.getProperty("line.separator");
@@ -804,8 +805,8 @@ public class TestExecutor {
 
     private static String getProcessOutput(Process p) throws Exception {
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
-        Future<Integer> future =
-                Executors.newSingleThreadExecutor().submit(() -> IOUtils.copy(p.getInputStream(),
new OutputStream() {
+        Future<Integer> future = Executors.newSingleThreadExecutor()
+                .submit(() -> IOUtils.copy(p.getInputStream(), new OutputStream() {
                     @Override
                     public void write(int b) throws IOException {
                         baos.write(b);
@@ -1091,11 +1092,12 @@ public class TestExecutor {
         final String trimmedPathAndQuery = stripLineComments(stripJavaComments(statement)).trim();
         final String variablesReplaced = replaceVarRef(trimmedPathAndQuery, variableCtx);
         final List<Parameter> params = extractParameters(statement);
+        final Predicate<Integer> statusCodePredicate = extractStatusCodePredicate(statement);
         InputStream resultStream;
         if ("http".equals(extension)) {
-            resultStream = executeHttp(reqType, variablesReplaced, fmt, params);
+            resultStream = executeHttp(reqType, variablesReplaced, fmt, params, statusCodePredicate);
         } else if ("uri".equals(extension)) {
-            resultStream = executeURI(reqType, URI.create(variablesReplaced), fmt, params);
+            resultStream = executeURI(reqType, URI.create(variablesReplaced), fmt, params,
statusCodePredicate);
         } else {
             throw new IllegalArgumentException("Unexpected format for method " + reqType
+ ": " + extension);
         }
@@ -1287,17 +1289,35 @@ public class TestExecutor {
         return params;
     }
 
-    protected InputStream executeHttp(String ctxType, String endpoint, OutputFormat fmt,
List<Parameter> params)
-            throws Exception {
+    protected static Predicate<Integer> extractStatusCodePredicate(String statement)
{
+        List<Integer> codes = new ArrayList<>();
+        final Matcher m = HTTP_STATUSCODE_PATTERN.matcher(statement);
+        while (m.find()) {
+            codes.add(Integer.parseInt(m.group(1)));
+        }
+        if (codes.isEmpty()) {
+            return code -> code == HttpStatus.SC_OK;
+        } else {
+            return codes::contains;
+        }
+    }
+
+    protected InputStream executeHttp(String ctxType, String endpoint, OutputFormat fmt,
List<Parameter> params,
+            Predicate<Integer> statusCodePredicate) throws Exception {
         String[] split = endpoint.split("\\?");
         URI uri = createEndpointURI(split[0], split.length > 1 ? split[1] : null);
-        return executeURI(ctxType, uri, fmt, params);
+        return executeURI(ctxType, uri, fmt, params, statusCodePredicate);
     }
 
     private InputStream executeURI(String ctxType, URI uri, OutputFormat fmt, List<Parameter>
params) throws Exception {
         return executeJSON(fmt, ctxType.toUpperCase(), uri, params);
     }
 
+    private InputStream executeURI(String ctxType, URI uri, OutputFormat fmt, List<Parameter>
params,
+            Predicate<Integer> responseCodeValidator) throws Exception {
+        return executeJSON(fmt, ctxType.toUpperCase(), uri, params, responseCodeValidator);
+    }
+
     private void killNC(String nodeId, CompilationUnit cUnit) throws Exception {
         //get node process id
         OutputFormat fmt = OutputFormat.forCompilationUnit(cUnit);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e0e2cfc2/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
index 2babc73..fa3cc57 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.Map;
+import java.util.regex.Pattern;
 
 import org.apache.hyracks.http.api.IServletRequest;
 import org.apache.hyracks.http.api.IServletResponse;
@@ -34,6 +35,8 @@ import io.netty.handler.codec.http.HttpRequest;
 
 public class HttpUtil {
 
+    private static final Pattern PARENT_DIR = Pattern.compile("/[^./]+/\\.\\./");
+
     private HttpUtil() {
     }
 
@@ -131,4 +134,15 @@ public class HttpUtil {
                 return null;
         }
     }
+
+    public static String canonicalize(CharSequence requestURL) {
+        String clusterURL = "";
+        String newClusterURL = requestURL.toString();
+        while (!clusterURL.equals(newClusterURL)) {
+            clusterURL = newClusterURL;
+            newClusterURL = PARENT_DIR.matcher(clusterURL).replaceAll("/");
+        }
+        return clusterURL;
+    }
+
 }


Mime
View raw message