asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ti...@apache.org
Subject asterixdb git commit: add support for JSON encoded requests
Date Sat, 24 Sep 2016 00:50:26 GMT
Repository: asterixdb
Updated Branches:
  refs/heads/master 8bef7a3a8 -> 6762f03ec


add support for JSON encoded requests

- run SQL++ execution test *queries* using JSON encoded requests
- fix metadata cleanup at the end of the tests

Change-Id: I1cc934d5dd984b476d4adb1755572d2e2f451985
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1201
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Yingyi Bu <buyingyi@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/6762f03e
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/6762f03e
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/6762f03e

Branch: refs/heads/master
Commit: 6762f03ec4c35d951c0af7c69363ca9898afba6e
Parents: 8bef7a3
Author: Till Westmann <tillw@apache.org>
Authored: Fri Sep 23 14:41:57 2016 -0700
Committer: Till Westmann <tillw@apache.org>
Committed: Fri Sep 23 17:49:57 2016 -0700

----------------------------------------------------------------------
 .../api/http/servlet/QueryServiceServlet.java   | 90 ++++++++++++++------
 .../apache/asterix/test/aql/TestExecutor.java   | 79 ++++++++++-------
 2 files changed, 111 insertions(+), 58 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6762f03e/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java
index 132737b..856aa40 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java
@@ -56,6 +56,8 @@ import org.apache.hyracks.algebricks.core.algebra.prettyprint.AlgebricksAppendab
 import org.apache.hyracks.api.client.IHyracksClientConnection;
 import org.apache.hyracks.api.dataset.IHyracksDataset;
 import org.apache.hyracks.client.dataset.HyracksDataset;
+import org.json.JSONException;
+import org.json.JSONObject;
 
 public class QueryServiceServlet extends HttpServlet {
     private static final long serialVersionUID = 1L;
@@ -71,10 +73,9 @@ public class QueryServiceServlet extends HttpServlet {
     }
 
     public enum Parameter {
-        // Standard
         STATEMENT("statement"),
         FORMAT("format"),
-        // Asterix
+        CLIENT_ID("client_context_id"),
         PRETTY("pretty");
 
         private final String str;
@@ -121,6 +122,7 @@ public class QueryServiceServlet extends HttpServlet {
 
     public enum ResultFields {
         REQUEST_ID("requestID"),
+        CLIENT_ID("clientContextID"),
         SIGNATURE("signature"),
         TYPE("type"),
         STATUS("status"),
@@ -217,6 +219,13 @@ public class QueryServiceServlet extends HttpServlet {
         }
     }
 
+    static class RequestParameters {
+        String statement;
+        String format;
+        boolean pretty;
+        String clientContextID;
+    }
+
     private static String getParameterValue(String content, String attribute) {
         if (content == null || attribute == null) {
             return null;
@@ -255,11 +264,7 @@ public class QueryServiceServlet extends HttpServlet {
         return SessionConfig.OutputFormat.CLEAN_JSON;
     }
 
-    /**
-     * Construct a SessionConfig with the appropriate output writer and
-     * output-format based on the Accept: header and other servlet parameters.
-     */
-    private static SessionConfig createSessionConfig(HttpServletRequest request, PrintWriter
resultWriter) {
+    private static SessionConfig createSessionConfig(RequestParameters param, PrintWriter
resultWriter) {
         SessionConfig.ResultDecorator resultPrefix = (AlgebricksAppendable app) -> {
             app.append("\t\"");
             app.append(ResultFields.RESULTS.str());
@@ -272,16 +277,14 @@ public class QueryServiceServlet extends HttpServlet {
             return app;
         };
 
-        final String formatstr = toLower(request.getParameter(Parameter.FORMAT.str()));
-        SessionConfig.OutputFormat format = getFormat(formatstr);
+        SessionConfig.OutputFormat format = getFormat(param.format);
         SessionConfig sessionConfig = new SessionConfig(resultWriter, format, resultPrefix,
resultPostfix);
         sessionConfig.set(SessionConfig.FORMAT_WRAPPER_ARRAY, true);
-        boolean indentJson = Boolean.parseBoolean(request.getParameter(Parameter.PRETTY.str()));
-        sessionConfig.set(SessionConfig.FORMAT_INDENT_JSON, indentJson);
+        sessionConfig.set(SessionConfig.FORMAT_INDENT_JSON, param.pretty);
         sessionConfig.set(SessionConfig.FORMAT_QUOTE_RECORD,
                 format != SessionConfig.OutputFormat.CLEAN_JSON && format != SessionConfig.OutputFormat.LOSSLESS_JSON);
-        sessionConfig.set(SessionConfig.FORMAT_CSV_HEADER,
-                format == SessionConfig.OutputFormat.CSV && "present".equals(getParameterValue(formatstr,
"header")));
+        sessionConfig.set(SessionConfig.FORMAT_CSV_HEADER, format == SessionConfig.OutputFormat.CSV
+                && "present".equals(getParameterValue(param.format, "header")));
         return sessionConfig;
     }
 
@@ -307,6 +310,12 @@ public class QueryServiceServlet extends HttpServlet {
         return requestId;
     }
 
+    private static void printClientContextID(PrintWriter pw, RequestParameters params) {
+        if (params.clientContextID != null && !params.clientContextID.isEmpty())
{
+            printField(pw, ResultFields.CLIENT_ID.str(), params.clientContextID);
+        }
+    }
+
     private static void printSignature(PrintWriter pw) {
         printField(pw, ResultFields.SIGNATURE.str(), "*");
     }
@@ -370,16 +379,9 @@ public class QueryServiceServlet extends HttpServlet {
     }
 
     @Override
-    protected void doPost(HttpServletRequest request, HttpServletResponse response)
-            throws ServletException, IOException {
-        String query = request.getParameter(Parameter.STATEMENT.str());
+    protected void doPost(HttpServletRequest request, HttpServletResponse response) throws
ServletException {
         try {
-            if (query == null) {
-                StringWriter sw = new StringWriter();
-                IOUtils.copy(request.getInputStream(), sw, StandardCharsets.UTF_8.name());
-                query = sw.toString();
-            }
-            handleRequest(request, response, query);
+            handleRequest(getRequestParameters(request), response);
         } catch (IOException e) {
             // Servlet methods should not throw exceptions
             // http://cwe.mitre.org/data/definitions/600.html
@@ -387,13 +389,46 @@ public class QueryServiceServlet extends HttpServlet {
         }
     }
 
-    private void handleRequest(HttpServletRequest request, HttpServletResponse response,
String query)
-            throws IOException {
+    private RequestParameters getRequestParameters(HttpServletRequest request) throws IOException
{
+        final String contentTypeParam = request.getContentType();
+        int sep = contentTypeParam.indexOf(';');
+        final String contentType = sep < 0 ? contentTypeParam.trim() : contentTypeParam.substring(0,
sep).trim();
+        RequestParameters param = new RequestParameters();
+        if (MediaType.JSON.str().equals(contentType)) {
+            try {
+                JSONObject jsonRequest = new JSONObject(getRequestBody(request));
+                param.statement = jsonRequest.getString(Parameter.STATEMENT.str());
+                param.format = toLower(jsonRequest.optString(Parameter.FORMAT.str()));
+                param.pretty = jsonRequest.optBoolean(Parameter.PRETTY.str());
+                param.clientContextID = jsonRequest.optString(Parameter.CLIENT_ID.str());
+            } catch (JSONException e) {
+                // if the JSON parsing fails, the statement is empty and we get an empty
statement error
+                GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, e.getMessage(), e);
+            }
+        } else {
+            param.statement = request.getParameter(Parameter.STATEMENT.str());
+            if (param.statement == null) {
+                param.statement = getRequestBody(request);
+            }
+            param.format = toLower(request.getParameter(Parameter.FORMAT.str()));
+            param.pretty = Boolean.parseBoolean(request.getParameter(Parameter.PRETTY.str()));
+            param.clientContextID = request.getParameter(Parameter.CLIENT_ID.str());
+        }
+        return param;
+    }
+
+    private static String getRequestBody(HttpServletRequest request) throws IOException {
+        StringWriter sw = new StringWriter();
+        IOUtils.copy(request.getInputStream(), sw, StandardCharsets.UTF_8.name());
+        return sw.toString();
+    }
+
+    private void handleRequest(RequestParameters param, HttpServletResponse response) throws
IOException {
         long elapsedStart = System.nanoTime();
         final StringWriter stringWriter = new StringWriter();
         final PrintWriter resultWriter = new PrintWriter(stringWriter);
 
-        SessionConfig sessionConfig = createSessionConfig(request, resultWriter);
+        SessionConfig sessionConfig = createSessionConfig(param, resultWriter);
         response.setCharacterEncoding("utf-8");
         response.setContentType(MediaType.JSON.str());
 
@@ -404,10 +439,11 @@ public class QueryServiceServlet extends HttpServlet {
 
         resultWriter.print("{\n");
         printRequestId(resultWriter);
+        printClientContextID(resultWriter, param);
         printSignature(resultWriter);
         printType(resultWriter, sessionConfig);
         try {
-            if (query == null || query.isEmpty()) {
+            if (param.statement == null || param.statement.isEmpty()) {
                 throw new AsterixException("Empty request, no statement provided");
             }
             IHyracksClientConnection hcc;
@@ -421,7 +457,7 @@ public class QueryServiceServlet extends HttpServlet {
                     context.setAttribute(HYRACKS_DATASET_ATTR, hds);
                 }
             }
-            IParser parser = compilationProvider.getParserFactory().createParser(query);
+            IParser parser = compilationProvider.getParserFactory().createParser(param.statement);
             List<Statement> aqlStatements = parser.parse();
             MetadataManager.INSTANCE.init();
             IStatementExecutor translator = statementExecutorFactory.create(aqlStatements,
sessionConfig,

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6762f03e/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java
b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java
index 1c48dbb..08a0342 100644
--- a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java
+++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java
@@ -60,10 +60,13 @@ import org.apache.http.HttpStatus;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpUriRequest;
 import org.apache.http.client.methods.RequestBuilder;
+import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.http.impl.client.StandardHttpRequestRetryHandler;
 import org.apache.http.util.EntityUtils;
+import org.json.JSONArray;
+import org.json.JSONException;
 import org.json.JSONObject;
 
 public class TestExecutor {
@@ -347,9 +350,7 @@ public class TestExecutor {
     }
 
     protected HttpResponse executeHttpRequest(HttpUriRequest method) throws Exception {
-        HttpClient client = HttpClients.custom()
-                .setRetryHandler(StandardHttpRequestRetryHandler.INSTANCE)
-                .build();
+        HttpClient client = HttpClients.custom().setRetryHandler(StandardHttpRequestRetryHandler.INSTANCE).build();
         try {
             return client.execute(method);
         } catch (Exception e) {
@@ -395,13 +396,14 @@ public class TestExecutor {
     }
 
     public InputStream executeQueryService(String str, String url) throws Exception {
-        return executeQueryService(str, OutputFormat.CLEAN_JSON, url, new ArrayList<>());
+        return executeQueryService(str, OutputFormat.CLEAN_JSON, url, new ArrayList<>(),
false);
     }
 
     public InputStream executeQueryService(String str, OutputFormat fmt, String url,
-            List<CompilationUnit.Parameter> params) throws Exception {
+            List<CompilationUnit.Parameter> params, boolean jsonEncoded) throws Exception
{
         setFormatParam(params, fmt);
-        HttpUriRequest method = constructPostMethod(str, url, "statement", true, params);
+        HttpUriRequest method = jsonEncoded ? constructPostMethodJson(str, url, "statement",
params)
+                : constructPostMethodUrl(str, url, "statement", params);
         // Set accepted output response type
         method.setHeader("Accept", OutputFormat.CLEAN_JSON.mimeType());
         HttpResponse response = executeHttpRequest(method);
@@ -409,19 +411,16 @@ public class TestExecutor {
     }
 
     protected void setFormatParam(List<CompilationUnit.Parameter> params, OutputFormat
fmt) {
-        boolean formatSet = false;
         for (CompilationUnit.Parameter param : params) {
             if ("format".equals(param.getName())) {
                 param.setValue(fmt.mimeType());
-                formatSet = true;
+                return;
             }
         }
-        if (!formatSet) {
-            CompilationUnit.Parameter formatParam = new CompilationUnit.Parameter();
-            formatParam.setName("format");
-            formatParam.setValue(fmt.mimeType());
-            params.add(formatParam);
-        }
+        CompilationUnit.Parameter formatParam = new CompilationUnit.Parameter();
+        formatParam.setName("format");
+        formatParam.setValue(fmt.mimeType());
+        params.add(formatParam);
     }
 
     private HttpUriRequest constructHttpMethod(String statement, String endpoint, String
stmtParam,
@@ -431,7 +430,8 @@ public class TestExecutor {
             return constructGetMethod(statement, endpoint, stmtParam, otherParams);
         } else {
             // Use POST for bigger ones to avoid 413 FULL_HEAD
-            return constructPostMethod(statement, endpoint, stmtParam, postStmtAsParam, otherParams);
+            String stmtParamName = (postStmtAsParam ? stmtParam : null);
+            return constructPostMethodUrl(statement, endpoint, stmtParamName, otherParams);
         }
     }
 
@@ -445,10 +445,10 @@ public class TestExecutor {
         return builder.build();
     }
 
-    protected HttpUriRequest constructPostMethod(String statement, String endpoint, String
stmtParam,
-            boolean postStmtAsParam, List<CompilationUnit.Parameter> otherParams) {
+    protected HttpUriRequest constructPostMethodUrl(String statement, String endpoint, String
stmtParam,
+            List<CompilationUnit.Parameter> otherParams) {
         RequestBuilder builder = RequestBuilder.post(endpoint);
-        if (postStmtAsParam) {
+        if (stmtParam != null) {
             for (CompilationUnit.Parameter param : otherParams) {
                 builder.addParameter(param.getName(), param.getValue());
             }
@@ -461,6 +461,26 @@ public class TestExecutor {
         return builder.build();
     }
 
+    protected HttpUriRequest constructPostMethodJson(String statement, String endpoint, String
stmtParam,
+            List<CompilationUnit.Parameter> otherParams) {
+        if (stmtParam == null) {
+            throw new NullPointerException("Statement parameter required.");
+        }
+        RequestBuilder builder = RequestBuilder.post(endpoint);
+        JSONObject content = new JSONObject();
+        try {
+            content.put(stmtParam, statement);
+            for (CompilationUnit.Parameter param : otherParams) {
+                content.put(param.getName(), param.getValue());
+            }
+        } catch (JSONException e) {
+            throw new IllegalArgumentException("Request object construction failed.", e);
+        }
+        builder.setEntity(new StringEntity(content.toString(), ContentType.APPLICATION_JSON));
+        builder.setCharset(StandardCharsets.UTF_8);
+        return builder.build();
+    }
+
     public InputStream executeClusterStateQuery(OutputFormat fmt, String url) throws Exception
{
         HttpUriRequest request = RequestBuilder.get(url).setHeader("Accept", fmt.mimeType()).build();
 
@@ -485,9 +505,7 @@ public class TestExecutor {
         // Create a method instance.
         HttpUriRequest request = RequestBuilder.post(url)
                 .addParameter("mode", defer ? "asynchronous-deferred" : "asynchronous")
-                .setEntity(new StringEntity(str, StandardCharsets.UTF_8))
-                .setHeader("Accept", fmt.mimeType())
-                .build();
+                .setEntity(new StringEntity(str, StandardCharsets.UTF_8)).setHeader("Accept",
fmt.mimeType()).build();
 
         HttpResponse response = executeAndCheckHttpRequest(request);
         InputStream resultStream = response.getEntity().getContent();
@@ -664,7 +682,7 @@ public class TestExecutor {
                 } else {
                     if (ctx.getType().equalsIgnoreCase("query")) {
                         resultStream = executeQueryService(statement, fmt, getEndpoint(Servlets.QUERY_SERVICE),
-                                cUnit.getParameter());
+                                cUnit.getParameter(), true);
                         resultStream = ResultExtractor.extract(resultStream);
                     } else if (ctx.getType().equalsIgnoreCase("async")) {
                         resultStream = executeAnyAQLAsync(statement, false, fmt, getEndpoint(Servlets.SQLPP));
@@ -968,20 +986,18 @@ public class TestExecutor {
     public void cleanup(String testCase, List<String> badtestcases) throws Exception
{
         try {
             ArrayList<String> toBeDropped = new ArrayList<>();
-            InputStream resultStream = null;
-            OutputFormat fmt = OutputFormat.ADM;
-            resultStream = executeQueryService("select dv.DataverseName from Metadata.`Dataverse`
as dv;", fmt,
-                    getEndpoint(Servlets.QUERY_SERVICE), new ArrayList<>());
+            InputStream resultStream = executeQueryService("select dv.DataverseName from
Metadata.`Dataverse` as dv;",
+                    getEndpoint(Servlets.QUERY_SERVICE));
             resultStream = ResultExtractor.extract(resultStream);
-            BufferedReader reader = new BufferedReader(new InputStreamReader(resultStream));
-            String dataverse = reader.readLine();
-            while (dataverse != null) {
-                JSONObject json = new JSONObject(dataverse);
+            StringWriter sw = new StringWriter();
+            IOUtils.copy(resultStream, sw, StandardCharsets.UTF_8.name());
+            JSONArray result = new JSONArray(sw.toString());
+            for (int i = 0; i < result.length(); ++i) {
+                JSONObject json = result.getJSONObject(i);
                 String dvName = json.getString("DataverseName");
                 if (!dvName.equals("Metadata") && !dvName.equals("Default")) {
                     toBeDropped.add(dvName);
                 }
-                dataverse = reader.readLine();
             }
             if (!toBeDropped.isEmpty()) {
                 badtestcases.add(testCase);
@@ -998,6 +1014,7 @@ public class TestExecutor {
             }
         } catch (Throwable th) {
             th.printStackTrace();
+            throw th;
         }
     }
 }


Mime
View raw message