asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ti...@apache.org
Subject asterixdb git commit: QueryService API updates
Date Thu, 07 Jul 2016 22:40:18 GMT
Repository: asterixdb
Updated Branches:
  refs/heads/master 2da62dc4c -> 2ae8af7c9


QueryService API updates

- API returns non-JSON results (ADM/CSV) as arrays of (escaped) strings
- fix encoding and content-length of response
- run SQL++ query tests through QueryService API
- fix tests/expected errors
- correct execution times in the case of errors
- re-structure printing of CSV headers
- improve parameter handling
- small API cleanup

Change-Id: Ie67ad4ea31699400726c8c026c4a91edc698f2b5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/896
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Yingyi Bu <buyingyi@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/2ae8af7c
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/2ae8af7c
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/2ae8af7c

Branch: refs/heads/master
Commit: 2ae8af7c97225df366a9ed02dc518fbb93da5d28
Parents: 2da62dc
Author: Till Westmann <tillw@apache.org>
Authored: Thu Jul 7 14:06:50 2016 -0700
Committer: Till Westmann <tillw@apache.org>
Committed: Thu Jul 7 15:39:24 2016 -0700

----------------------------------------------------------------------
 .../asterix/api/common/SessionConfig.java       |   7 +-
 .../asterix/api/http/servlet/APIServlet.java    |  38 +--
 .../asterix/api/http/servlet/JSONUtil.java      | 136 --------
 .../api/http/servlet/QueryResultAPIServlet.java |   2 +-
 .../api/http/servlet/QueryServiceServlet.java   | 330 ++++++++++++-------
 .../asterix/aql/translator/QueryTranslator.java |  12 +-
 .../org/apache/asterix/result/ResultUtils.java  |  80 ++---
 .../queries_sqlpp/records/RecordsQueries.xml    |   6 +-
 .../resources/runtimets/testsuite_sqlpp.xml     |  43 ++-
 .../apache/asterix/common/utils/JSONUtil.java   | 149 +++++++++
 .../asterix/test/aql/ResultExtractor.java       | 171 ++++++++++
 .../apache/asterix/test/aql/TestExecutor.java   |  88 +++--
 12 files changed, 674 insertions(+), 388 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2ae8af7c/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java
index 7bfc55c..c09b424 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java
@@ -95,7 +95,12 @@ public class SessionConfig {
     /**
      * Format flag: indent JSON results.
      */
-    public static final String INDENT_JSON = "indent-json";
+    public static final String FORMAT_INDENT_JSON = "indent-json";
+
+    /**
+     * Format flag: quote records in the results array.
+     */
+    public static final String FORMAT_QUOTE_RECORD = "quote-record";
 
     public interface ResultDecorator {
         PrintWriter print(PrintWriter pw);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2ae8af7c/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java
index ba8644a..7a47ca9 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java
@@ -27,6 +27,7 @@ import java.io.OutputStream;
 import java.io.PrintWriter;
 import java.util.List;
 import java.util.logging.Level;
+import java.util.logging.Logger;
 
 import javax.imageio.ImageIO;
 import javax.servlet.ServletContext;
@@ -57,53 +58,34 @@ import org.apache.hyracks.client.dataset.HyracksDataset;
 public class APIServlet extends HttpServlet {
     private static final long serialVersionUID = 1L;
 
-    private static final String HYRACKS_CONNECTION_ATTR = "org.apache.asterix.HYRACKS_CONNECTION";
+    private static final Logger LOGGER = Logger.getLogger(APIServlet.class.getName());
 
+    private static final String HYRACKS_CONNECTION_ATTR = "org.apache.asterix.HYRACKS_CONNECTION";
     private static final String HYRACKS_DATASET_ATTR = "org.apache.asterix.HYRACKS_DATASET";
 
     private final ILangCompilationProvider aqlCompilationProvider;
-    private final IParserFactory aqlParserFactory;
     private final ILangCompilationProvider sqlppCompilationProvider;
-    private final IParserFactory sqlppParserFactory;
 
     public APIServlet() {
         this.aqlCompilationProvider = new AqlCompilationProvider();
-        this.aqlParserFactory = aqlCompilationProvider.getParserFactory();
-
         this.sqlppCompilationProvider = new SqlppCompilationProvider();
-        this.sqlppParserFactory = sqlppCompilationProvider.getParserFactory();
     }
 
     @Override
     public void doPost(HttpServletRequest request, HttpServletResponse response) throws IOException {
         // Query language
-        ILangCompilationProvider compilationProvider;
-        IParserFactory parserFactory;
-        String lang = request.getParameter("query-language");
-        if (lang.equals("AQL")) {
-            // Uses AQL compiler.
-            compilationProvider = aqlCompilationProvider;
-            parserFactory = aqlParserFactory;
-        } else {
-            // Uses SQL++ compiler.
-            compilationProvider = sqlppCompilationProvider;
-            parserFactory = sqlppParserFactory;
-        }
+        ILangCompilationProvider compilationProvider = "AQL".equals(request.getParameter("query-language"))
+                ? aqlCompilationProvider : sqlppCompilationProvider;
+        IParserFactory parserFactory = compilationProvider.getParserFactory();
 
         // Output format.
         OutputFormat format;
         boolean csv_and_header = false;
         String output = request.getParameter("output-format");
-        if (output.equals("ADM")) {
-            format = OutputFormat.ADM;
-        } else if (output.equals("CSV")) {
-            format = OutputFormat.CSV;
-        } else if (output.equals("CSV-Header")) {
-            format = OutputFormat.CSV;
-            csv_and_header = true;
-        } else if (output.equals("LOSSLESS_JSON")) {
-            format = OutputFormat.LOSSLESS_JSON;
-        } else {
+        try {
+            format = OutputFormat.valueOf(output);
+        } catch (IllegalArgumentException e) {
+            LOGGER.info(output + ": unsupported output-format, using " + OutputFormat.CLEAN_JSON + " instead");
             // Default output format
             format = OutputFormat.CLEAN_JSON;
         }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2ae8af7c/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/JSONUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/JSONUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/JSONUtil.java
deleted file mode 100644
index d69c3c1..0000000
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/JSONUtil.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.api.http.servlet;
-
-import java.util.Iterator;
-
-import org.json.JSONArray;
-import org.json.JSONException;
-import org.json.JSONObject;
-
-public class JSONUtil {
-
-    static final String INDENT = "\t";
-
-    public static String indent(String str) {
-        return indent(str, 0);
-    }
-
-    public static String indent(String str, int initialIndent) {
-        try {
-            return append(new StringBuilder(), new JSONObject(str), initialIndent).toString();
-        } catch (JSONException e) {
-            return str;
-        }
-    }
-
-    static StringBuilder append(StringBuilder sb, Object o, int indent) throws JSONException {
-        if (o instanceof JSONObject) {
-            return append(sb, (JSONObject) o, indent);
-        } else if (o instanceof JSONArray) {
-            return append(sb, (JSONArray) o, indent);
-        } else if (o instanceof String) {
-            return quote(sb, (String) o);
-        } else if (JSONObject.NULL.equals(o) || o instanceof Number || o instanceof Boolean) {
-            return sb.append(String.valueOf(o));
-        }
-        throw new UnsupportedOperationException(o.getClass().getSimpleName());
-    }
-
-    static StringBuilder append(StringBuilder sb, JSONObject jobj, int indent) throws JSONException {
-        sb = sb.append("{\n");
-        boolean first = true;
-        for (Iterator it = jobj.keys(); it.hasNext();) {
-            final String key = (String) it.next();
-            if (first) {
-                first = false;
-            } else {
-                sb = sb.append(",\n");
-            }
-            sb = indent(sb, indent + 1);
-            sb = quote(sb, key);
-            sb = sb.append(": ");
-            sb = append(sb, jobj.get(key), indent + 1);
-        }
-        sb = sb.append("\n");
-        return indent(sb, indent).append("}");
-    }
-
-    static StringBuilder append(StringBuilder sb, JSONArray jarr, int indent) throws JSONException {
-        sb = sb.append("[\n");
-        for (int i = 0; i < jarr.length(); ++i) {
-            if (i > 0) {
-                sb = sb.append(",\n");
-            }
-            sb = indent(sb, indent + 1);
-            sb = append(sb, jarr.get(i), indent + 1);
-        }
-        sb = sb.append("\n");
-        return indent(sb, indent).append("]");
-    }
-
-    static StringBuilder quote(StringBuilder sb, String str) {
-        return sb.append('"').append(str).append('"');
-    }
-
-    static StringBuilder indent(StringBuilder sb, int indent) {
-        while (indent > 0) {
-            sb.append(INDENT);
-            --indent;
-        }
-        return sb;
-    }
-
-    public static String escape(String str) {
-        StringBuilder result = new StringBuilder();
-        for (int i = 0; i < str.length(); ++i) {
-            appendEsc(result, str.charAt(i));
-        }
-        return result.toString();
-    }
-
-    public static StringBuilder appendEsc(StringBuilder sb, char c) {
-        switch (c) {
-            case '"':
-                return sb.append("\\\"");
-            case '\\':
-                return sb.append("\\\\");
-            case '/':
-                return sb.append("\\/");
-            case '\b':
-                return sb.append("\\b");
-            case '\n':
-                return sb.append("\\n");
-            case '\f':
-                return sb.append("\\f");
-            case '\r':
-                return sb.append("\\r");
-            case '\t':
-                return sb.append("\\t");
-            default:
-                return sb.append(c);
-        }
-    }
-
-    public static void main(String[] args) {
-        String json = args.length > 0 ? args[0] : "{\"a\":[\"b\",\"c\\\nd\"],\"e\":42}";
-        System.out.println(json);
-        System.out.println(indent(json));
-    }
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2ae8af7c/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java
index d150e5d..3198759 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java
@@ -88,7 +88,7 @@ public class QueryResultAPIServlet extends HttpServlet {
             // originally determined there. Need to save this value on
             // some object that we can obtain here.
             SessionConfig sessionConfig = RESTAPIServlet.initResponse(request, response);
-            ResultUtils.displayResults(resultReader, sessionConfig, new ResultUtils.Stats());
+            ResultUtils.displayResults(resultReader, sessionConfig, new ResultUtils.Stats(), null);
 
         } catch (Exception e) {
             out.println(e.getMessage());

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2ae8af7c/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java
index c62389f..8fa09a4 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java
@@ -37,6 +37,7 @@ import org.apache.asterix.api.common.SessionConfig;
 import org.apache.asterix.aql.translator.QueryTranslator;
 import org.apache.asterix.common.config.GlobalConfig;
 import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.utils.JSONUtil;
 import org.apache.asterix.compiler.provider.ILangCompilationProvider;
 import org.apache.asterix.compiler.provider.SqlppCompilationProvider;
 import org.apache.asterix.lang.aql.parser.TokenMgrError;
@@ -55,24 +56,21 @@ public class QueryServiceServlet extends HttpServlet {
 
     private static final Logger LOGGER = Logger.getLogger(QueryServiceServlet.class.getName());
 
-    public static final String HYRACKS_CONNECTION_ATTR = "org.apache.asterix.HYRACKS_CONNECTION";
-    public static final String HYRACKS_DATASET_ATTR = "org.apache.asterix.HYRACKS_DATASET";
+    private static final String HYRACKS_CONNECTION_ATTR = "org.apache.asterix.HYRACKS_CONNECTION";
+    private static final String HYRACKS_DATASET_ATTR = "org.apache.asterix.HYRACKS_DATASET";
+
+    private transient final ILangCompilationProvider compilationProvider = new SqlppCompilationProvider();
 
     public enum Parameter {
         // Standard
-        statement,
-        format,
+        STATEMENT("statement"),
+        FORMAT("format"),
         // Asterix
-        header,
-        indent
-    }
-
-    public enum Header {
-        Accept("Accept");
+        INDENT("indent");
 
         private final String str;
 
-        Header(String str) {
+        Parameter(String str) {
             this.str = str;
         }
 
@@ -81,9 +79,10 @@ public class QueryServiceServlet extends HttpServlet {
         }
     }
 
-    public enum MediaType {
+    private enum MediaType {
         CSV("text/csv"),
-        JSON("application/json");
+        JSON("application/json"),
+        ADM("application/x-adm");
 
         private final String str;
 
@@ -96,36 +95,92 @@ public class QueryServiceServlet extends HttpServlet {
         }
     }
 
-    public enum ResultFields {
-        requestID,
-        signature,
-        status,
-        results,
-        errors,
-        metrics
+    private enum Attribute {
+        HEADER("header"),
+        LOSSLESS("lossless");
+
+        private final String str;
+
+        Attribute(String str) {
+            this.str = str;
+        }
+
+        public String str() {
+            return str;
+        }
     }
 
-    public enum ResultStatus {
-        success,
-        timeout,
-        errors,
-        fatal
+    private enum ResultFields {
+        REQUEST_ID("requestID"),
+        SIGNATURE("signature"),
+        TYPE("type"),
+        STATUS("status"),
+        RESULTS("results"),
+        ERRORS("errors"),
+        METRICS("metrics");
+
+        private final String str;
+
+        ResultFields(String str) {
+            this.str = str;
+        }
+
+        public String str() {
+            return str;
+        }
     }
 
-    public enum ErrorField {
-        code,
-        msg,
-        stack
+    private enum ResultStatus {
+        SUCCESS("success"),
+        TIMEOUT("timeout"),
+        ERRORS("errors"),
+        FATAL("fatal");
+
+        private final String str;
+
+        ResultStatus(String str) {
+            this.str = str;
+        }
+
+        public String str() {
+            return str;
+        }
     }
 
-    public enum Metrics {
-        elapsedTime,
-        executionTime,
-        resultCount,
-        resultSize
+    private enum ErrorField {
+        CODE("code"),
+        MSG("msg"),
+        STACK("stack");
+
+        private final String str;
+
+        ErrorField(String str) {
+            this.str = str;
+        }
+
+        public String str() {
+            return str;
+        }
     }
 
-    public enum TimeUnit {
+    private enum Metrics {
+        ELAPSED_TIME("elapsedTime"),
+        EXECUTION_TIME("executionTime"),
+        RESULT_COUNT("resultCount"),
+        RESULT_SIZE("resultSize");
+
+        private final String str;
+
+        Metrics(String str) {
+            this.str = str;
+        }
+
+        public String str() {
+            return str;
+        }
+    }
+
+    enum TimeUnit {
         SEC("s", 9),
         MILLI("ms", 6),
         MICRO("┬Ás", 3),
@@ -153,18 +208,33 @@ public class QueryServiceServlet extends HttpServlet {
         }
     }
 
-    private final ILangCompilationProvider compilationProvider = new SqlppCompilationProvider();
-
-    static SessionConfig.OutputFormat getFormat(HttpServletRequest request) {
-        // First check the "format" parameter.
-        String format = request.getParameter(Parameter.format.name());
-        if (format != null && format.equals("CSV")) {
-            return SessionConfig.OutputFormat.CSV;
+    private static String getParameterValue(String content, String attribute) {
+        int sc = content.indexOf(';');
+        if (sc < 0) {
+            return null;
         }
-        // Second check the Accept: HTTP header.
-        String accept = request.getHeader(Header.Accept.str());
-        if (accept != null && accept.contains(MediaType.CSV.str())) {
-            return SessionConfig.OutputFormat.CSV;
+        int eq = content.indexOf('=', sc + 1);
+        if (eq < 0) {
+            return null;
+        }
+        if (content.substring(sc + 1, eq).trim().equalsIgnoreCase(attribute)) {
+            return content.substring(eq + 1).trim().toLowerCase();
+        }
+        return null;
+    }
+
+    private static SessionConfig.OutputFormat getFormat(String format) {
+        if (format != null) {
+            if (format.startsWith(MediaType.CSV.str())) {
+                return SessionConfig.OutputFormat.CSV;
+            }
+            if (format.equals(MediaType.ADM.str())) {
+                return SessionConfig.OutputFormat.ADM;
+            }
+            if (format.startsWith(MediaType.JSON.str())) {
+                return Boolean.parseBoolean(getParameterValue(format, "lossless"))
+                        ? SessionConfig.OutputFormat.LOSSLESS_JSON : SessionConfig.OutputFormat.CLEAN_JSON;
+            }
         }
         return SessionConfig.OutputFormat.CLEAN_JSON;
     }
@@ -173,60 +243,37 @@ public class QueryServiceServlet extends HttpServlet {
      * Construct a SessionConfig with the appropriate output writer and
      * output-format based on the Accept: header and other servlet parameters.
      */
-    static SessionConfig createSessionConfig(HttpServletRequest request, PrintWriter resultWriter) {
-        SessionConfig.ResultDecorator resultPrefix = new SessionConfig.ResultDecorator() {
-            @Override
-            public PrintWriter print(PrintWriter pw) {
-                pw.print("\t\"");
-                pw.print(ResultFields.results.name());
-                pw.print("\": ");
-                return pw;
-            }
+    private static SessionConfig createSessionConfig(HttpServletRequest request, PrintWriter resultWriter) {
+        SessionConfig.ResultDecorator resultPrefix = (PrintWriter pw) -> {
+            pw.print("\t\"");
+            pw.print(ResultFields.RESULTS.str());
+            pw.print("\": ");
+            return pw;
         };
 
-        SessionConfig.ResultDecorator resultPostfix = new SessionConfig.ResultDecorator() {
-            @Override
-            public PrintWriter print(PrintWriter pw) {
-                pw.print("\t,\n");
-                return pw;
-            }
+        SessionConfig.ResultDecorator resultPostfix = (PrintWriter pw) -> {
+            pw.print("\t,\n");
+            return pw;
         };
 
-        SessionConfig.OutputFormat format = getFormat(request);
+        String formatstr = request.getParameter(Parameter.FORMAT.str()).toLowerCase();
+        SessionConfig.OutputFormat format = getFormat(formatstr);
         SessionConfig sessionConfig = new SessionConfig(resultWriter, format, resultPrefix, resultPostfix);
-        sessionConfig.set(SessionConfig.FORMAT_WRAPPER_ARRAY, format == SessionConfig.OutputFormat.CLEAN_JSON);
-        boolean indentJson = Boolean.parseBoolean(request.getParameter(Parameter.indent.name()));
-        sessionConfig.set(SessionConfig.INDENT_JSON, indentJson);
-
-        if (format == SessionConfig.OutputFormat.CSV && ("present".equals(request.getParameter(Parameter.header.name()))
-                || request.getHeader(Header.Accept.str()).contains("header=present"))) {
-            sessionConfig.set(SessionConfig.FORMAT_CSV_HEADER, true);
-        }
+        sessionConfig.set(SessionConfig.FORMAT_WRAPPER_ARRAY, true);
+        boolean indentJson = Boolean.parseBoolean(request.getParameter(Parameter.INDENT.str()));
+        sessionConfig.set(SessionConfig.FORMAT_INDENT_JSON, indentJson);
+        sessionConfig.set(SessionConfig.FORMAT_QUOTE_RECORD,
+                format != SessionConfig.OutputFormat.CLEAN_JSON && format != SessionConfig.OutputFormat.LOSSLESS_JSON);
+        sessionConfig.set(SessionConfig.FORMAT_CSV_HEADER,
+                format == SessionConfig.OutputFormat.CSV && "present".equals(getParameterValue(formatstr, "header")));
         return sessionConfig;
     }
 
-    /**
-     * Initialize the Content-Type of the response based on a SessionConfig.
-     */
-    static void initResponse(HttpServletResponse response, SessionConfig sessionConfig) throws IOException {
-        response.setCharacterEncoding("utf-8");
-        switch (sessionConfig.fmt()) {
-            case CLEAN_JSON:
-                response.setContentType(MediaType.JSON.str());
-                break;
-            case CSV:
-                String contentType = MediaType.CSV.str() + "; header="
-                        + (sessionConfig.is(SessionConfig.FORMAT_CSV_HEADER) ? "present" : "absent");
-                response.setContentType(contentType);
-                break;
-        }
-    }
-
-    static void printField(PrintWriter pw, String name, String value) {
+    private static void printField(PrintWriter pw, String name, String value) {
         printField(pw, name, value, true);
     }
 
-    static void printField(PrintWriter pw, String name, String value, boolean comma) {
+    private static void printField(PrintWriter pw, String name, String value, boolean comma) {
         pw.print("\t\"");
         pw.print(name);
         pw.print("\": \"");
@@ -238,89 +285,120 @@ public class QueryServiceServlet extends HttpServlet {
         pw.print('\n');
     }
 
-    static UUID printRequestId(PrintWriter pw) {
+    private static UUID printRequestId(PrintWriter pw) {
         UUID requestId = UUID.randomUUID();
-        printField(pw, ResultFields.requestID.name(), requestId.toString());
+        printField(pw, ResultFields.REQUEST_ID.str(), requestId.toString());
         return requestId;
     }
 
-    static void printSignature(PrintWriter pw) {
-        printField(pw, ResultFields.signature.name(), "*");
+    private static void printSignature(PrintWriter pw) {
+        printField(pw, ResultFields.SIGNATURE.str(), "*");
+    }
+
+    private static void printType(PrintWriter pw, SessionConfig sessionConfig) {
+        switch (sessionConfig.fmt()) {
+            case ADM:
+                printField(pw, ResultFields.TYPE.str(), MediaType.ADM.str());
+                break;
+            case CSV:
+                String contentType = MediaType.CSV.str() + "; header="
+                        + (sessionConfig.is(SessionConfig.FORMAT_CSV_HEADER) ? "present" : "absent");
+                printField(pw, ResultFields.TYPE.str(), contentType);
+                break;
+            default:
+                break;
+        }
     }
 
-    static void printStatus(PrintWriter pw, ResultStatus rs) {
-        printField(pw, ResultFields.status.name(), rs.name());
+    private static void printStatus(PrintWriter pw, ResultStatus rs) {
+        printField(pw, ResultFields.STATUS.str(), rs.str());
     }
 
-    static void printError(PrintWriter pw, Throwable e) {
+    private static void printError(PrintWriter pw, Throwable e) {
         final boolean addStack = false;
         pw.print("\t\"");
-        pw.print(ResultFields.errors.name());
+        pw.print(ResultFields.ERRORS.str());
         pw.print("\": [{ \n");
-        printField(pw, ErrorField.code.name(), "1");
-        printField(pw, ErrorField.msg.name(), JSONUtil.escape(e.getMessage()), addStack);
+        printField(pw, ErrorField.CODE.str(), "1");
+        final String msg = e.getMessage();
+        printField(pw, ErrorField.MSG.str(), JSONUtil.escape(msg != null ? msg : e.getClass().getSimpleName()),
+                addStack);
         if (addStack) {
             StringWriter sw = new StringWriter();
             PrintWriter stackWriter = new PrintWriter(sw);
-            e.printStackTrace(stackWriter);
+            LOGGER.info(stackWriter.toString());
             stackWriter.close();
-            printField(pw, ErrorField.stack.name(), JSONUtil.escape(sw.toString()), false);
+            printField(pw, ErrorField.STACK.str(), JSONUtil.escape(sw.toString()), false);
         }
         pw.print("\t}],\n");
     }
 
-    static void printMetrics(PrintWriter pw, long elapsedTime, long executionTime, long resultCount, long resultSize) {
+    private static void printMetrics(PrintWriter pw, long elapsedTime, long executionTime, long resultCount,
+            long resultSize) {
         pw.print("\t\"");
-        pw.print(ResultFields.metrics.name());
+        pw.print(ResultFields.METRICS.str());
         pw.print("\": {\n");
         pw.print("\t");
-        printField(pw, Metrics.elapsedTime.name(), TimeUnit.formatNanos(elapsedTime));
+        printField(pw, Metrics.ELAPSED_TIME.str(), TimeUnit.formatNanos(elapsedTime));
         pw.print("\t");
-        printField(pw, Metrics.executionTime.name(), TimeUnit.formatNanos(executionTime));
+        printField(pw, Metrics.EXECUTION_TIME.str(), TimeUnit.formatNanos(executionTime));
         pw.print("\t");
-        printField(pw, Metrics.resultCount.name(), String.valueOf(resultCount));
+        printField(pw, Metrics.RESULT_COUNT.str(), String.valueOf(resultCount));
         pw.print("\t");
-        printField(pw, Metrics.resultSize.name(), String.valueOf(resultSize), false);
+        printField(pw, Metrics.RESULT_SIZE.str(), String.valueOf(resultSize), false);
         pw.print("\t}\n");
     }
 
     @Override
     protected void doPost(HttpServletRequest request, HttpServletResponse response)
             throws ServletException, IOException {
-        String query = request.getParameter(Parameter.statement.name());
-        if (query == null) {
-            StringWriter sw = new StringWriter();
-            IOUtils.copy(request.getInputStream(), sw, StandardCharsets.UTF_8.name());
-            query = sw.toString();
+        String query = request.getParameter(Parameter.STATEMENT.str());
+        try {
+            if (query == null) {
+                StringWriter sw = new StringWriter();
+                IOUtils.copy(request.getInputStream(), sw, StandardCharsets.UTF_8.name());
+                query = sw.toString();
+            }
+            handleRequest(request, response, query);
+        } catch (IOException e) {
+            // Servlet methods should not throw exceptions
+            // http://cwe.mitre.org/data/definitions/600.html
+            GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, e.getMessage(), e);
         }
-        handleRequest(request, response, query);
     }
 
     @Override
     public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException {
-        String query = request.getParameter(Parameter.statement.name());
-        handleRequest(request, response, query);
+        String query = request.getParameter(Parameter.STATEMENT.str());
+        try {
+            handleRequest(request, response, query);
+        } catch (IOException e) {
+            // Servlet methods should not throw exceptions
+            // http://cwe.mitre.org/data/definitions/600.html
+            GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, e.getMessage(), e);
+        }
     }
 
-    public void handleRequest(HttpServletRequest request, HttpServletResponse response, String query)
+    private void handleRequest(HttpServletRequest request, HttpServletResponse response, String query)
             throws IOException {
         long elapsedStart = System.nanoTime();
 
-        query = query + ";";
-
         final StringWriter stringWriter = new StringWriter();
         final PrintWriter resultWriter = new PrintWriter(stringWriter);
 
         SessionConfig sessionConfig = createSessionConfig(request, resultWriter);
-        initResponse(response, sessionConfig);
+        response.setCharacterEncoding("utf-8");
+        response.setContentType(MediaType.JSON.str());
 
         int respCode = HttpServletResponse.SC_OK;
         ResultUtils.Stats stats = new ResultUtils.Stats();
-        long execStart = 0, execEnd = 0;
+        long execStart = 0;
+        long execEnd = -1;
 
         resultWriter.print("{\n");
         UUID requestId = printRequestId(resultWriter);
         printSignature(resultWriter);
+        printType(resultWriter, sessionConfig);
         try {
             IHyracksClientConnection hcc;
             IHyracksDataset hds;
@@ -340,17 +418,21 @@ public class QueryServiceServlet extends HttpServlet {
             execStart = System.nanoTime();
             translator.compileAndExecute(hcc, hds, QueryTranslator.ResultDelivery.SYNC, stats);
             execEnd = System.nanoTime();
-            printStatus(resultWriter, ResultStatus.success);
+            printStatus(resultWriter, ResultStatus.SUCCESS);
         } catch (AsterixException | TokenMgrError | org.apache.asterix.aqlplus.parser.TokenMgrError pe) {
             GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, pe.getMessage(), pe);
             printError(resultWriter, pe);
-            printStatus(resultWriter, ResultStatus.fatal);
+            printStatus(resultWriter, ResultStatus.FATAL);
             respCode = HttpServletResponse.SC_BAD_REQUEST;
         } catch (Exception e) {
             GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, e.getMessage(), e);
             printError(resultWriter, e);
-            printStatus(resultWriter, ResultStatus.fatal);
+            printStatus(resultWriter, ResultStatus.FATAL);
             respCode = HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
+        } finally {
+            if (execEnd == -1) {
+                execEnd = System.nanoTime();
+            }
         }
         printMetrics(resultWriter, System.nanoTime() - elapsedStart, execEnd - execStart, stats.count, stats.size);
         resultWriter.print("}\n");
@@ -358,7 +440,6 @@ public class QueryServiceServlet extends HttpServlet {
         String result = stringWriter.toString();
 
         GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, result);
-        //result = JSONUtil.indent(result);
 
         response.getWriter().print(result);
         if (response.getWriter().checkError()) {
@@ -366,5 +447,4 @@ public class QueryServiceServlet extends HttpServlet {
         }
         response.setStatus(respCode);
     }
-
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2ae8af7c/asterixdb/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
index 7656006..95afc5b 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
@@ -39,7 +39,6 @@ import java.util.logging.Logger;
 
 import org.apache.asterix.api.common.APIFramework;
 import org.apache.asterix.api.common.SessionConfig;
-import org.apache.asterix.api.common.SessionConfig.OutputFormat;
 import org.apache.asterix.app.external.ExternalIndexingOperations;
 import org.apache.asterix.app.external.FeedJoint;
 import org.apache.asterix.app.external.FeedLifecycleListener;
@@ -2562,15 +2561,8 @@ public class QueryTranslator extends AbstractLangTranslator {
                         hcc.waitForCompletion(jobId);
                         ResultReader resultReader = new ResultReader(hcc, hdc);
                         resultReader.open(jobId, metadataProvider.getResultSetId());
-
-                        // In this case (the normal case), we don't use the
-                        // "response" JSONObject - just stream the results
-                        // to the "out" PrintWriter
-                        if (sessionConfig.fmt() == OutputFormat.CSV
-                                && sessionConfig.is(SessionConfig.FORMAT_CSV_HEADER)) {
-                            ResultUtils.displayCSVHeader(metadataProvider.findOutputRecordType(), sessionConfig);
-                        }
-                        ResultUtils.displayResults(resultReader, sessionConfig, stats);
+                        ResultUtils.displayResults(resultReader, sessionConfig, stats,
+                                metadataProvider.findOutputRecordType());
                         break;
                     case ASYNC_DEFERRED:
                         handle = new JSONArray();

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2ae8af7c/asterixdb/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java
index b140bef..73dd706 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java
@@ -34,8 +34,7 @@ import java.util.regex.Pattern;
 import org.apache.asterix.api.common.SessionConfig;
 import org.apache.asterix.api.common.SessionConfig.OutputFormat;
 import org.apache.asterix.api.http.servlet.APIServlet;
-import org.apache.asterix.api.http.servlet.JSONUtil;
-import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.utils.JSONUtil;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.http.ParseException;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -74,35 +73,25 @@ public class ResultUtils {
         return s;
     }
 
-    public static void displayCSVHeader(ARecordType recordType, SessionConfig conf) throws AsterixException {
-        if (recordType == null) {
-            throw new AsterixException("Cannot output CSV with header without specifying output-record-type");
-        }
-        // If HTML-ifying, we have to output this here before the header -
-        // pretty ugly
-        if (conf.is(SessionConfig.FORMAT_HTML)) {
-            conf.out().println("<h4>Results:</h4>");
-            conf.out().println("<pre>");
-        }
-
+    private static void printCSVHeader(ARecordType recordType, PrintWriter out) {
         String[] fieldNames = recordType.getFieldNames();
         boolean notfirst = false;
         for (String name : fieldNames) {
             if (notfirst) {
-                conf.out().print(',');
+                out.print(',');
             }
             notfirst = true;
-            conf.out().print('"');
-            conf.out().print(name.replace("\"", "\"\""));
-            conf.out().print('"');
+            out.print('"');
+            out.print(name.replace("\"", "\"\""));
+            out.print('"');
         }
-        conf.out().print("\r\n");
+        out.print("\r\n");
     }
 
     public static FrameManager resultDisplayFrameMgr = new FrameManager(ResultReader.FRAME_SIZE);
 
-    public static void displayResults(ResultReader resultReader, SessionConfig conf, Stats stats)
-            throws HyracksDataException {
+    public static void displayResults(ResultReader resultReader, SessionConfig conf, Stats stats,
+            ARecordType recordType) throws HyracksDataException {
         // Whether we are wrapping the output sequence in an array
         boolean wrap_array = false;
         // Whether this is the first instance being output
@@ -110,31 +99,37 @@ public class ResultUtils {
 
         // If we're outputting CSV with a header, the HTML header was already
         // output by displayCSVHeader(), so skip it here
-        if (conf.is(SessionConfig.FORMAT_HTML)
-                && !(conf.fmt() == OutputFormat.CSV && conf.is(SessionConfig.FORMAT_CSV_HEADER))) {
+        if (conf.is(SessionConfig.FORMAT_HTML)) {
             conf.out().println("<h4>Results:</h4>");
             conf.out().println("<pre>");
         }
 
         conf.resultPrefix(conf.out());
 
-        switch (conf.fmt()) {
-            case LOSSLESS_JSON:
-            case CLEAN_JSON:
-            case ADM:
-                if (conf.is(SessionConfig.FORMAT_WRAPPER_ARRAY)) {
-                    // Conveniently, LOSSLESS_JSON and ADM have the same syntax for an
-                    // "ordered list", and our representation of the result of a
-                    // statement is an ordered list of instances.
-                    conf.out().print("[ ");
-                    wrap_array = true;
-                }
-                break;
-            default:
-                break;
+        if (conf.is(SessionConfig.FORMAT_WRAPPER_ARRAY)) {
+            conf.out().print("[ ");
+            wrap_array = true;
         }
 
-        final boolean indentJSON = conf.is(SessionConfig.INDENT_JSON);
+        final boolean indentJSON = conf.is(SessionConfig.FORMAT_INDENT_JSON);
+        final boolean quoteRecord = conf.is(SessionConfig.FORMAT_QUOTE_RECORD);
+
+        if (conf.fmt() == OutputFormat.CSV && conf.is(SessionConfig.FORMAT_CSV_HEADER)) {
+            if (recordType == null) {
+                throw new HyracksDataException("Cannot print CSV with header without specifying output-record-type");
+            }
+            if (quoteRecord) {
+                StringWriter sw = new StringWriter();
+                PrintWriter pw = new PrintWriter(sw);
+                printCSVHeader(recordType, pw);
+                pw.close();
+                conf.out().print(JSONUtil.quoteAndEscape(sw.toString()));
+                conf.out().print("\n");
+                notfirst = true;
+            } else {
+                printCSVHeader(recordType, conf.out());
+            }
+        }
 
         final IFrameTupleAccessor fta = resultReader.getFrameTupleAccessor();
         final IFrame frame = new VSizeFrame(resultDisplayFrameMgr);
@@ -161,10 +156,15 @@ public class ResultUtils {
                     // TODO(tillw): this is inefficient - do this during result generation
                     result = JSONUtil.indent(result, 2);
                 }
-                conf.out().print(result);
                 if (conf.fmt() == OutputFormat.CSV) {
-                    conf.out().print("\r\n");
+                    // TODO(tillw): this is inefficient as well
+                    result = result + "\r\n";
                 }
+                if (quoteRecord) {
+                    // TODO(tillw): this is inefficient as well
+                    result = JSONUtil.quoteAndEscape(result);
+                }
+                conf.out().print(result);
                 ++stats.count;
                 // TODO(tillw) fix this approximation
                 stats.size += result.length();
@@ -193,7 +193,7 @@ public class ResultUtils {
         errorArray.put(errorMessage);
         try {
             errorResp.put("error-code", errorArray);
-            if (! "".equals(errorSummary)) {
+            if (!"".equals(errorSummary)) {
                 errorResp.put("summary", errorSummary);
             } else {
                 //parse exception

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2ae8af7c/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/records/RecordsQueries.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/records/RecordsQueries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/records/RecordsQueries.xml
index 6a61b67..7c2b0a4 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/records/RecordsQueries.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/records/RecordsQueries.xml
@@ -115,19 +115,19 @@
   <test-case FilePath="records">
     <compilation-unit name="closed-closed-fieldname-conflict_issue173">
       <output-dir compare="Text">closed-closed-fieldname-conflict_issue173</output-dir>
-      <expected-error>java.lang.IllegalStateException: Closed fields 0 and 1 have the same field name "name"</expected-error>
+      <expected-error>Closed fields 0 and 1 have the same field name "name"</expected-error>
     </compilation-unit>
   </test-case>
   <test-case FilePath="records">
     <compilation-unit name="open-closed-fieldname-conflict_issue173">
       <output-dir compare="Text">open-closed-fieldname-conflict_issue173</output-dir>
-      <expected-error>org.apache.hyracks.api.exceptions.HyracksDataException: Open field "name" has the same field name as closed field at index 0</expected-error>
+      <expected-error>Open field "name" has the same field name as closed field at index 0</expected-error>
     </compilation-unit>
   </test-case>
   <test-case FilePath="records">
     <compilation-unit name="open-open-fieldname-conflict_issue173">
       <output-dir compare="Text">open-open-fieldname-conflict_issue173</output-dir>
-      <expected-error>org.apache.hyracks.api.exceptions.HyracksDataException: Open fields 0 and 1 have the same field name "name"</expected-error>
+      <expected-error>Open fields 0 and 1 have the same field name "name"</expected-error>
     </compilation-unit>
   </test-case>
 </test-group>

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2ae8af7c/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 195906b..96d7c4f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -17,10 +17,9 @@
  ! under the License.
  !-->
 <!DOCTYPE test-suite [
+  <!ENTITY RecordsQueries SYSTEM "queries_sqlpp/records/RecordsQueries.xml">
 
-             <!ENTITY RecordsQueries SYSTEM "queries_sqlpp/records/RecordsQueries.xml">
-
-             ]>
+]>
 <test-suite
              xmlns="urn:xml.testframework.asterix.apache.org"
              ResultOffsetPath="results"
@@ -1076,43 +1075,43 @@
     <test-case FilePath="comparison">
       <compilation-unit name="issue363_inequality_duration">
         <output-dir compare="Text">issue363_inequality_duration</output-dir>
-        <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: Comparison operations (GT, GE, LT, and LE) for the DURATION type are not defined</expected-error>
+        <expected-error>Comparison operations (GT, GE, LT, and LE) for the DURATION type are not defined</expected-error>
       </compilation-unit>
     </test-case>
     <test-case FilePath="comparison">
       <compilation-unit name="issue363_inequality_interval">
         <output-dir compare="Text">issue363_inequality_interval</output-dir>
-        <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: Comparison operations (GT, GE, LT, and LE) for the INTERVAL type are not defined</expected-error>
+        <expected-error>Comparison operations (GT, GE, LT, and LE) for the INTERVAL type are not defined</expected-error>
       </compilation-unit>
     </test-case>
     <test-case FilePath="comparison">
       <compilation-unit name="issue363_inequality_point">
         <output-dir compare="Text">issue363_inequality_point</output-dir>
-        <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: Comparison operations (GT, GE, LT, and LE) for the POINT type are not defined</expected-error>
+        <expected-error>Comparison operations (GT, GE, LT, and LE) for the POINT type are not defined</expected-error>
       </compilation-unit>
     </test-case>
     <test-case FilePath="comparison">
       <compilation-unit name="issue363_inequality_line">
         <output-dir compare="Text">issue363_inequality_line</output-dir>
-        <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: Comparison operations (GT, GE, LT, and LE) for the LINE type are not defined</expected-error>
+        <expected-error>Comparison operations (GT, GE, LT, and LE) for the LINE type are not defined</expected-error>
       </compilation-unit>
     </test-case>
     <test-case FilePath="comparison">
       <compilation-unit name="issue363_inequality_polygon">
         <output-dir compare="Text">issue363_inequality_polygon</output-dir>
-        <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: Comparison operations (GT, GE, LT, and LE) for the POLYGON type are not defined</expected-error>
+        <expected-error>Comparison operations (GT, GE, LT, and LE) for the POLYGON type are not defined</expected-error>
       </compilation-unit>
     </test-case>
     <test-case FilePath="comparison">
       <compilation-unit name="issue363_inequality_rectangle">
         <output-dir compare="Text">issue363_inequality_rectangle</output-dir>
-        <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: Comparison operations (GT, GE, LT, and LE) for the RECTANGLE type are not defined</expected-error>
+        <expected-error>Comparison operations (GT, GE, LT, and LE) for the RECTANGLE type are not defined</expected-error>
       </compilation-unit>
     </test-case>
     <test-case FilePath="comparison">
       <compilation-unit name="issue363_inequality_circle">
         <output-dir compare="Text">issue363_inequality_circle</output-dir>
-        <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: Comparison operations (GT, GE, LT, and LE) for the CIRCLE type are not defined</expected-error>
+        <expected-error>Comparison operations (GT, GE, LT, and LE) for the CIRCLE type are not defined</expected-error>
       </compilation-unit>
     </test-case>
     <test-case FilePath="comparison">
@@ -1350,7 +1349,7 @@
     <test-case FilePath="custord">
       <compilation-unit name="join_q_07">
         <output-dir compare="Text">join_q_06</output-dir>
-        <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: Undefined alias (variable) reference for identifier c</expected-error>
+        <expected-error>Undefined alias (variable) reference for identifier c</expected-error>
       </compilation-unit>
     </test-case>
     <test-case FilePath="custord">
@@ -2336,19 +2335,19 @@
     <test-case FilePath="global-aggregate">
       <compilation-unit name="q05_error">
         <output-dir compare="Text">q01</output-dir>
-        <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: The first argument should be a RECORD, but it is</expected-error>
+        <expected-error>The first argument should be a RECORD, but it is [ FacebookUserType: open</expected-error>
       </compilation-unit>
     </test-case>
     <test-case FilePath="global-aggregate">
       <compilation-unit name="q06_error">
         <output-dir compare="Text">q01</output-dir>
-        <expected-error>Caused by: org.apache.asterix.common.exceptions.AsterixException: Unsupported type: STRING</expected-error>
+        <expected-error>Unsupported type: STRING</expected-error>
       </compilation-unit>
     </test-case>
     <test-case FilePath="global-aggregate">
       <compilation-unit name="q07_error">
         <output-dir compare="Text">q01</output-dir>
-        <expected-error>org.apache.asterix.common.exceptions.AsterixException: COUNT is a SQL-92 aggregate function. The SQL++ core aggregate function coll_count could potentially express the intent.</expected-error>
+        <expected-error>COUNT is a SQL-92 aggregate function. The SQL++ core aggregate function coll_count could potentially express the intent.</expected-error>
       </compilation-unit>
     </test-case>
     <test-case FilePath="global-aggregate">
@@ -2805,8 +2804,8 @@
       <compilation-unit name="partition-by-nonexistent-field">
         <output-dir compare="Text">partition-by-nonexistent-field</output-dir>
         <expected-error>java.lang.NullPointerException</expected-error>
-        <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: Cannot find dataset</expected-error>
-        <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: Undefined alias (variable) reference for identifier testds</expected-error>
+        <expected-error>Cannot find dataset</expected-error>
+        <expected-error>Undefined alias (variable) reference for identifier testds</expected-error>
       </compilation-unit>
     </test-case>
     <test-case FilePath="misc">
@@ -6332,7 +6331,7 @@
     <test-case FilePath="cross-dataverse">
       <compilation-unit name="cross-dv13">
         <output-dir compare="Text">cross-dv13</output-dir>
-        <expected-error>Error: Recursive invocation testdv2.fun03@0</expected-error>
+        <expected-error>Recursive invocation testdv2.fun03@0</expected-error>
       </compilation-unit>
     </test-case>
     <test-case FilePath="cross-dataverse">
@@ -6348,7 +6347,7 @@
     <test-case FilePath="cross-dataverse">
       <compilation-unit name="cross-dv16">
         <output-dir compare="Text">cross-dv16</output-dir>
-        <expected-error>Error: Recursive invocation testdv1.fun04@0</expected-error>
+        <expected-error>Recursive invocation testdv1.fun04@0</expected-error>
       </compilation-unit>
     </test-case>
     <!--NotImplementedException: No binary comparator factory implemented for type RECORD.
@@ -6410,7 +6409,7 @@
     <test-case FilePath="user-defined-functions">
       <compilation-unit name="query-issue455">
         <output-dir compare="Text">query-issue455</output-dir>
-        <expected-error>Error: function test.printName@0 is undefined</expected-error>
+        <expected-error>function test.printName@0 is undefined</expected-error>
       </compilation-unit>
     </test-case>
     <test-case FilePath="user-defined-functions">
@@ -6554,7 +6553,7 @@
     <test-case FilePath="user-defined-functions">
       <compilation-unit name="udf26">
         <output-dir compare="Text">udf26</output-dir>
-        <expected-error>Error: function test.needs_f1@1 depends upon function test.f1@0 which is undefined</expected-error>
+        <expected-error>function test.needs_f1@1 depends upon function test.f1@0 which is undefined</expected-error>
       </compilation-unit>
     </test-case>
     <test-case FilePath="user-defined-functions">
@@ -6576,13 +6575,13 @@
     <test-case FilePath="user-defined-functions">
       <compilation-unit name="udf30">
         <output-dir compare="Text">udf30</output-dir>
-        <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: Undefined alias (variable) reference for identifier y</expected-error>
+        <expected-error>Undefined alias (variable) reference for identifier y</expected-error>
       </compilation-unit>
     </test-case>
     <test-case FilePath="user-defined-functions">
       <compilation-unit name="f01">
         <output-dir compare="Text">f01</output-dir>
-        <expected-error>Error: function test.int8@0 is undefined</expected-error>
+        <expected-error>function test.int8@0 is undefined</expected-error>
       </compilation-unit>
     </test-case>
     <!-- This test case is not valid anymore since we do not required "IMPORT_PRIVATE_FUNCTIONS" flag anymore -->

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2ae8af7c/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/JSONUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/JSONUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/JSONUtil.java
new file mode 100644
index 0000000..0b973e1
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/JSONUtil.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.utils;
+
+import java.util.Iterator;
+import java.util.logging.Logger;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+public class JSONUtil {
+
+    private static final Logger LOGGER = Logger.getLogger(JSONUtil.class.getName());
+
+    private static final String INDENT = "\t";
+
+    private JSONUtil() {
+    }
+
+    public static String indent(String str, int initialIndent) {
+        try {
+            return append(new StringBuilder(), new JSONObject(str), initialIndent).toString();
+        } catch (JSONException e) {
+            LOGGER.finest("Could not indent JSON string, returning the input string: " + str);
+            return str;
+        }
+    }
+
+    private static StringBuilder append(StringBuilder sb, Object o, int indent) throws JSONException {
+        if (o instanceof JSONObject) {
+            return append(sb, (JSONObject) o, indent);
+        } else if (o instanceof JSONArray) {
+            return append(sb, (JSONArray) o, indent);
+        } else if (o instanceof String) {
+            return quote(sb, (String) o);
+        } else if (JSONObject.NULL.equals(o) || o instanceof Number || o instanceof Boolean) {
+            return sb.append(String.valueOf(o));
+        }
+        throw new UnsupportedOperationException(o.getClass().getSimpleName());
+    }
+
+    private static StringBuilder append(StringBuilder builder, JSONObject jobj, int indent) throws JSONException {
+        StringBuilder sb = builder.append("{\n");
+        boolean first = true;
+        for (Iterator it = jobj.keys(); it.hasNext();) {
+            final String key = (String) it.next();
+            if (first) {
+                first = false;
+            } else {
+                sb = sb.append(",\n");
+            }
+            sb = indent(sb, indent + 1);
+            sb = quote(sb, key);
+            sb = sb.append(": ");
+            sb = append(sb, jobj.get(key), indent + 1);
+        }
+        sb = sb.append("\n");
+        return indent(sb, indent).append("}");
+    }
+
+    private static StringBuilder append(StringBuilder builder, JSONArray jarr, int indent) throws JSONException {
+        StringBuilder sb = builder.append("[\n");
+        for (int i = 0; i < jarr.length(); ++i) {
+            if (i > 0) {
+                sb = sb.append(",\n");
+            }
+            sb = indent(sb, indent + 1);
+            sb = append(sb, jarr.get(i), indent + 1);
+        }
+        sb = sb.append("\n");
+        return indent(sb, indent).append("]");
+    }
+
+    private static StringBuilder quote(StringBuilder sb, String str) {
+        return sb.append('"').append(str).append('"');
+    }
+
+    private static StringBuilder indent(StringBuilder sb, int i) {
+        int indent = i;
+        while (indent > 0) {
+            sb.append(INDENT);
+            --indent;
+        }
+        return sb;
+    }
+
+    public static String quoteAndEscape(String str) {
+        StringBuilder sb = new StringBuilder();
+        sb.append('"');
+        escape(sb, str);
+        return sb.append('"').toString();
+    }
+
+    public static String escape(String str) {
+        return escape(new StringBuilder(), str).toString();
+    }
+
+    private static StringBuilder escape(StringBuilder sb, String str) {
+        for (int i = 0; i < str.length(); ++i) {
+            appendEsc(sb, str.charAt(i));
+        }
+        return sb;
+    }
+
+    private static StringBuilder appendEsc(StringBuilder sb, char c) {
+        CharSequence cs = esc(c);
+        return cs != null ? sb.append(cs) : sb.append(c);
+    }
+
+    public static CharSequence esc(char c) {
+        switch (c) {
+            case '"':
+                return "\\\"";
+            case '\\':
+                return "\\\\";
+            case '/':
+                return "\\/";
+            case '\b':
+                return "\\b";
+            case '\n':
+                return "\\n";
+            case '\f':
+                return "\\f";
+            case '\r':
+                return "\\r";
+            case '\t':
+                return "\\t";
+            default:
+                return null;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2ae8af7c/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/ResultExtractor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/ResultExtractor.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/ResultExtractor.java
new file mode 100644
index 0000000..13ab717
--- /dev/null
+++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/ResultExtractor.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.aql;
+
+import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.util.logging.Logger;
+
+import org.apache.commons.io.IOUtils;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONTokener;
+
+/**
+ * extracts results from the response of the QueryServiceServlet.
+ *
+ * As the response is not necessarily valid JSON, non-JSON content has to be extracted in some cases.
+ * The current implementation creates a toomany copies of the data to be usable for larger results.
+ */
+public class ResultExtractor {
+
+    private static final Logger LOGGER = Logger.getLogger(ResultExtractor.class.getName());
+
+    static InputStream extract(InputStream resultStream) throws Exception {
+        String result = IOUtils.toString(resultStream, Charset.forName("UTF-8"));
+
+        LOGGER.fine("+++++++\n" + result + "\n+++++++\n");
+
+        JSONTokener tokener = new JSONTokener(result);
+        tokener.nextTo('{');
+        tokener.next('{');
+        String name;
+        String type = null;
+        String results = "";
+        while ((name = getFieldName(tokener)) != null) {
+            if ("requestID".equals(name) || "signature".equals(name) || "status".equals(name)) {
+                getStringField(tokener);
+            } else if ("type".equals(name)) {
+                type = getStringField(tokener);
+            } else if ("metrics".equals(name)) {
+                JSONObject metrics = getObjectField(tokener);
+                LOGGER.fine(name + ": " + metrics);
+            } else if ("errors".equals(name)) {
+                JSONArray errors = getArrayField(tokener);
+                LOGGER.fine(name + ": " + errors);
+                JSONObject err = errors.getJSONObject(0);
+                throw new Exception(err.getString("msg"));
+            } else if ("results".equals(name)) {
+                results = getResults(tokener, type);
+            } else {
+                throw tokener.syntaxError(name + ": unanticipated field");
+            }
+        }
+        while (tokener.more() && tokener.skipTo('}') != '}') {
+            // skip along
+        }
+        tokener.next('}');
+        return IOUtils.toInputStream(results);
+    }
+
+    private static String getFieldName(JSONTokener tokener) throws JSONException {
+        char c = tokener.skipTo('"');
+        if (c != '"') {
+            return null;
+        }
+        tokener.next('"');
+        return tokener.nextString('"');
+    }
+
+    private static String getStringField(JSONTokener tokener) throws JSONException {
+        tokener.skipTo('"');
+        tokener.next('"');
+        return tokener.nextString('"');
+    }
+
+    private static JSONArray getArrayField(JSONTokener tokener) throws JSONException {
+        tokener.skipTo(':');
+        tokener.next(':');
+        Object obj = tokener.nextValue();
+        if (obj instanceof JSONArray) {
+            return (JSONArray) obj;
+        } else {
+            throw tokener.syntaxError(String.valueOf(obj) + ": unexpected value");
+        }
+    }
+
+    private static JSONObject getObjectField(JSONTokener tokener) throws JSONException {
+        tokener.skipTo(':');
+        tokener.next(':');
+        Object obj = tokener.nextValue();
+        if (obj instanceof JSONObject) {
+            return (JSONObject) obj;
+        } else {
+            throw tokener.syntaxError(String.valueOf(obj) + ": unexpected value");
+        }
+    }
+
+    private static String getResults(JSONTokener tokener, String type) throws JSONException {
+        tokener.skipTo(':');
+        tokener.next(':');
+        StringBuilder result = new StringBuilder();
+        if (type != null) {
+            // a type was provided in the response and so the result is encoded as an array of escaped strings that
+            // need to be concatenated
+            Object obj = tokener.nextValue();
+            if (!(obj instanceof JSONArray)) {
+                throw tokener.syntaxError("array expected");
+            }
+            JSONArray array = (JSONArray) obj;
+            for (int i = 0; i < array.length(); ++i) {
+                result.append(array.getString(i));
+            }
+            return result.toString();
+        } else {
+            int level = 0;
+            boolean inQuote = false;
+            while (tokener.more()) {
+                char c = tokener.next();
+                switch (c) {
+                    case '{':
+                    case '[':
+                        ++level;
+                        result.append(c);
+                        break;
+                    case '}':
+                    case ']':
+                        --level;
+                        result.append(c);
+                        break;
+                    case '"':
+                        if (inQuote) {
+                            --level;
+                        } else {
+                            ++level;
+                        }
+                        inQuote = !inQuote;
+                        result.append(c);
+                        break;
+                    case ',':
+                        if (level == 0) {
+                            return result.toString().trim();
+                        } else {
+                            result.append(c);
+                        }
+                        break;
+                    default:
+                        result.append(c);
+                        break;
+                }
+            }
+        }
+        return null;
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2ae8af7c/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java
index 1af6b80..58c6a91 100644
--- a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java
+++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java
@@ -46,6 +46,7 @@ import org.apache.asterix.test.server.TestServerProvider;
 import org.apache.asterix.testframework.context.TestCaseContext;
 import org.apache.asterix.testframework.context.TestCaseContext.OutputFormat;
 import org.apache.asterix.testframework.context.TestFileContext;
+import org.apache.asterix.testframework.xml.TestCase;
 import org.apache.asterix.testframework.xml.TestCase.CompilationUnit;
 import org.apache.asterix.testframework.xml.TestGroup;
 import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
@@ -277,33 +278,74 @@ public class TestExecutor {
         return statusCode;
     }
 
-    // Executes Query and returns results as JSONArray
-    public InputStream executeQuery(String str, OutputFormat fmt, String url, List<CompilationUnit.Parameter> params)
-            throws Exception {
-        HttpMethodBase method = null;
-        if (str.length() + url.length() < MAX_URL_LENGTH) {
+    public InputStream executeQuery(String str, OutputFormat fmt, String url,
+            List<CompilationUnit.Parameter> params) throws Exception {
+        HttpMethod method = constructHttpMethod(str, url, "query", false, params);
+        // Set accepted output response type
+        method.setRequestHeader("Accept", fmt.mimeType());
+        executeHttpMethod(method);
+        return method.getResponseBodyAsStream();
+    }
+
+    public InputStream executeQueryService(String str, OutputFormat fmt, String url,
+            List<CompilationUnit.Parameter> params) throws Exception {
+        setFormatParam(params, fmt);
+        HttpMethod method = constructHttpMethod(str, url, "statement", true, params);
+        // Set accepted output response type
+        method.setRequestHeader("Accept", OutputFormat.CLEAN_JSON.mimeType());
+        executeHttpMethod(method);
+        return method.getResponseBodyAsStream();
+    }
+
+    private void setFormatParam(List<CompilationUnit.Parameter> params, OutputFormat fmt) {
+        boolean formatSet = false;
+        for (CompilationUnit.Parameter param : params) {
+            if ("format".equals(param.getName())) {
+                param.setValue(fmt.mimeType());
+                formatSet = true;
+            }
+        }
+        if (!formatSet) {
+            CompilationUnit.Parameter formatParam = new CompilationUnit.Parameter();
+            formatParam.setName("format");
+            formatParam.setValue(fmt.mimeType());
+            params.add(formatParam);
+        }
+    }
+
+    private HttpMethod constructHttpMethod(String statement, String endpoint, String stmtParam, boolean postStmtAsParam,
+            List<CompilationUnit.Parameter> otherParams) {
+        HttpMethod method;
+        if (statement.length() + endpoint.length() < MAX_URL_LENGTH) {
             // Use GET for small-ish queries
-            method = new GetMethod(url);
-            NameValuePair[] parameters = new NameValuePair[params.size() + 1];
-            parameters[0] = new NameValuePair("query", str);
+            GetMethod getMethod = new GetMethod(endpoint);
+            NameValuePair[] parameters = new NameValuePair[otherParams.size() + 1];
+            parameters[0] = new NameValuePair(stmtParam, statement);
             int i = 1;
-            for (CompilationUnit.Parameter param : params) {
+            for (CompilationUnit.Parameter param : otherParams) {
                 parameters[i++] = new NameValuePair(param.getName(), param.getValue());
             }
-            method.setQueryString(parameters);
+            getMethod.setQueryString(parameters);
+            method = getMethod;
         } else {
             // Use POST for bigger ones to avoid 413 FULL_HEAD
-            // QQQ POST API doesn't allow encoding additional parameters
-            method = new PostMethod(url);
-            ((PostMethod) method).setRequestEntity(new StringRequestEntity(str));
+            PostMethod postMethod = new PostMethod(endpoint);
+            if (postStmtAsParam) {
+                for (CompilationUnit.Parameter param : otherParams) {
+                    postMethod.setParameter(param.getName(), param.getValue());
+                }
+                postMethod.setParameter("statement", statement);
+            } else {
+                // this seems pretty bad - we should probably fix the API and not the client
+                postMethod.setRequestEntity(new StringRequestEntity(statement));
+            }
+            method = postMethod;
         }
-
-        // Set accepted output response type
-        method.setRequestHeader("Accept", fmt.mimeType());
         // Provide custom retry handler is necessary
-        method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));
-        executeHttpMethod(method);
-        return method.getResponseBodyAsStream();
+        HttpMethodParams httpMethodParams = method.getParams();
+        httpMethodParams.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));
+        httpMethodParams.setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, StandardCharsets.UTF_8.name());
+        return method;
     }
 
     public InputStream executeClusterStateQuery(OutputFormat fmt, String url) throws Exception {
@@ -473,6 +515,7 @@ public class TestExecutor {
         executeTest(actualPath, testCaseCtx, pb, isDmlRecoveryTest, null);
     }
 
+
     public void executeTest(TestCaseContext testCaseCtx, TestFileContext ctx, String statement,
             boolean isDmlRecoveryTest, ProcessBuilder pb, CompilationUnit cUnit, MutableInt queryCount,
             List<TestFileContext> expectedResultFileCtxs, File testFile, String actualPath) throws Exception {
@@ -523,8 +566,9 @@ public class TestExecutor {
                     }
                 } else {
                     if (ctx.getType().equalsIgnoreCase("query")) {
-                        resultStream = executeQuery(statement, fmt,
-                                "http://" + host + ":" + port + Servlets.SQLPP_QUERY.getPath(), cUnit.getParameter());
+                        resultStream = executeQueryService(statement, fmt,
+                                "http://" + host + ":" + port + Servlets.QUERY_SERVICE.getPath(), cUnit.getParameter());
+                        resultStream = ResultExtractor.extract(resultStream);
                     } else if (ctx.getType().equalsIgnoreCase("async")) {
                         resultStream = executeAnyAQLAsync(statement, false, fmt,
                                 "http://" + host + ":" + port + Servlets.SQLPP.getPath());
@@ -533,7 +577,6 @@ public class TestExecutor {
                                 "http://" + host + ":" + port + Servlets.SQLPP.getPath());
                     }
                 }
-
                 if (queryCount.intValue() >= expectedResultFileCtxs.size()) {
                     throw new IllegalStateException("no result file for " + testFile.toString() + "; queryCount: "
                             + queryCount + ", filectxs.size: " + expectedResultFileCtxs.size());
@@ -770,6 +813,7 @@ public class TestExecutor {
                     } else {
                         // Get the expected exception
                         String expectedError = cUnit.getExpectedError().get(numOfErrors - 1);
+                        System.err.println("+++++\n" + expectedError + "\n+++++\n");
                         if (e.toString().contains(expectedError)) {
                             System.err.println("...but that was expected.");
                         } else {


Mime
View raw message