hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shiv...@apache.org
Subject incubator-hawq git commit: HAWQ-577. Updated PXF metadata api to stream response
Date Tue, 29 Mar 2016 18:30:49 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master bb749ebb4 -> 193fb0586


HAWQ-577. Updated PXF metadata api to stream response


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/193fb058
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/193fb058
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/193fb058

Branch: refs/heads/master
Commit: 193fb0586cd7ddb1a1d135165742b99e213f0932
Parents: bb749eb
Author: Shivram Mani <shivram.mani@gmail.com>
Authored: Tue Mar 29 11:30:40 2016 -0700
Committer: Shivram Mani <shivram.mani@gmail.com>
Committed: Tue Mar 29 11:30:40 2016 -0700

----------------------------------------------------------------------
 .../hawq/pxf/service/MetadataResponse.java      | 93 ++++++++++++++++++++
 .../pxf/service/MetadataResponseFormatter.java  | 80 ++++++-----------
 .../hawq/pxf/service/rest/MetadataResource.java |  9 +-
 .../service/MetadataResponseFormatterTest.java  | 58 +++++++-----
 4 files changed, 159 insertions(+), 81 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/193fb058/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponse.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponse.java
b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponse.java
new file mode 100644
index 0000000..ff73499
--- /dev/null
+++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponse.java
@@ -0,0 +1,93 @@
+package org.apache.hawq.pxf.service;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.List;
+
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.StreamingOutput;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hawq.pxf.api.Metadata;
+
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.annotate.JsonSerialize.Inclusion;
+
+
+/**
+ * Class for serializing metadata in JSON format. The class implements
+ * {@link StreamingOutput} so the serialization will be done in a stream and not
+ * in one bulk, this in order to avoid running out of memory when processing a
+ * lot of items.
+ */
+public class MetadataResponse implements StreamingOutput {
+
+    private static final Log Log = LogFactory.getLog(MetadataResponse.class);
+    private static final String METADATA_DEFAULT_RESPONSE = "{\"PXFMetadata\":[]}";
+
+    private List<Metadata> metadataList;
+
+    /**
+     * Constructs metadata response out of a metadata list
+     *
+     * @param metadataList metadata list
+     */
+    public MetadataResponse(List<Metadata> metadataList) {
+        this.metadataList = metadataList;
+    }
+
+    /**
+     * Serializes the metadata list in JSON, To be used as the result string for HAWQ.
+     */
+    @Override
+    public void write(OutputStream output) throws IOException,
+            WebApplicationException {
+        DataOutputStream dos = new DataOutputStream(output);
+        ObjectMapper mapper = new ObjectMapper();
+        mapper.setSerializationInclusion(Inclusion.NON_EMPTY); // ignore empty fields
+
+        if(metadataList == null || metadataList.isEmpty()) {
+            dos.write(METADATA_DEFAULT_RESPONSE.getBytes());
+            return;
+        }
+
+        dos.write("{\"PXFMetadata\":[".getBytes());
+
+        String prefix = "";
+        for (Metadata metadata : metadataList) {
+            if(metadata == null) {
+                throw new IllegalArgumentException("metadata object is null - cannot serialize");
+            }
+            if ((metadata.getFields() == null) || metadata.getFields().isEmpty()) {
+                throw new IllegalArgumentException("metadata for " + metadata.getItem() +
" contains no fields - cannot serialize");
+            }
+            StringBuilder result = new StringBuilder();
+            result.append(prefix).append(mapper.writeValueAsString(metadata));
+            prefix = ",";
+            dos.write(result.toString().getBytes());
+        }
+
+        dos.write("]}".getBytes());
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/193fb058/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponseFormatter.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponseFormatter.java
b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponseFormatter.java
index a420ca7..92d11de 100644
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponseFormatter.java
+++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponseFormatter.java
@@ -37,7 +37,6 @@ import org.apache.hawq.pxf.api.Metadata;
 public class MetadataResponseFormatter {
 
     private static final Log LOG = LogFactory.getLog(MetadataResponseFormatter.class);
-    private static final String METADATA_DEFAULT_RESPONSE = "{\"PXFMetadata\":[]}";
 
     /**
      * Converts list of {@link Metadata} to JSON String format.
@@ -46,76 +45,49 @@ public class MetadataResponseFormatter {
      * @return JSON formatted response
      * @throws IOException if converting the data to JSON fails
      */
-    public static String formatResponseString(List<Metadata> metadataList) throws IOException
{
-        return MetadataResponseFormatter.metadataToJSON(metadataList);
-    }
-
-    /**
-     * Serializes a metadata in JSON,
-     * To be used as the result string for HAWQ.
-     * An example result is as follows:
-     *
-     * {"PXFMetadata":[{"item":{"path":"default","name":"t1"},"fields":[{"name":"a","type":"int"},{"name":"b","type":"float"}]}]}
-     */
-    private static String metadataToJSON(List<Metadata> metadataList) throws IOException
{
-
-        if (metadataList == null || metadataList.isEmpty()) {
-               return METADATA_DEFAULT_RESPONSE;
+    public static MetadataResponse formatResponse(List<Metadata> metadataList, String
path) throws IOException {
+        /* print the fragment list to log when in debug level */
+        if (LOG.isDebugEnabled()) {
+            MetadataResponseFormatter.printMetadata(metadataList, path);
         }
 
-        StringBuilder result = null;
-
-        for(Metadata metadata: metadataList) {
-            if(metadata == null) {
-                throw new IllegalArgumentException("metadata object is null - cannot serialize");
-            }
-            if ((metadata.getFields() == null) || metadata.getFields().isEmpty()) {
-                throw new IllegalArgumentException("metadata for " + metadata.getItem() +
" contains no fields - cannot serialize");
-            }
-            if (result == null) {
-                result = new StringBuilder("{\"PXFMetadata\":["); /* prefix info */
-            } else {
-                result.append(",");
-            }
-
-            ObjectMapper mapper = new ObjectMapper();
-            mapper.setSerializationInclusion(Inclusion.NON_EMPTY); // ignore empty fields
-            result.append(mapper.writeValueAsString(metadata));
-        }
-
-        return result.append("]}").toString(); /* append suffix info */
-
+        return new MetadataResponse(metadataList);
     }
 
     /**
      * Converts metadata list to a readable string.
      * Intended for debugging purposes only.
      */
-    private static String metadataToString(List<Metadata> metadataList) {
-        StringBuilder result = new StringBuilder("Metadata:");
+    private static void printMetadata(List<Metadata> metadataList, String path) {
+        LOG.debug("Metadata List for path " + path + ": ");
+
+        if (null == metadataList || metadataList.isEmpty()) {
+            LOG.debug("No metadata");
+            return;
+        }
 
         for(Metadata metadata: metadataList) {
-            result.append(" Metadata for item \"");
+            StringBuilder result = new StringBuilder();
 
             if (metadata == null) {
-                return "No metadata";
+                result.append("None");
+                LOG.debug(result);
+                continue;
             }
 
-            result.append(metadata.getItem()).append("\": ");
+            result.append("Metadata for item \"").append(metadata.getItem()).append("\":
");
 
             if ((metadata.getFields() == null) || metadata.getFields().isEmpty()) {
-                result.append("no fields in item");
-                return result.toString();
-            }
-
-            int i = 0;
-            for (Metadata.Field field : metadata.getFields()) {
-                result.append("Field #").append(++i).append(": [")
-                        .append("Name: ").append(field.getName())
-                        .append(", Type: ").append(field.getType()).append("] ");
+                result.append("None");
+            } else {
+                int i = 0;
+                for (Metadata.Field field : metadata.getFields()) {
+                    result.append("Field #").append(++i).append(": [")
+                            .append("Name: ").append(field.getName())
+                            .append(", Type: ").append(field.getType()).append("] ");
+                }
             }
+            LOG.debug(result);
         }
-
-        return result.toString();
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/193fb058/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/MetadataResource.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/MetadataResource.java
b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/MetadataResource.java
index e16d14a..3f85bb8 100644
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/MetadataResource.java
+++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/MetadataResource.java
@@ -41,6 +41,7 @@ import org.apache.hawq.pxf.api.Metadata;
 import org.apache.hawq.pxf.api.MetadataFetcher;
 import org.apache.hawq.pxf.api.utilities.InputData;
 import org.apache.hawq.pxf.service.MetadataFetcherFactory;
+import org.apache.hawq.pxf.service.MetadataResponse;
 import org.apache.hawq.pxf.service.MetadataResponseFormatter;
 import org.apache.hawq.pxf.service.utilities.ProtocolData;
 import org.apache.hawq.pxf.service.utilities.SecuredHDFS;
@@ -105,10 +106,11 @@ public class MetadataResource extends RestResource {
             // 2. get Metadata
             List<Metadata> metadata = metadataFetcher.getMetadata(pattern);
 
-            // 3. serialize to JSON
-            jsonOutput = MetadataResponseFormatter.formatResponseString(metadata);
+            // 3. stream JSON ouptput
+            MetadataResponse metadataResponse = MetadataResponseFormatter.formatResponse(
+                    metadata, pattern);
 
-            LOG.debug("getMetadata output: " + jsonOutput);
+            return Response.ok(metadataResponse, MediaType.APPLICATION_JSON_TYPE).build();
 
         } catch (ClientAbortException e) {
             LOG.error("Remote connection closed by HAWQ", e);
@@ -118,6 +120,5 @@ public class MetadataResource extends RestResource {
             throw e;
         }
 
-        return Response.ok(jsonOutput, MediaType.APPLICATION_JSON_TYPE).build();
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/193fb058/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/MetadataResponseFormatterTest.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/MetadataResponseFormatterTest.java
b/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/MetadataResponseFormatterTest.java
index b4583aa..0182835 100644
--- a/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/MetadataResponseFormatterTest.java
+++ b/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/MetadataResponseFormatterTest.java
@@ -19,21 +19,29 @@ package org.apache.hawq.pxf.service;
  * under the License.
  */
 
-
-import static org.junit.Assert.*;
-
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.commons.lang.StringUtils;
-import org.apache.directory.shared.kerberos.components.MethodData;
-import org.junit.Test;
-
+import static org.junit.Assert.*;
 import org.apache.hawq.pxf.api.Metadata;
 
+import org.junit.Test;
+
 public class MetadataResponseFormatterTest {
 
-    String result = null;
+    MetadataResponse response = null;
+
+    private String convertResponseToString(MetadataResponse data) {
+        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+        try {
+            data.write(outputStream);
+        } catch (IOException e) {
+            fail(e.toString());
+        }
+        return outputStream.toString();
+    }
 
     @Test
     public void formatResponseString() throws Exception {
@@ -45,12 +53,12 @@ public class MetadataResponseFormatterTest {
         fields.add(new Metadata.Field("field2", "text"));
         metadataList.add(metadata);
 
-        result = MetadataResponseFormatter.formatResponseString(metadataList);
+        response = MetadataResponseFormatter.formatResponse(metadataList, "path.file");
         StringBuilder expected = new StringBuilder("{\"PXFMetadata\":[{");
         expected.append("\"item\":{\"path\":\"default\",\"name\":\"table1\"},")
                 .append("\"fields\":[{\"name\":\"field1\",\"type\":\"int\"},{\"name\":\"field2\",\"type\":\"text\"}]}]}");
 
-        assertEquals(expected.toString(), result);
+        assertEquals(expected.toString(), convertResponseToString(response));
     }
 
     @Test
@@ -63,12 +71,12 @@ public class MetadataResponseFormatterTest {
         fields.add(new Metadata.Field("field2", "text", new String[] {}));
         metadataList.add(metadata);
 
-        result = MetadataResponseFormatter.formatResponseString(metadataList);
+        response = MetadataResponseFormatter.formatResponse(metadataList, "path.file");
         StringBuilder expected = new StringBuilder("{\"PXFMetadata\":[{");
         expected.append("\"item\":{\"path\":\"default\",\"name\":\"table1\"},")
                 .append("\"fields\":[{\"name\":\"field1\",\"type\":\"int\"},{\"name\":\"field2\",\"type\":\"text\"}]}]}");
 
-        assertEquals(expected.toString(), result);
+        assertEquals(expected.toString(), convertResponseToString(response));
     }
 
     @Test
@@ -84,7 +92,7 @@ public class MetadataResponseFormatterTest {
                 new String[] {"50"}));
         metadataList.add(metadata);
 
-        result = MetadataResponseFormatter.formatResponseString(metadataList);
+        response = MetadataResponseFormatter.formatResponse(metadataList, "path.file");
         StringBuilder expected = new StringBuilder("{\"PXFMetadata\":[{");
         expected.append("\"item\":{\"path\":\"default\",\"name\":\"table1\"},")
                 .append("\"fields\":[")
@@ -93,15 +101,16 @@ public class MetadataResponseFormatterTest {
                 .append("{\"name\":\"field3\",\"type\":\"char\",\"modifiers\":[\"50\"]}")
                 .append("]}]}");
 
-        assertEquals(expected.toString(), result);
+        assertEquals(expected.toString(), convertResponseToString(response));
     }
 
     @Test
     public void formatResponseStringNull() throws Exception {
         List<Metadata> metadataList = null;
-        result = MetadataResponseFormatter.formatResponseString(metadataList);
+        response = MetadataResponseFormatter.formatResponse(metadataList, "path.file");
         String expected = new String("{\"PXFMetadata\":[]}");
-        assertEquals(expected, result);
+
+        assertEquals(expected, convertResponseToString(response));
     }
 
     @Test
@@ -111,7 +120,8 @@ public class MetadataResponseFormatterTest {
         Metadata metadata = new Metadata(itemName, null);
         metadataList.add(metadata);
         try {
-            result = MetadataResponseFormatter.formatResponseString(metadataList);
+            response = MetadataResponseFormatter.formatResponse(metadataList, "path.file");
+            convertResponseToString(response);
             fail("formatting should fail because fields field is null");
         } catch (IllegalArgumentException e) {
             assertEquals("metadata for " + metadata.getItem() + " contains no fields - cannot
serialize", e.getMessage());
@@ -122,7 +132,8 @@ public class MetadataResponseFormatterTest {
         metadata = new Metadata(itemName, fields);
         metadataList.add(metadata);
         try {
-            result = MetadataResponseFormatter.formatResponseString(metadataList);
+            response = MetadataResponseFormatter.formatResponse(metadataList, "path.file");
+            convertResponseToString(response);
             fail("formatting should fail because there are no fields");
         } catch (IllegalArgumentException e) {
             assertEquals("metadata for " + metadata.getItem() + " contains no fields - cannot
serialize", e.getMessage());
@@ -139,7 +150,8 @@ public class MetadataResponseFormatterTest {
         metadataList.add(null);
         metadataList.add(metadata);
         try {
-            result = MetadataResponseFormatter.formatResponseString(metadataList);
+            response = MetadataResponseFormatter.formatResponse(metadataList, "path.file");
+            convertResponseToString(response);
             fail("formatting should fail because one of the metdata object is null");
         } catch (IllegalArgumentException e) {
             assertEquals("metadata object is null - cannot serialize", e.getMessage());
@@ -157,7 +169,7 @@ public class MetadataResponseFormatterTest {
             fields.add(new Metadata.Field("field2", "text"));
             metdataList.add(metadata);
         }
-        result = MetadataResponseFormatter.formatResponseString(metdataList);
+        response = MetadataResponseFormatter.formatResponse(metdataList, "path.file");
 
         StringBuilder expected = new StringBuilder();
         for (int i=1; i<=10; i++) {
@@ -171,7 +183,7 @@ public class MetadataResponseFormatterTest {
         }
         expected.append("]}");
 
-        assertEquals(expected.toString(), result);
+        assertEquals(expected.toString(), convertResponseToString(response));
     }
 
     @Test
@@ -185,7 +197,7 @@ public class MetadataResponseFormatterTest {
             fields.add(new Metadata.Field("field2", "text"));
             metdataList.add(metadata);
         }
-        result = MetadataResponseFormatter.formatResponseString(metdataList);
+        response = MetadataResponseFormatter.formatResponse(metdataList, "path.file");
         StringBuilder expected = new StringBuilder();
         for (int i=1; i<=10; i++) {
             if(i==1) {
@@ -198,7 +210,7 @@ public class MetadataResponseFormatterTest {
         }
         expected.append("]}");
 
-        assertEquals(expected.toString(), result);
+        assertEquals(expected.toString(), convertResponseToString(response));
     }
 }
 


Mime
View raw message