drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From par...@apache.org
Subject drill git commit: DRILL-2172: make web UI bound check before accessing a vector, returning null for overflowing indices; fix a non-string type rendering problem;
Date Mon, 09 Feb 2015 20:01:03 GMT
Repository: drill
Updated Branches:
  refs/heads/master 0d867b0da -> 3d863b5eb


DRILL-2172: make web UI bound check before accessing a vector, returning null for overflowing
indices; fix a non-string type rendering problem;


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/3d863b5e
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/3d863b5e
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/3d863b5e

Branch: refs/heads/master
Commit: 3d863b5ebff53ba5b8fcde922303643137f4b19b
Parents: 0d867b0
Author: Hanifi Gunes <hgunes@maprtech.com>
Authored: Thu Feb 5 14:16:52 2015 -0800
Committer: Parth Chandra <pchandra@maprtech.com>
Committed: Mon Feb 9 12:00:45 2015 -0800

----------------------------------------------------------------------
 .../drill/exec/server/rest/QueryResources.java  |  70 +++++------
 .../drill/exec/server/rest/QueryWrapper.java    | 126 +++++++++----------
 .../src/main/resources/rest/query/result.ftl    |   8 +-
 3 files changed, 88 insertions(+), 116 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/3d863b5e/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryResources.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryResources.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryResources.java
index f8998ed..9ef6676 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryResources.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryResources.java
@@ -17,7 +17,7 @@
  */
 package org.apache.drill.exec.server.rest;
 
-import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 
@@ -30,6 +30,7 @@ import javax.ws.rs.Path;
 import javax.ws.rs.Produces;
 import javax.ws.rs.core.MediaType;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.exec.coord.ClusterCoordinator;
@@ -55,7 +56,7 @@ public class QueryResources {
   @Path("/query.json")
   @Consumes(MediaType.APPLICATION_JSON)
   @Produces(MediaType.APPLICATION_JSON)
-  public List<Map<String, Object>> submitQueryJSON(QueryWrapper query) throws
Exception {
+  public QueryWrapper.QueryResult submitQueryJSON(QueryWrapper query) throws Exception {
     final DrillConfig config = work.getContext().getConfig();
     final ClusterCoordinator coordinator = work.getContext().getClusterCoordinator();
     final BufferAllocator allocator = work.getContext().getAllocator();
@@ -67,58 +68,43 @@ public class QueryResources {
   @Consumes(MediaType.APPLICATION_FORM_URLENCODED)
   @Produces(MediaType.TEXT_HTML)
   public Viewable submitQuery(@FormParam("query") String query, @FormParam("queryType") String
queryType) throws Exception {
-    List<Map<String, Object>> result = submitQueryJSON(new QueryWrapper(query,
queryType));
-
-    List<String> columnNames;
-    if (result.isEmpty()) {
-      columnNames = Lists.newArrayList();
-    } else {
-      columnNames = Lists.newArrayList(result.get(0).keySet());
-    }
-    List<List<Object>> records = new ArrayList<>();
-
-    if(!isEmptyResult(result)) {
-      for (Map m : result) {
-        records.add(new ArrayList<Object>(m.values()));
-      }
-    }
-
-    Table table = new Table(columnNames, records);
-
-    return new Viewable("/rest/query/result.ftl", table);
+    final QueryWrapper.QueryResult result = submitQueryJSON(new QueryWrapper(query, queryType));
+    return new Viewable("/rest/query/result.ftl", new TabularResult(result));
   }
 
-  private boolean isEmptyResult(List<Map<String, Object>> result) {
-    if (result.size() > 1) {
-      return false;
-    } else {
-      for(Object col : result.get(0).values()) {
-        if(col != null) {
-          return false;
+  public static class TabularResult {
+    private final List<String> columns;
+    private final List<List<String>> rows;
+
+    public TabularResult(QueryWrapper.QueryResult result) {
+      final List<List<String>> rows = Lists.newArrayList();
+      for (Map<String, String> rowMap:result.rows) {
+        final List<String> row = Lists.newArrayList();
+        for (String col:result.columns) {
+          row.add(rowMap.get(col));
         }
+        rows.add(row);
       }
 
-      return true;
+      this.columns = ImmutableList.copyOf(result.columns);
+      this.rows = rows;
     }
-  }
 
-  public class Table {
-    private List<String> columnNames;
-    private List<List<Object>> records;
-
-    public Table(List<String> columnNames, List<List<Object>> records)
{
-      this.columnNames = columnNames;
-      this.records = records;
+    public TabularResult(List<String> columns, List<List<String>> rows)
{
+      this.columns = columns;
+      this.rows = rows;
     }
 
-    public boolean isEmpty() { return getColumnNames().isEmpty(); }
+    public boolean isEmpty() {
+      return columns.isEmpty();
+    }
 
-    public List<String> getColumnNames() {
-      return columnNames;
+    public List<String> getColumns() {
+      return columns;
     }
 
-    public List<List<Object>> getRecords() {
-      return records;
+    public List<List<String>> getRows() {
+      return rows;
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/3d863b5e/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
index 922533a..8996a69 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
@@ -18,24 +18,23 @@
 
 package org.apache.drill.exec.server.rest;
 
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.LinkedList;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicInteger;
 
 import javax.xml.bind.annotation.XmlRootElement;
 
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.exec.client.DrillClient;
 import org.apache.drill.exec.coord.ClusterCoordinator;
 import org.apache.drill.exec.exception.SchemaChangeException;
 import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.physical.impl.flatten.FlattenRecordBatch;
 import org.apache.drill.exec.proto.UserBitShared;
-import org.apache.drill.exec.record.MaterializedField;
 import org.apache.drill.exec.record.RecordBatchLoader;
 import org.apache.drill.exec.record.VectorWrapper;
 import org.apache.drill.exec.rpc.RpcException;
@@ -43,10 +42,10 @@ import org.apache.drill.exec.rpc.user.ConnectionThrottle;
 import org.apache.drill.exec.rpc.user.QueryResultBatch;
 import org.apache.drill.exec.rpc.user.UserResultsListener;
 import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.proto.UserBitShared.SerializedField;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import parquet.Preconditions;
 
 @XmlRootElement
 public class QueryWrapper {
@@ -80,24 +79,36 @@ public class QueryWrapper {
     return type;
   }
 
-  public List<Map<String, Object>> run(DrillConfig config, ClusterCoordinator
coordinator, BufferAllocator allocator)
+  public QueryResult run(DrillConfig config, ClusterCoordinator coordinator, BufferAllocator
allocator)
     throws Exception {
     try(DrillClient client = new DrillClient(config, coordinator, allocator)){
-      Listener listener = new Listener(new RecordBatchLoader(allocator));
+      Listener listener = new Listener(allocator);
 
       client.connect();
       client.runQuery(getType(), query, listener);
 
-      List<Map<String, Object>> result = listener.waitForCompletion();
-      if (result.isEmpty()) {
-        Map<String, Object> dumbRecord = new HashMap<>();
-        for (String columnName : listener.getColumnNames()) {
-          dumbRecord.put(columnName, null);
+      listener.waitForCompletion();
+      if (listener.results.isEmpty()) {
+        listener.results.add(Maps.<String, String>newHashMap());
+      }
+      final Map<String, String> first = listener.results.get(0);
+      for (String columnName : listener.columns) {
+        if (!first.containsKey(columnName)) {
+          first.put(columnName, null);
         }
-        result.add(dumbRecord);
       }
 
-      return result;
+      return new QueryResult(listener.columns, listener.results);
+    }
+  }
+
+  public static class QueryResult {
+    public final Collection<String> columns;
+    public final List<Map<String, String>> rows;
+
+    public QueryResult(Collection<String> columns, List<Map<String, String>>
rows) {
+      this.columns = columns;
+      this.rows = rows;
     }
   }
 
@@ -109,15 +120,13 @@ public class QueryWrapper {
 
   private static class Listener implements UserResultsListener {
     private volatile Exception exception;
-    private AtomicInteger count = new AtomicInteger();
-    private CountDownLatch latch = new CountDownLatch(1);
-    private List<Map<String, Object>> output = new LinkedList<>();
-    private ArrayList<String> columnNames;
-    private RecordBatchLoader loader;
-    private boolean schemaAdded = false;
-
-    Listener(RecordBatchLoader loader) {
-      this.loader = loader;
+    private final CountDownLatch latch = new CountDownLatch(1);
+    private final BufferAllocator allocator;
+    public final List<Map<String, String>> results = Lists.newArrayList();
+    public final Set<String> columns = Sets.newLinkedHashSet();
+
+    Listener(BufferAllocator allocator) {
+      this.allocator = Preconditions.checkNotNull(allocator, "allocator cannot be null");
     }
 
     @Override
@@ -129,68 +138,45 @@ public class QueryWrapper {
 
     @Override
     public void resultArrived(QueryResultBatch result, ConnectionThrottle throttle) {
-      int rows = result.getHeader().getRowCount();
-      if (result.hasData()) {
-        count.addAndGet(rows);
-        try {
+      try {
+        final int rows = result.getHeader().getRowCount();
+        if (result.hasData()) {
+          final RecordBatchLoader loader = new RecordBatchLoader(allocator);
           loader.load(result.getHeader().getDef(), result.getData());
-          if (!schemaAdded || output.isEmpty()) {
-                columnNames = new ArrayList<>();
-            for (int i = 0; i < loader.getSchema().getFieldCount(); ++i) {
-              columnNames.add(loader.getSchema().getColumn(i).getPath().getAsUnescapedPath());
-            }
-            schemaAdded = true;
+          for (int i = 0; i < loader.getSchema().getFieldCount(); ++i) {
+            columns.add(loader.getSchema().getColumn(i).getPath().getAsUnescapedPath());
           }
-        } catch (SchemaChangeException e) {
-          throw new RuntimeException(e);
-        }
-        for (int i = 0; i < rows; ++i) {
-          Map<String, Object> record = new HashMap<>();
-          int j = 0;
-          for (VectorWrapper<?> vw : loader) {
-            ValueVector.Accessor accessor = vw.getValueVector().getAccessor();
-            Object object = accessor.getObject(i);
-            if (object != null && (! object.getClass().getName().startsWith("java.lang")))
{
-              object = object.toString();
-            }
-            if (object != null) {
-              record.put(columnNames.get(j), object);
-            } else {
-              record.put(columnNames.get(j), null);
+          for (int i = 0; i < rows; ++i) {
+            final Map<String, String> record = Maps.newHashMap();
+            for (VectorWrapper<?> vw : loader) {
+              final String field = vw.getValueVector().getMetadata().getNamePart().getName();
+              final ValueVector.Accessor accessor = vw.getValueVector().getAccessor();
+              final Object value = i < accessor.getValueCount() ? accessor.getObject(i)
: null;
+              final String display = value == null ? null : value.toString();
+              record.put(field, display);
             }
-            ++j;
+            results.add(record);
           }
-          output.add(record);
         }
-      } else if (!schemaAdded) {
-        columnNames = new ArrayList<>();
-        schemaAdded = true;
-        for (SerializedField fmd : result.getHeader().getDef().getFieldList()) {
-          MaterializedField fieldDef = MaterializedField.create(fmd);
-          columnNames.add(fieldDef.getPath().getAsUnescapedPath());
+      } catch (SchemaChangeException e) {
+        throw new RuntimeException(e);
+      } finally {
+        result.release();
+        if (result.getHeader().getIsLastChunk()) {
+          latch.countDown();
         }
       }
-
-      result.release();
-      if (result.getHeader().getIsLastChunk()) {
-        latch.countDown();
-      }
     }
 
     @Override
     public void queryIdArrived(UserBitShared.QueryId queryId) {
     }
 
-    public List<Map<String, Object>> waitForCompletion() throws Exception {
+    public void waitForCompletion() throws Exception {
       latch.await();
       if (exception != null) {
         throw exception;
       }
-      return output;
-    }
-
-    public List<String> getColumnNames() {
-      return new ArrayList<String> (columnNames);
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/3d863b5e/exec/java-exec/src/main/resources/rest/query/result.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/rest/query/result.ftl b/exec/java-exec/src/main/resources/rest/query/result.ftl
index b69bd6d..7fe52a4 100644
--- a/exec/java-exec/src/main/resources/rest/query/result.ftl
+++ b/exec/java-exec/src/main/resources/rest/query/result.ftl
@@ -33,16 +33,16 @@
     <table id="result" class="table table-striped table-bordered table-condensed" style="table-layout:
auto; width=100%;">
       <thead>
         <tr>
-          <#list model.getColumnNames() as value>
-          <th>${value}</th>
+          <#list model.getColumns() as value>
+          <th>${value?html}</th>
           </#list>
         </tr>
       </thead>
       <tbody>
-      <#list model.getRecords() as record>
+      <#list model.getRows() as record>
         <tr>
           <#list record as value>
-          <td><#if value??>${value}<#else>null</#if></td>
+          <td>${value!"null"?html}</td>
           </#list>
         </tr>
       </#list>


Mime
View raw message