drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject [27/50] [abbrv] drill git commit: MD-703: Cannot select `_id` field from MapR-DB JSON Table
Date Tue, 13 Sep 2016 01:32:14 GMT
MD-703: Cannot select `_id` field from MapR-DB JSON Table

+ Included Smidth's fix for column shuffling.
+ Fix the private-drill-mapr-plugin build.


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/5d36f801
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/5d36f801
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/5d36f801

Branch: refs/heads/master
Commit: 5d36f8013829406f698610bc4a4f7185698365e6
Parents: 16d4ab8
Author: Aditya <aditya@mapr.com>
Authored: Thu Feb 4 16:43:27 2016 -0800
Committer: Aditya Kishore <adi@apache.org>
Committed: Fri Sep 9 10:08:35 2016 -0700

----------------------------------------------------------------------
 contrib/format-maprdb/pom.xml                   |  6 +++++
 .../exec/store/maprdb/MapRDBGroupScan.java      |  2 +-
 .../store/maprdb/MapRDBPushFilterIntoScan.java  |  4 ++--
 .../maprdb/json/MaprDBJsonRecordReader.java     | 24 ++++++++++++--------
 .../drill/maprdb/tests/json/TestSimpleJson.java |  4 ++--
 5 files changed, 26 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/5d36f801/contrib/format-maprdb/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/pom.xml b/contrib/format-maprdb/pom.xml
index bcb6c29..db0572f 100644
--- a/contrib/format-maprdb/pom.xml
+++ b/contrib/format-maprdb/pom.xml
@@ -35,6 +35,7 @@
     <mapr.version>${project.parent.version}</mapr.version>
     <drill.version>${project.version}</drill.version>
     <hbase.version>0.98.12-mapr-1506</hbase.version>
+    <guava.version>18.0</guava.version>
     <maprdb.TestSuite>**/MaprDBTestsSuite.class</maprdb.TestSuite>
   </properties>
 
@@ -118,6 +119,11 @@
       </exclusions>
     </dependency>
     <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <version>${guava.version}</version>
+    </dependency>
+    <dependency>
       <groupId>org.apache.drill.exec</groupId>
       <artifactId>drill-java-exec</artifactId>
       <version>${drill.version}</version>

http://git-wip-us.apache.org/repos/asf/drill/blob/5d36f801/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java
index cbfb18c..4d30399 100644
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBGroupScan.java
@@ -61,7 +61,7 @@ public abstract class MapRDBGroupScan extends AbstractGroupScan {
 
   private boolean filterPushedDown = false;
 
-  private Stopwatch watch = new Stopwatch();
+  private Stopwatch watch = Stopwatch.createUnstarted();
 
   private static final Comparator<List<MapRDBSubScanSpec>> LIST_SIZE_COMPARATOR
= new Comparator<List<MapRDBSubScanSpec>>() {
     @Override

http://git-wip-us.apache.org/repos/asf/drill/blob/5d36f801/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java
index 714221f..8a0902e 100644
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java
@@ -90,11 +90,11 @@ public abstract class MapRDBPushFilterIntoScan extends StoragePluginOptimizerRul
 
       if (scan.getGroupScan() instanceof BinaryTableGroupScan) {
         BinaryTableGroupScan groupScan = (BinaryTableGroupScan)scan.getGroupScan();
-        doPushFilterIntoBinaryGroupScan(call, filter, null, scan, groupScan, condition);
+        doPushFilterIntoBinaryGroupScan(call, filter, project, scan, groupScan, condition);
       } else {
         assert(scan.getGroupScan() instanceof JsonTableGroupScan);
         JsonTableGroupScan groupScan = (JsonTableGroupScan)scan.getGroupScan();
-        doPushFilterIntoJsonGroupScan(call, filter, null, scan, groupScan, condition);
+        doPushFilterIntoJsonGroupScan(call, filter, project, scan, groupScan, condition);
       }
     }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/5d36f801/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java
index 2e0e0c1..b69f4c4 100644
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java
@@ -17,11 +17,9 @@
  */
 package org.apache.drill.exec.store.maprdb.json;
 
-import static org.ojai.DocumentConstants.ID_FIELD;
 import static org.ojai.DocumentConstants.ID_KEY;
 
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
@@ -51,7 +49,6 @@ import org.ojai.store.QueryCondition;
 
 import com.google.common.base.Preconditions;
 import com.google.common.base.Stopwatch;
-import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import com.mapr.db.MapRDB;
 import com.mapr.db.Table;
@@ -98,18 +95,27 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader {
   protected Collection<SchemaPath> transformColumns(Collection<SchemaPath> columns)
{
     Set<SchemaPath> transformed = Sets.newLinkedHashSet();
     if (!isStarQuery()) {
-      ArrayList<Object> projectedFieldsList = Lists.newArrayList();
+      Set<FieldPath> projectedFieldsSet = Sets.newTreeSet();
       for (SchemaPath column : columns) {
         if (column.getRootSegment().getPath().equalsIgnoreCase(ID_KEY)) {
+          /*
+           * we do not include _id field in the set of projected fields
+           * because the DB currently can not return a document if only
+           * the _id field was projected. This should really be fixed in
+           * the DB client (Bug 21708) to avoid transferring the entire
+           * document when only _id is requested.
+           */
+          // projectedFieldsList.add(ID_FIELD);
           transformed.add(ID_PATH);
-          projectedFieldsList.add(ID_FIELD);
           includeId = true;
         } else {
           transformed.add(SchemaPath.getSimplePath(column.getRootSegment().getPath()));
-          projectedFieldsList.add(FieldPath.parseFrom(column.getAsUnescapedPath()));
+          projectedFieldsSet.add(FieldPath.parseFrom(column.getAsUnescapedPath()));
         }
       }
-      projectedFields = projectedFieldsList.toArray(new FieldPath[projectedFieldsList.size()]);
+      if (projectedFieldsSet.size() > 0) {
+        projectedFields = projectedFieldsSet.toArray(new FieldPath[projectedFieldsSet.size()]);
+      }
     } else {
       transformed.add(ID_PATH);
       includeId = true;
@@ -135,7 +141,7 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader {
 
   @Override
   public int next() {
-    Stopwatch watch = new Stopwatch();
+    Stopwatch watch = Stopwatch.createUnstarted();
     watch.start();
 
     writer.allocate();
@@ -230,7 +236,7 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader {
         default:
           throw unsupportedError("Unsupported type: %s encountered during the query.", event);
         }
-      } catch (IllegalArgumentException e) {
+      } catch (IllegalStateException | IllegalArgumentException e) {
         logger.warn(String.format("Possible schema change at _id: '%s', field: '%s'",
             IdCodec.asString(reader.getId()), fieldName), e);
       }

http://git-wip-us.apache.org/repos/asf/drill/blob/5d36f801/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java
b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java
index f05b87a..414b823 100644
--- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java
+++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java
@@ -48,9 +48,9 @@ public class TestSimpleJson extends BaseTestQuery {
 
   @Test
   public void testMe() throws Exception {
-    setColumnWidths(new int[] {25, 40, 40, 40});
+    setColumnWidths(new int[] {23});
     final String sql = "SELECT\n"
-        + "  _id, name, categories, full_address\n"
+        + "  _id\n"
         + "FROM\n"
         + "  hbase.`business` business";
     runSQLAndVerifyCount(sql, 10);


Mime
View raw message