drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sudhe...@apache.org
Subject [14/14] drill git commit: DRILL-5065: Optimize count(*) queries on MapR-DB JSON Tables
Date Tue, 13 Dec 2016 00:40:11 GMT
DRILL-5065: Optimize count(*) queries on MapR-DB JSON Tables

In MapR-DB v5.2.0, we enabled '_id' only projection for JSON
tables. Hence, we can now optimize the following queries:

a. count(*) by projecting only the '_id' column.

b. '_id' only projections, including count(_id)

Change the format plugin config parameter name.

Fix setter of config parameter `disableCountOptimization` for drill-maprdb plugin

closes #678


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/97da0199
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/97da0199
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/97da0199

Branch: refs/heads/master
Commit: 97da01995ecfd9a7d157ad183f0656cd15993f26
Parents: 94f3ba0
Author: Smidth Panchamia <spanchamia@mapr.com>
Authored: Mon Nov 28 13:59:34 2016 -0800
Committer: Sudheesh Katkam <sudheesh@apache.org>
Committed: Mon Dec 12 15:59:59 2016 -0800

----------------------------------------------------------------------
 .../store/mapr/db/MapRDBFormatPluginConfig.java | 12 ++++
 .../mapr/db/json/MaprDBJsonRecordReader.java    | 64 ++++++++++++--------
 2 files changed, 51 insertions(+), 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/97da0199/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPluginConfig.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPluginConfig.java
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPluginConfig.java
index 1bb07ed..8b89b78 100644
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPluginConfig.java
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPluginConfig.java
@@ -31,6 +31,7 @@ public class MapRDBFormatPluginConfig extends TableFormatPluginConfig {
   public boolean enablePushdown = true;
   public boolean ignoreSchemaChange = false;
   public boolean readAllNumbersAsDouble = false;
+  public boolean disableCountOptimization = false;
 
   @Override
   public int hashCode() {
@@ -48,6 +49,8 @@ public class MapRDBFormatPluginConfig extends TableFormatPluginConfig {
       return false;
     } else if (enablePushdown != other.enablePushdown) {
       return false;
+    } else if (disableCountOptimization != other.disableCountOptimization) {
+      return false;
     }
     return true;
   }
@@ -65,6 +68,15 @@ public class MapRDBFormatPluginConfig extends TableFormatPluginConfig {
     allTextMode = mode;
   }
 
+  @JsonProperty("disableCountOptimization")
+  public void setDisableCountOptimization(boolean mode) {
+    disableCountOptimization = mode;
+  }
+
+  public boolean shouldDisableCountOptimization() {
+    return disableCountOptimization;
+  }
+
   @JsonProperty("readAllNumbersAsDouble")
   public void setReadAllNumbersAsDouble(boolean read) {
     readAllNumbersAsDouble = read;

http://git-wip-us.apache.org/repos/asf/drill/blob/97da0199/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/MaprDBJsonRecordReader.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/MaprDBJsonRecordReader.java
b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/MaprDBJsonRecordReader.java
index ee6b15d..3105bec 100644
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/MaprDBJsonRecordReader.java
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/MaprDBJsonRecordReader.java
@@ -18,6 +18,7 @@
 package org.apache.drill.exec.store.mapr.db.json;
 
 import static org.ojai.DocumentConstants.ID_KEY;
+import static org.ojai.DocumentConstants.ID_FIELD;
 
 import java.nio.ByteBuffer;
 import java.util.Collection;
@@ -93,6 +94,7 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader {
   private boolean disablePushdown;
   private final boolean allTextMode;
   private final boolean ignoreSchemaChange;
+  private final boolean disableCountOptimization;
 
   public MaprDBJsonRecordReader(MapRDBSubScanSpec subScanSpec,
       MapRDBFormatPluginConfig formatPluginConfig,
@@ -110,6 +112,7 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader {
       condition = com.mapr.db.impl.ConditionImpl.parseFrom(ByteBufs.wrap(serializedFilter));
     }
 
+    disableCountOptimization = formatPluginConfig.shouldDisableCountOptimization();
     setColumns(projectedColumns);
     unionEnabled = context.getOptions().getOption(ExecConstants.ENABLE_UNION_TYPE);
     readNumbersAsDouble = formatPluginConfig.isReadAllNumbersAsDouble();
@@ -121,36 +124,47 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader {
   @Override
   protected Collection<SchemaPath> transformColumns(Collection<SchemaPath> columns)
{
     Set<SchemaPath> transformed = Sets.newLinkedHashSet();
-    if (!isStarQuery() && !disablePushdown) {
-      Set<FieldPath> projectedFieldsSet = Sets.newTreeSet();
-      for (SchemaPath column : columns) {
-        if (column.getRootSegment().getPath().equalsIgnoreCase(ID_KEY)) {
-          /*
-           * we do not include _id field in the set of projected fields
-           * because the DB currently can not return a document if only
-           * the _id field was projected. This should really be fixed in
-           * the DB client (Bug 21708) to avoid transferring the entire
-           * document when only _id is requested.
-           */
-          // projectedFieldsList.add(ID_FIELD);
-          includeId = true;
-        } else {
-          projectedFieldsSet.add(getFieldPathForProjection(column));
+    if (disablePushdown) {
+      transformed.add(AbstractRecordReader.STAR_COLUMN);
+      includeId = true;
+      return transformed;
+    }
+
+    if (isStarQuery()) {
+      transformed.add(AbstractRecordReader.STAR_COLUMN);
+      includeId = true;
+      if (isSkipQuery()) {
+    	// `SELECT COUNT(*)` query
+    	if (!disableCountOptimization) {
+          projectedFields = new FieldPath[1];
+          projectedFields[0] = ID_FIELD;
         }
-        transformed.add(column);
       }
-      if (projectedFieldsSet.size() > 0) {
-        projectedFields = projectedFieldsSet.toArray(new FieldPath[projectedFieldsSet.size()]);
+      return transformed;
+    }
+
+    Set<FieldPath> projectedFieldsSet = Sets.newTreeSet();
+    for (SchemaPath column : columns) {
+      if (column.getRootSegment().getPath().equalsIgnoreCase(ID_KEY)) {
+        includeId = true;
+        if (!disableCountOptimization) {
+          projectedFieldsSet.add(ID_FIELD);
+        }
+      } else {
+        projectedFieldsSet.add(getFieldPathForProjection(column));
       }
-    } else {
-      transformed.add(AbstractRecordReader.STAR_COLUMN);
-      includeId = true;
+
+      transformed.add(column);
+    }
+
+    if (projectedFieldsSet.size() > 0) {
+      projectedFields = projectedFieldsSet.toArray(new FieldPath[projectedFieldsSet.size()]);
+    }
+
+    if (disableCountOptimization) {
+      idOnly = (projectedFields == null);
     }
 
-    /*
-     * (Bug 21708) if we are projecting only the id field, save that condition here.
-     */
-    idOnly = !isStarQuery() && (projectedFields == null);
     return transformed;
   }
 


Mime
View raw message