carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [05/50] [abbrv] carbondata git commit: [CARBONDATA-1449]Fixed date and timestamp filter gc issue in case of direct dictionary
Date Tue, 10 Oct 2017 03:07:52 GMT
[CARBONDATA-1449]Fixed date and timestamp filter gc issue in case of direct dictionary

Problem: When date or timestamp filter is going to row level there is lots of gc this is because
for each row it is creating new direct dictionary object for data type;
Solution: create one object and use the same.
Improvement tested with 2million rows:
~19 seconds to ~9 seconds

This closes #1381


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2cd22e1d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2cd22e1d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2cd22e1d

Branch: refs/heads/streaming_ingest
Commit: 2cd22e1d682f29c54a2491deec18314e65a03f4e
Parents: 7f6b08a
Author: kumarvishal <kumarvishal.1802@gmail.com>
Authored: Mon Sep 25 17:06:21 2017 +0530
Committer: Ravindra Pesala <ravi.pesala@gmail.com>
Committed: Tue Sep 26 20:06:50 2017 +0530

----------------------------------------------------------------------
 .../executer/RowLevelFilterExecuterImpl.java    | 27 +++++++++++++++-----
 1 file changed, 21 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/2cd22e1d/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index b79f18d..8f3eb93 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -100,6 +100,16 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
    */
   protected boolean isNaturalSorted;
 
+  /**
+   * date direct dictionary generator
+   */
+  private DirectDictionaryGenerator dateDictionaryGenerator;
+
+  /**
+   * timestamp direct dictionary generator
+   */
+  private DirectDictionaryGenerator timestampDictionaryGenerator;
+
   public RowLevelFilterExecuterImpl(List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
       List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
       AbsoluteTableIdentifier tableIdentifier, SegmentProperties segmentProperties,
@@ -132,6 +142,10 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
     this.exp = exp;
     this.tableIdentifier = tableIdentifier;
     this.complexDimensionInfoMap = complexDimensionInfoMap;
+    this.dateDictionaryGenerator =
+        DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(DataType.DATE);
+    this.timestampDictionaryGenerator =
+        DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(DataType.TIMESTAMP);
     initDimensionBlockIndexes();
     initMeasureBlockIndexes();
   }
@@ -408,13 +422,14 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
    */
   private Object getFilterActualValueFromDirectDictionaryValue(
       DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int dictionaryValue) {
-    Object memberString = null;
-    DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
-        .getDirectDictionaryGenerator(dimColumnEvaluatorInfo.getDimension().getDataType());
-    if (null != directDictionaryGenerator) {
-      memberString = directDictionaryGenerator.getValueFromSurrogate(dictionaryValue);
+    switch (dimColumnEvaluatorInfo.getDimension().getDataType()) {
+      case DATE:
+        return dateDictionaryGenerator.getValueFromSurrogate(dictionaryValue);
+      case TIMESTAMP:
+        return timestampDictionaryGenerator.getValueFromSurrogate(dictionaryValue);
+      default:
+        throw new RuntimeException("Invalid data type for dierct dictionary");
     }
-    return memberString;
   }
 
   /**


Mime
View raw message