carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vimal...@apache.org
Subject [1/2] incubator-carbondata git commit: Fiex Data Mismatch issue in filter query
Date Wed, 31 Aug 2016 02:20:01 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 523d0132a -> be642c972


Fiex Data Mismatch issue in filter query


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/fe56ad0d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/fe56ad0d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/fe56ad0d

Branch: refs/heads/master
Commit: fe56ad0dd1c30008b032af7796b23ebb0628d1aa
Parents: 523d013
Author: kumarvishal <kumarvishal.1802@gmail.com>
Authored: Mon Aug 29 18:25:46 2016 +0530
Committer: Vimal Das Kammath <vkammath@VIMALDK-M01.vmware.com>
Committed: Wed Aug 31 07:43:52 2016 +0530

----------------------------------------------------------------------
 .../carbondata/scan/filter/FilterUtil.java      | 31 +++++++++++++++-----
 .../executer/ExcludeFilterExecuterImpl.java     |  8 ++---
 .../executer/IncludeFilterExecuterImpl.java     |  6 ++--
 .../executer/RestructureFilterExecuterImpl.java |  7 ++---
 .../RowLevelRangeFilterResolverImpl.java        |  2 +-
 5 files changed, 33 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fe56ad0d/core/src/main/java/org/apache/carbondata/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/scan/filter/FilterUtil.java
index a96b0b2..71ac1bf 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/FilterUtil.java
@@ -111,7 +111,7 @@ public final class FilterUtil {
         case RESTRUCTURE:
           return new RestructureFilterExecuterImpl(
               filterExpressionResolverTree.getDimColResolvedFilterInfo(),
-              segmentProperties.getDimensionKeyGenerator());
+              segmentProperties);
         case ROWLEVEL_LESSTHAN:
         case ROWLEVEL_LESSTHAN_EQUALTO:
         case ROWLEVEL_GREATERTHAN_EQUALTO:
@@ -609,12 +609,23 @@ public final class FilterUtil {
     return columnFilterInfo;
   }
 
+  /**
+   * Below method will be used to covert the filter surrogate keys
+   * to mdkey
+   *
+   * @param dimColumnFilterInfo
+   * @param carbonDimension
+   * @param segmentProperties
+   * @return
+   */
   public static byte[][] getKeyArray(DimColumnFilterInfo dimColumnFilterInfo,
-      CarbonDimension carbonDimension, KeyGenerator blockLevelKeyGenerator) {
+      CarbonDimension carbonDimension, SegmentProperties segmentProperties) {
     if (!carbonDimension.hasEncoding(Encoding.DICTIONARY)) {
       return dimColumnFilterInfo.getNoDictionaryFilterValuesList()
           .toArray((new byte[dimColumnFilterInfo.getNoDictionaryFilterValuesList().size()][]));
     }
+    KeyGenerator blockLevelKeyGenerator = segmentProperties.getDimensionKeyGenerator();
+    int[] dimColumnsCardinality = segmentProperties.getDimColumnsCardinality();
     int[] keys = new int[blockLevelKeyGenerator.getDimCount()];
     List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
     Arrays.fill(keys, 0);
@@ -623,9 +634,13 @@ public final class FilterUtil {
     if (null != dimColumnFilterInfo) {
       for (Integer surrogate : dimColumnFilterInfo.getFilterList()) {
         try {
-          keys[carbonDimension.getKeyOrdinal()] = surrogate;
-          filterValuesList
-              .add(getMaskedKey(rangesForMaskedByte, blockLevelKeyGenerator.generateKey(keys)));
+          if (surrogate <= dimColumnsCardinality[carbonDimension.getKeyOrdinal()]) {
+            keys[carbonDimension.getKeyOrdinal()] = surrogate;
+            filterValuesList
+                .add(getMaskedKey(rangesForMaskedByte, blockLevelKeyGenerator.generateKey(keys)));
+          } else {
+            break;
+          }
         } catch (KeyGenException e) {
           LOGGER.error(e.getMessage());
         }
@@ -979,14 +994,14 @@ public final class FilterUtil {
    * API will prepare the Keys from the surrogates of particular filter resolver
    *
    * @param filterValues
-   * @param blockKeyGenerator
+   * @param segmentProperties
    * @param dimension
    * @param dimColumnExecuterInfo
    */
   public static void prepareKeysFromSurrogates(DimColumnFilterInfo filterValues,
-      KeyGenerator blockKeyGenerator, CarbonDimension dimension,
+      SegmentProperties segmentProperties, CarbonDimension dimension,
       DimColumnExecuterFilterInfo dimColumnExecuterInfo) {
-    byte[][] keysBasedOnFilter = getKeyArray(filterValues, dimension, blockKeyGenerator);
+    byte[][] keysBasedOnFilter = getKeyArray(filterValues, dimension, segmentProperties);
     dimColumnExecuterInfo.setFilterKeys(keysBasedOnFilter);
 
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fe56ad0d/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeFilterExecuterImpl.java
b/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeFilterExecuterImpl.java
index 5ebae79..db4326b 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeFilterExecuterImpl.java
@@ -42,9 +42,8 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
     this.dimColEvaluatorInfo = dimColEvaluatorInfo;
     dimColumnExecuterInfo = new DimColumnExecuterFilterInfo();
     this.segmentProperties = segmentProperties;
-    FilterUtil.prepareKeysFromSurrogates(dimColEvaluatorInfo.getFilterValues(),
-        segmentProperties.getDimensionKeyGenerator(), dimColEvaluatorInfo.getDimension(),
-        dimColumnExecuterInfo);
+    FilterUtil.prepareKeysFromSurrogates(dimColEvaluatorInfo.getFilterValues(), segmentProperties,
+        dimColEvaluatorInfo.getDimension(), dimColumnExecuterInfo);
   }
 
   @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder) {
@@ -58,8 +57,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
       blockChunkHolder.getDimensionDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
           .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
     }
-    return getFilteredIndexes(
-        blockChunkHolder.getDimensionDataChunk()[blockIndex],
+    return getFilteredIndexes(blockChunkHolder.getDimensionDataChunk()[blockIndex],
         blockChunkHolder.getDataBlock().nodeSize());
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fe56ad0d/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeFilterExecuterImpl.java
b/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeFilterExecuterImpl.java
index 8e645b3..f399aa9 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeFilterExecuterImpl.java
@@ -42,9 +42,9 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
     this.dimColumnEvaluatorInfo = dimColumnEvaluatorInfo;
     this.segmentProperties = segmentProperties;
     dimColumnExecuterInfo = new DimColumnExecuterFilterInfo();
-    FilterUtil.prepareKeysFromSurrogates(dimColumnEvaluatorInfo.getFilterValues(),
-        segmentProperties.getDimensionKeyGenerator(), dimColumnEvaluatorInfo.getDimension(),
-        dimColumnExecuterInfo);
+    FilterUtil
+        .prepareKeysFromSurrogates(dimColumnEvaluatorInfo.getFilterValues(), segmentProperties,
+            dimColumnEvaluatorInfo.getDimension(), dimColumnExecuterInfo);
 
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fe56ad0d/core/src/main/java/org/apache/carbondata/scan/filter/executer/RestructureFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RestructureFilterExecuterImpl.java
b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RestructureFilterExecuterImpl.java
index 84d7898..343fd03 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RestructureFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RestructureFilterExecuterImpl.java
@@ -20,21 +20,20 @@ package org.apache.carbondata.scan.filter.executer;
 
 import java.util.BitSet;
 
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
+import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties;
 import org.apache.carbondata.scan.filter.FilterUtil;
 import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
 import org.apache.carbondata.scan.processor.BlocksChunkHolder;
 
-
 public class RestructureFilterExecuterImpl implements FilterExecuter {
 
   DimColumnExecuterFilterInfo dimColumnExecuterInfo;
 
   public RestructureFilterExecuterImpl(DimColumnResolvedFilterInfo dimColumnResolvedFilterInfo,
-      KeyGenerator blockKeyGenerator) {
+      SegmentProperties segmentProperties) {
     dimColumnExecuterInfo = new DimColumnExecuterFilterInfo();
     FilterUtil
-        .prepareKeysFromSurrogates(dimColumnResolvedFilterInfo.getFilterValues(), blockKeyGenerator,
+        .prepareKeysFromSurrogates(dimColumnResolvedFilterInfo.getFilterValues(), segmentProperties,
             dimColumnResolvedFilterInfo.getDimension(), dimColumnExecuterInfo);
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fe56ad0d/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
index 0f9b47d..f07fe98 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
@@ -88,7 +88,7 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
         .get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
       return FilterUtil.getKeyArray(this.dimColEvaluatorInfoList.get(0).getFilterValues(),
           this.dimColEvaluatorInfoList.get(0).getDimension(),
-          segmentProperties.getDimensionKeyGenerator());
+          segmentProperties);
     }
     return null;
 


Mime
View raw message