carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gvram...@apache.org
Subject [04/14] incubator-carbondata git commit: Following changes are done as part of this commit.
Date Thu, 16 Mar 2017 09:28:47 GMT
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
index 27377c2..f188a67 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
@@ -26,6 +26,7 @@ import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
 import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
 import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
 import org.apache.carbondata.core.scan.expression.Expression;
 import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
 import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
@@ -39,6 +40,11 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
 
   protected byte[][] filterRangeValues;
 
+  /**
+   * flag to check whether default values is present in the filter value list
+   */
+  private boolean isDefaultValuePresentInFilter;
+
   public RowLevelRangeGrtrThanEquaToFilterExecuterImpl(
       List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
       List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
@@ -47,44 +53,71 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
     super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties,
         null);
     this.filterRangeValues = filterRangeValues;
+    checkIfDefaultValueIsPresentInFilterList();
+  }
+
+  /**
+   * This method will check whether default value is present in the given filter values
+   */
+  private void checkIfDefaultValueIsPresentInFilterList() {
+    if (!this.isDimensionPresentInCurrentBlock[0]) {
+      CarbonDimension dimension = this.dimColEvaluatorInfoList.get(0).getDimension();
+      byte[] defaultValue = dimension.getDefaultValue();
+      if (null != defaultValue) {
+        for (int k = 0; k < filterRangeValues.length; k++) {
+          int maxCompare =
+              ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterRangeValues[k], defaultValue);
+          if (maxCompare <= 0) {
+            isDefaultValuePresentInFilter = true;
+            break;
+          }
+        }
+      }
+    }
   }
 
   @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
     BitSet bitSet = new BitSet(1);
-    byte[][] filterValues = this.filterRangeValues;
-    int columnIndex = this.dimColEvaluatorInfoList.get(0).getColumnIndex();
-    boolean isScanRequired = isScanRequired(blockMaxValue[columnIndex], filterValues);
+    boolean isScanRequired =
+        isScanRequired(blockMaxValue[dimensionBlocksIndex[0]], filterRangeValues);
     if (isScanRequired) {
       bitSet.set(0);
     }
     return bitSet;
-
   }
 
   private boolean isScanRequired(byte[] blockMaxValue, byte[][] filterValues) {
     boolean isScanRequired = false;
-    for (int k = 0; k < filterValues.length; k++) {
-      // filter value should be in range of max and min value i.e
-      // max>filtervalue>min
-      // so filter-max should be negative
-      int maxCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue);
-      // if any filter value is in range than this block needs to be
-      // scanned less than equal to max range.
-      if (maxCompare <= 0) {
-        isScanRequired = true;
-        break;
+    if (isDimensionPresentInCurrentBlock[0]) {
+      for (int k = 0; k < filterValues.length; k++) {
+        // filter value should be in range of max and min value i.e
+        // max>filtervalue>min
+        // so filter-max should be negative
+        int maxCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue);
+        // if any filter value is in range than this block needs to be
+        // scanned less than equal to max range.
+        if (maxCompare <= 0) {
+          isScanRequired = true;
+          break;
+        }
       }
+    } else {
+      isScanRequired = isDefaultValuePresentInFilter;
     }
     return isScanRequired;
   }
 
   @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
       throws FilterUnsupportedException, IOException {
+    // select all rows if dimension does not exists in the current block
+    if (!isDimensionPresentInCurrentBlock[0]) {
+      return getDefaultBitSetGroup(blockChunkHolder.getDataBlock().nodeSize());
+    }
     if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
       return super.applyFilter(blockChunkHolder);
     }
     int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
-        .get(dimColEvaluatorInfoList.get(0).getColumnIndex());
+        .get(dimensionBlocksIndex[0]);
     if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
       blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
           .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
@@ -225,14 +258,15 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
   }
 
   @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
-    if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
-      super.readBlocks(blockChunkHolder);
-    }
-    int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
-        .get(dimColEvaluatorInfoList.get(0).getColumnIndex());
-    if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
-      blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
-          .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+    if (isDimensionPresentInCurrentBlock[0]) {
+      if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
+        super.readBlocks(blockChunkHolder);
+      }
+      int blockIndex = dimensionBlocksIndex[0];
+      if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
+        blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+            .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+      }
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index c7e2acc..f455acf 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -28,6 +28,7 @@ import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionary
 import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
 import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
 import org.apache.carbondata.core.scan.expression.Expression;
 import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
 import org.apache.carbondata.core.scan.filter.FilterUtil;
@@ -41,6 +42,11 @@ import org.apache.carbondata.core.util.CarbonUtil;
 public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilterExecuterImpl {
   protected byte[][] filterRangeValues;
 
+  /**
+   * flag to check whether default values is present in the filter value list
+   */
+  private boolean isDefaultValuePresentInFilter;
+
   public RowLevelRangeLessThanEqualFilterExecuterImpl(
       List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
       List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
@@ -49,13 +55,33 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
     super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties,
         null);
     this.filterRangeValues = filterRangeValues;
+    checkIfDefaultValueIsPresentInFilterList();
+  }
+
+  /**
+   * This method will check whether default value is present in the given filter values
+   */
+  private void checkIfDefaultValueIsPresentInFilterList() {
+    if (!this.isDimensionPresentInCurrentBlock[0]) {
+      CarbonDimension dimension = this.dimColEvaluatorInfoList.get(0).getDimension();
+      byte[] defaultValue = dimension.getDefaultValue();
+      if (null != defaultValue) {
+        for (int k = 0; k < filterRangeValues.length; k++) {
+          int maxCompare =
+              ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterRangeValues[k], defaultValue);
+          if (maxCompare >= 0) {
+            isDefaultValuePresentInFilter = true;
+            break;
+          }
+        }
+      }
+    }
   }
 
   @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
     BitSet bitSet = new BitSet(1);
-    byte[][] filterValues = this.filterRangeValues;
-    int columnIndex = this.dimColEvaluatorInfoList.get(0).getColumnIndex();
-    boolean isScanRequired = isScanRequired(blockMinValue[columnIndex], filterValues);
+    boolean isScanRequired =
+        isScanRequired(blockMinValue[dimensionBlocksIndex[0]], filterRangeValues);
     if (isScanRequired) {
       bitSet.set(0);
     }
@@ -64,28 +90,36 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
 
   private boolean isScanRequired(byte[] blockMinValue, byte[][] filterValues) {
     boolean isScanRequired = false;
-    for (int k = 0; k < filterValues.length; k++) {
-      // and filter-min should be positive
-      int minCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMinValue);
+    if (isDimensionPresentInCurrentBlock[0]) {
+      for (int k = 0; k < filterValues.length; k++) {
+        // and filter-min should be positive
+        int minCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMinValue);
 
-      // if any filter applied is not in range of min and max of block
-      // then since its a less than equal to fiter validate whether the block
-      // min range is less than equal to applied filter member
-      if (minCompare >= 0) {
-        isScanRequired = true;
-        break;
+        // if any filter applied is not in range of min and max of block
+        // then since its a less than equal to fiter validate whether the block
+        // min range is less than equal to applied filter member
+        if (minCompare >= 0) {
+          isScanRequired = true;
+          break;
+        }
       }
+    } else {
+      isScanRequired = isDefaultValuePresentInFilter;
     }
     return isScanRequired;
   }
 
   @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
       throws FilterUnsupportedException, IOException {
+    // select all rows if dimension does not exists in the current block
+    if (!isDimensionPresentInCurrentBlock[0]) {
+      return getDefaultBitSetGroup(blockChunkHolder.getDataBlock().nodeSize());
+    }
     if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
       return super.applyFilter(blockChunkHolder);
     }
     int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
-        .get(dimColEvaluatorInfoList.get(0).getColumnIndex());
+        .get(dimensionBlocksIndex[0]);
     if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
       blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
           .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
@@ -117,7 +151,9 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
           .getDirectDictionaryGenerator(
               dimColEvaluatorInfoList.get(0).getDimension().getDataType());
       int key = directDictionaryGenerator.generateDirectSurrogateKey(null) + 1;
-      defaultValue = FilterUtil.getMaskKey(key, dimColEvaluatorInfoList.get(0).getDimension(),
+      CarbonDimension currentBlockDimension =
+          segmentProperties.getDimensions().get(dimensionBlocksIndex[0]);
+      defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension,
           this.segmentProperties.getDimensionKeyGenerator());
     }
     if (dimensionColumnDataChunk.isExplicitSorted()
@@ -264,14 +300,15 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
   }
 
   @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
-    if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
-      super.readBlocks(blockChunkHolder);
-    }
-    int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
-        .get(dimColEvaluatorInfoList.get(0).getColumnIndex());
-    if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
-      blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
-          .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+    if (isDimensionPresentInCurrentBlock[0]) {
+      if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
+        super.readBlocks(blockChunkHolder);
+      }
+      int blockIndex = dimensionBlocksIndex[0];
+      if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
+        blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+            .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+      }
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
index d9795eb..69c0538 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
@@ -28,6 +28,7 @@ import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionary
 import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
 import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
 import org.apache.carbondata.core.scan.expression.Expression;
 import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
 import org.apache.carbondata.core.scan.filter.FilterUtil;
@@ -41,6 +42,11 @@ import org.apache.carbondata.core.util.CarbonUtil;
 public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecuterImpl {
   private byte[][] filterRangeValues;
 
+  /**
+   * flag to check whether default values is present in the filter value list
+   */
+  private boolean isDefaultValuePresentInFilter;
+
   public RowLevelRangeLessThanFiterExecuterImpl(
       List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
       List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
@@ -49,13 +55,33 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
     super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties,
         null);
     this.filterRangeValues = filterRangeValues;
+    checkIfDefaultValueIsPresentInFilterList();
+  }
+
+  /**
+   * This method will check whether default value is present in the given filter values
+   */
+  private void checkIfDefaultValueIsPresentInFilterList() {
+    if (!this.isDimensionPresentInCurrentBlock[0]) {
+      CarbonDimension dimension = this.dimColEvaluatorInfoList.get(0).getDimension();
+      byte[] defaultValue = dimension.getDefaultValue();
+      if (null != defaultValue) {
+        for (int k = 0; k < filterRangeValues.length; k++) {
+          int maxCompare =
+              ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterRangeValues[k], defaultValue);
+          if (maxCompare > 0) {
+            isDefaultValuePresentInFilter = true;
+            break;
+          }
+        }
+      }
+    }
   }
 
   @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
     BitSet bitSet = new BitSet(1);
-    byte[][] filterValues = this.filterRangeValues;
-    int columnIndex = this.dimColEvaluatorInfoList.get(0).getColumnIndex();
-    boolean isScanRequired = isScanRequired(blockMinValue[columnIndex], filterValues);
+    boolean isScanRequired =
+        isScanRequired(blockMinValue[dimensionBlocksIndex[0]], filterRangeValues);
     if (isScanRequired) {
       bitSet.set(0);
     }
@@ -64,28 +90,36 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
 
   private boolean isScanRequired(byte[] blockMinValue, byte[][] filterValues) {
     boolean isScanRequired = false;
-    for (int k = 0; k < filterValues.length; k++) {
-      // and filter-min should be positive
-      int minCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMinValue);
+    if (isDimensionPresentInCurrentBlock[0]) {
+      for (int k = 0; k < filterValues.length; k++) {
+        // and filter-min should be positive
+        int minCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMinValue);
 
-      // if any filter applied is not in range of min and max of block
-      // then since its a less than fiter validate whether the block
-      // min range is less  than applied filter member
-      if (minCompare > 0) {
-        isScanRequired = true;
-        break;
+        // if any filter applied is not in range of min and max of block
+        // then since its a less than fiter validate whether the block
+        // min range is less  than applied filter member
+        if (minCompare > 0) {
+          isScanRequired = true;
+          break;
+        }
       }
+    } else {
+      isScanRequired = isDefaultValuePresentInFilter;
     }
     return isScanRequired;
   }
 
   @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
       throws FilterUnsupportedException, IOException {
+    // select all rows if dimension does not exists in the current block
+    if (!isDimensionPresentInCurrentBlock[0]) {
+      return getDefaultBitSetGroup(blockChunkHolder.getDataBlock().nodeSize());
+    }
     if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
       return super.applyFilter(blockChunkHolder);
     }
     int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
-        .get(dimColEvaluatorInfoList.get(0).getColumnIndex());
+        .get(dimensionBlocksIndex[0]);
     if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
       blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
           .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
@@ -117,7 +151,9 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
           .getDirectDictionaryGenerator(
               dimColEvaluatorInfoList.get(0).getDimension().getDataType());
       int key = directDictionaryGenerator.generateDirectSurrogateKey(null) + 1;
-      defaultValue = FilterUtil.getMaskKey(key, dimColEvaluatorInfoList.get(0).getDimension(),
+      CarbonDimension currentBlockDimension =
+          segmentProperties.getDimensions().get(dimensionBlocksIndex[0]);
+      defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension,
           this.segmentProperties.getDimensionKeyGenerator());
     }
     if (dimensionColumnDataChunk.isExplicitSorted()
@@ -272,14 +308,15 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
   }
 
   @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
-    if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
-      super.readBlocks(blockChunkHolder);
-    }
-    int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
-        .get(dimColEvaluatorInfoList.get(0).getColumnIndex());
-    if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
-      blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
-          .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+    if (isDimensionPresentInCurrentBlock[0]) {
+      if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
+        super.readBlocks(blockChunkHolder);
+      }
+      int blockIndex = dimensionBlocksIndex[0];
+      if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
+        blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+            .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+      }
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
index 549dcfd..69560c3 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
@@ -185,12 +185,12 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
   /**
    * method will calculates the start key based on the filter surrogates
    */
-  public void getStartKey(long[] startKey,
+  public void getStartKey(SegmentProperties segmentProperties, long[] startKey,
       SortedMap<Integer, byte[]> setOfStartKeyByteArray, List<long[]> startKeyList) {
     FilterUtil.getStartKey(dimColResolvedFilterInfo.getDimensionResolvedFilterInstance(),
-        startKey, startKeyList);
+        segmentProperties, startKey, startKeyList);
     FilterUtil.getStartKeyForNoDictionaryDimension(dimColResolvedFilterInfo,
-        setOfStartKeyByteArray);
+        segmentProperties, setOfStartKeyByteArray);
   }
 
   /**
@@ -200,7 +200,7 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
       SortedMap<Integer, byte[]> setOfEndKeyByteArray, List<long[]> endKeyList) {
     FilterUtil.getEndKey(dimColResolvedFilterInfo.getDimensionResolvedFilterInstance(), endKeys,
         segmentProperties, endKeyList);
-    FilterUtil.getEndKeyForNoDictionaryDimension(dimColResolvedFilterInfo,
+    FilterUtil.getEndKeyForNoDictionaryDimension(dimColResolvedFilterInfo, segmentProperties,
         setOfEndKeyByteArray);
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java
index 72237cb..746b96d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java
@@ -68,11 +68,13 @@ public interface FilterResolverIntf extends Serializable {
   /**
    * API will get the start key based on the filter applied based on the key generator
    *
+   * @param segmentProperties
    * @param startKey
    * @param setOfStartKeyByteArray
+   * @param startKeyList
    */
-  void getStartKey(long[] startKey, SortedMap<Integer, byte[]> setOfStartKeyByteArray,
-      List<long[]> startKeyList);
+  void getStartKey(SegmentProperties segmentProperties, long[] startKey,
+      SortedMap<Integer, byte[]> setOfStartKeyByteArray, List<long[]> startKeyList);
 
   /**
    * API will read the end key based on the max surrogate of

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java
index 5b3221a..db35823 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java
@@ -84,9 +84,8 @@ public class LogicalFilterResolverImpl implements FilterResolverIntf {
     return null;
   }
 
-  @Override
-  public void getStartKey(long[] startKey, SortedMap<Integer, byte[]> setOfStartKeyByteArray,
-      List<long[]> startKeyList) {
+  @Override public void getStartKey(SegmentProperties segmentProperties, long[] startKey,
+      SortedMap<Integer, byte[]> setOfStartKeyByteArray, List<long[]> startKeyList) {
 
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
index 0e2ed22..acb3f8b 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
@@ -68,6 +68,7 @@ public class RowLevelFilterResolverImpl extends ConditionalFilterResolverImpl {
           dimColEvaluatorInfoList.add(dimColumnEvaluatorInfo);
         } else {
           msrColumnEvalutorInfo = new MeasureColumnResolvedFilterInfo();
+          msrColumnEvalutorInfo.setCarbonColumn(columnExpression.getCarbonColumn());
           msrColumnEvalutorInfo.setRowIndex(index++);
           msrColumnEvalutorInfo
               .setColumnIndex(columnExpression.getCarbonColumn().getOrdinal());

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
index af5568e..8a91499 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
@@ -29,6 +29,7 @@ import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionary
 import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
 import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
 import org.apache.carbondata.core.scan.expression.ColumnExpression;
 import org.apache.carbondata.core.scan.expression.Expression;
 import org.apache.carbondata.core.scan.expression.ExpressionResult;
@@ -42,6 +43,7 @@ import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
 import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
 import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
 import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.CarbonUtil;
 
 public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverImpl {
 
@@ -78,9 +80,13 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
       return noDictFilterValuesList.toArray((new byte[noDictFilterValuesList.size()][]));
     } else if (null != dimColEvaluatorInfoList.get(0).getFilterValues() && dimColEvaluatorInfoList
         .get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
-      return FilterUtil.getKeyArray(this.dimColEvaluatorInfoList.get(0).getFilterValues(),
-          this.dimColEvaluatorInfoList.get(0).getDimension(),
-          segmentProperties);
+      CarbonDimension dimensionFromCurrentBlock = CarbonUtil
+          .getDimensionFromCurrentBlock(segmentProperties.getDimensions(),
+              this.dimColEvaluatorInfoList.get(0).getDimension());
+      if (null != dimensionFromCurrentBlock) {
+        return FilterUtil.getKeyArray(this.dimColEvaluatorInfoList.get(0).getFilterValues(),
+            dimensionFromCurrentBlock, segmentProperties);
+      }
     }
     return null;
 
@@ -91,15 +97,16 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
    *
    * @return start IndexKey
    */
-  public void getStartKey(long[] startKey, SortedMap<Integer, byte[]> noDictStartKeys,
-      List<long[]> startKeyList) {
+  public void getStartKey(SegmentProperties segmentProperties, long[] startKey,
+      SortedMap<Integer, byte[]> noDictStartKeys, List<long[]> startKeyList) {
     switch (exp.getFilterExpressionType()) {
       case GREATERTHAN:
       case GREATERTHAN_EQUALTO:
         FilterUtil.getStartKey(dimColEvaluatorInfoList.get(0).getDimensionResolvedFilterInstance(),
-            startKey, startKeyList);
+            segmentProperties, startKey, startKeyList);
         FilterUtil
-            .getStartKeyForNoDictionaryDimension(dimColEvaluatorInfoList.get(0), noDictStartKeys);
+            .getStartKeyForNoDictionaryDimension(dimColEvaluatorInfoList.get(0), segmentProperties,
+                noDictStartKeys);
         break;
       default:
         //do nothing
@@ -119,7 +126,9 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
         FilterUtil
             .getEndKey(dimColEvaluatorInfoList.get(0).getDimensionResolvedFilterInstance(), endKeys,
                 segmentProperties, endKeyList);
-        FilterUtil.getEndKeyForNoDictionaryDimension(dimColEvaluatorInfoList.get(0), noDicEndKeys);
+        FilterUtil
+            .getEndKeyForNoDictionaryDimension(dimColEvaluatorInfoList.get(0), segmentProperties,
+                noDicEndKeys);
         break;
       default:
         //do nothing

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/DimColumnResolvedFilterInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/DimColumnResolvedFilterInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/DimColumnResolvedFilterInfo.java
index abd4652..fee15a4 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/DimColumnResolvedFilterInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/DimColumnResolvedFilterInfo.java
@@ -48,8 +48,6 @@ public class DimColumnResolvedFilterInfo implements Serializable {
 
   private boolean isDimensionExistsInCurrentSilce = true;
 
-  private String defaultValue;
-
   private CarbonDimension dimension;
 
   /**
@@ -119,14 +117,6 @@ public class DimColumnResolvedFilterInfo implements Serializable {
     this.isDimensionExistsInCurrentSilce = isDimensionExistsInCurrentSilce;
   }
 
-  public String getDefaultValue() {
-    return defaultValue;
-  }
-
-  public void setDefaultValue(String defaultValue) {
-    this.defaultValue = defaultValue;
-  }
-
   public void populateFilterInfoBasedOnColumnType(ResolvedFilterInfoVisitorIntf visitor,
       FilterResolverMetadata metadata) throws FilterUnsupportedException, IOException {
     if (null != visitor) {
@@ -138,4 +128,18 @@ public class DimColumnResolvedFilterInfo implements Serializable {
     }
 
   }
+
+  /**
+   * This method will clone the current object
+   *
+   * @return
+   */
+  public DimColumnResolvedFilterInfo getCopyObject() {
+    DimColumnResolvedFilterInfo dimColumnResolvedFilterInfo = new DimColumnResolvedFilterInfo();
+    dimColumnResolvedFilterInfo.resolvedFilterValueObj = this.resolvedFilterValueObj;
+    dimColumnResolvedFilterInfo.rowIndex = this.rowIndex;
+    dimColumnResolvedFilterInfo.dimensionResolvedFilter = this.dimensionResolvedFilter;
+    dimColumnResolvedFilterInfo.isDimensionExistsInCurrentSilce = isDimensionExistsInCurrentSilce;
+    return dimColumnResolvedFilterInfo;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
index 3d69553..4c50825 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
@@ -19,6 +19,8 @@ package org.apache.carbondata.core.scan.filter.resolver.resolverinfo;
 
 import java.io.Serializable;
 
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
+
 public class MeasureColumnResolvedFilterInfo implements Serializable {
   /**
    *
@@ -31,6 +33,8 @@ public class MeasureColumnResolvedFilterInfo implements Serializable {
 
   private Object defaultValue;
 
+  private CarbonColumn carbonColumn;
+
   private org.apache.carbondata.core.metadata.datatype.DataType type;
 
   public int getColumnIndex() {
@@ -65,4 +69,11 @@ public class MeasureColumnResolvedFilterInfo implements Serializable {
     return defaultValue;
   }
 
+  public CarbonColumn getCarbonColumn() {
+    return carbonColumn;
+  }
+
+  public void setCarbonColumn(CarbonColumn carbonColumn) {
+    this.carbonColumn = carbonColumn;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
index bc1def7..c51133d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
@@ -16,9 +16,11 @@
  */
 package org.apache.carbondata.core.scan.filter.resolver.resolverinfo.visitor;
 
+import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
 import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
 import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
 import org.apache.carbondata.core.scan.filter.DimColumnFilterInfo;
@@ -45,7 +47,15 @@ public class NoDictionaryTypeVisitor implements ResolvedFilterInfoVisitorIntf {
     DimColumnFilterInfo resolvedFilterObject = null;
     List<String> evaluateResultListFinal;
     try {
-      evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
+      ExpressionResult result = metadata.getExpression().evaluate(null);
+      Boolean booleanResult = result.getBoolean();
+      // handling for is null case scenarios
+      if (null != booleanResult && booleanResult == metadata.isIncludeFilter()) {
+        evaluateResultListFinal = new ArrayList<>(1);
+        evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
+      } else {
+        evaluateResultListFinal = result.getListAsString();
+      }
       // Adding default  null member inorder to not display the same while
       // displaying the report as per hive compatibility.
       if (!metadata.isIncludeFilter() && !evaluateResultListFinal

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
index 210ee11..9162ff8 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
@@ -172,7 +172,7 @@ public class QueryModel implements Serializable {
 
   private static CarbonMeasure getCarbonMetadataMeasure(String name, List<CarbonMeasure> measures) {
     for (CarbonMeasure measure : measures) {
-      if (measure.getColName().equalsIgnoreCase(name)) {
+      if (!measure.isInvisible() && measure.getColName().equalsIgnoreCase(name)) {
         return measure;
       }
     }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
index f0cebf4..e4314da 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
@@ -28,10 +28,8 @@ import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.datastore.DataRefNode;
 import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.scan.collector.ResultCollectorFactory;
 import org.apache.carbondata.core.scan.collector.ScannedResultCollector;
-import org.apache.carbondata.core.scan.collector.impl.DictionaryBasedResultCollector;
-import org.apache.carbondata.core.scan.collector.impl.DictionaryBasedVectorResultCollector;
-import org.apache.carbondata.core.scan.collector.impl.RawBasedResultCollector;
 import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
 import org.apache.carbondata.core.scan.result.AbstractScannedResult;
 import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
@@ -98,19 +96,8 @@ public abstract class AbstractDataBlockIterator extends CarbonIterator<List<Obje
     } else {
       blockletScanner = new NonFilterScanner(blockExecutionInfo, queryStatisticsModel);
     }
-    if (blockExecutionInfo.isRawRecordDetailQuery()) {
-      LOGGER.info("Row based raw collector is used to scan and collect the data");
-      this.scannerResultAggregator =
-          new RawBasedResultCollector(blockExecutionInfo);
-    } else if (blockExecutionInfo.isVectorBatchCollector()) {
-      LOGGER.info("Vector based dictionary collector is used to scan and collect the data");
-      this.scannerResultAggregator =
-          new DictionaryBasedVectorResultCollector(blockExecutionInfo);
-    } else {
-      LOGGER.info("Row based dictionary collector is used to scan and collect the data");
-      this.scannerResultAggregator =
-          new DictionaryBasedResultCollector(blockExecutionInfo);
-    }
+    this.scannerResultAggregator =
+        ResultCollectorFactory.getScannedResultCollector(blockExecutionInfo);
     this.batchSize = batchSize;
     this.executorService = executorService;
     this.nextBlock = new AtomicBoolean(false);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 36e75f3..33d9aab 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -39,6 +39,9 @@ import java.util.Set;
 
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.cache.Cache;
+import org.apache.carbondata.core.cache.CacheProvider;
+import org.apache.carbondata.core.cache.CacheType;
 import org.apache.carbondata.core.cache.dictionary.Dictionary;
 import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
@@ -53,13 +56,18 @@ import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
 import org.apache.carbondata.core.datastore.compression.MeasureMetaDataModel;
 import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
 import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
 import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
 import org.apache.carbondata.core.metadata.ValueEncoderMeta;
 import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
 import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
 import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
 import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
 import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
@@ -1060,20 +1068,22 @@ public final class CarbonUtil {
     List<Boolean> isDictionaryDimensions = new ArrayList<Boolean>();
     Set<Integer> processedColumnGroup = new HashSet<Integer>();
     for (CarbonDimension carbonDimension : tableDimensionList) {
-      List<CarbonDimension> childs = carbonDimension.getListOfChildDimensions();
-      //assuming complex dimensions will always be atlast
-      if (null != childs && childs.size() > 0) {
-        break;
-      }
-      if (carbonDimension.isColumnar() && hasEncoding(carbonDimension.getEncoder(),
-          Encoding.DICTIONARY)) {
-        isDictionaryDimensions.add(true);
-      } else if (!carbonDimension.isColumnar()) {
-        if (processedColumnGroup.add(carbonDimension.columnGroupId())) {
+      if (!carbonDimension.isInvisible()) {
+        List<CarbonDimension> childs = carbonDimension.getListOfChildDimensions();
+        //assuming complex dimensions will always be atlast
+        if (null != childs && childs.size() > 0) {
+          break;
+        }
+        if (carbonDimension.isColumnar() && hasEncoding(carbonDimension.getEncoder(),
+            Encoding.DICTIONARY)) {
           isDictionaryDimensions.add(true);
+        } else if (!carbonDimension.isColumnar()) {
+          if (processedColumnGroup.add(carbonDimension.columnGroupId())) {
+            isDictionaryDimensions.add(true);
+          }
+        } else {
+          isDictionaryDimensions.add(false);
         }
-      } else {
-        isDictionaryDimensions.add(false);
       }
     }
     boolean[] primitive = ArrayUtils
@@ -1147,7 +1157,7 @@ public final class CarbonUtil {
   public static CarbonDimension findDimension(List<CarbonDimension> dimensions, String carbonDim) {
     CarbonDimension findDim = null;
     for (CarbonDimension dimension : dimensions) {
-      if (dimension.getColName().equalsIgnoreCase(carbonDim)) {
+      if (!dimension.isInvisible() && dimension.getColName().equalsIgnoreCase(carbonDim)) {
         findDim = dimension;
         break;
       }
@@ -1156,6 +1166,44 @@ public final class CarbonUtil {
   }
 
   /**
+   * This method will search for a given dimension in the current block dimensions list
+   *
+   * @param blockDimensions
+   * @param dimensionToBeSearched
+   * @return
+   */
+  public static CarbonDimension getDimensionFromCurrentBlock(
+      List<CarbonDimension> blockDimensions, CarbonDimension dimensionToBeSearched) {
+    CarbonDimension currentBlockDimension = null;
+    for (CarbonDimension blockDimension : blockDimensions) {
+      if (dimensionToBeSearched.getColumnId().equals(blockDimension.getColumnId())) {
+        currentBlockDimension = blockDimension;
+        break;
+      }
+    }
+    return currentBlockDimension;
+  }
+
+  /**
+   * This method will search for a given measure in the current block measures list
+   *
+   * @param blockMeasures
+   * @param columnId
+   * @return
+   */
+  public static CarbonMeasure getMeasureFromCurrentBlock(List<CarbonMeasure> blockMeasures,
+      String columnId) {
+    CarbonMeasure currentBlockMeasure = null;
+    for (CarbonMeasure blockMeasure : blockMeasures) {
+      if (columnId.equals(blockMeasure.getColumnId())) {
+        currentBlockMeasure = blockMeasure;
+        break;
+      }
+    }
+    return currentBlockMeasure;
+  }
+
+  /**
    * This method will be used to clear the dictionary cache after its usage is complete
    * so that if memory threshold is reached it can evicted from LRU cache
    *
@@ -1195,7 +1243,9 @@ public final class CarbonUtil {
     List<ColumnSchema> wrapperColumnSchemaList = new ArrayList<ColumnSchema>();
     fillCollumnSchemaListForComplexDims(carbonDimensionsList, wrapperColumnSchemaList);
     for (CarbonMeasure carbonMeasure : carbonMeasureList) {
-      wrapperColumnSchemaList.add(carbonMeasure.getColumnSchema());
+      if (!carbonMeasure.isInvisible()) {
+        wrapperColumnSchemaList.add(carbonMeasure.getColumnSchema());
+      }
     }
     return wrapperColumnSchemaList;
   }
@@ -1203,10 +1253,12 @@ public final class CarbonUtil {
   private static void fillCollumnSchemaListForComplexDims(
       List<CarbonDimension> carbonDimensionsList, List<ColumnSchema> wrapperColumnSchemaList) {
     for (CarbonDimension carbonDimension : carbonDimensionsList) {
-      wrapperColumnSchemaList.add(carbonDimension.getColumnSchema());
-      List<CarbonDimension> childDims = carbonDimension.getListOfChildDimensions();
-      if (null != childDims && childDims.size() > 0) {
-        fillCollumnSchemaListForComplexDims(childDims, wrapperColumnSchemaList);
+      if (!carbonDimension.isInvisible()) {
+        wrapperColumnSchemaList.add(carbonDimension.getColumnSchema());
+        List<CarbonDimension> childDims = carbonDimension.getListOfChildDimensions();
+        if (null != childDims && childDims.size() > 0) {
+          fillCollumnSchemaListForComplexDims(childDims, wrapperColumnSchemaList);
+        }
       }
     }
   }
@@ -1614,6 +1666,122 @@ public final class CarbonUtil {
   }
 
   /**
+   * This method will delete the dictionary files for the given column IDs and
+   * clear the dictionary cache
+   *
+   * @param dictionaryColumns
+   * @param carbonTable
+   */
+  public static void deleteDictionaryFileAndCache(List<CarbonColumn> dictionaryColumns,
+      CarbonTable carbonTable) {
+    if (!dictionaryColumns.isEmpty()) {
+      CarbonTableIdentifier carbonTableIdentifier = carbonTable.getCarbonTableIdentifier();
+      CarbonTablePath carbonTablePath =
+          CarbonStorePath.getCarbonTablePath(carbonTable.getStorePath(), carbonTableIdentifier);
+      String metadataDirectoryPath = carbonTablePath.getMetadataDirectoryPath();
+      CarbonFile metadataDir = FileFactory
+          .getCarbonFile(metadataDirectoryPath, FileFactory.getFileType(metadataDirectoryPath));
+      for (final CarbonColumn column : dictionaryColumns) {
+        // sort index file is created with dictionary size appended to it. So all the files
+        // with a given column ID need to be listed
+        CarbonFile[] listFiles = metadataDir.listFiles(new CarbonFileFilter() {
+          @Override public boolean accept(CarbonFile path) {
+            if (path.getName().startsWith(column.getColumnId())) {
+              return true;
+            }
+            return false;
+          }
+        });
+        for (CarbonFile file : listFiles) {
+          // try catch is inside for loop because even if one deletion fails, other files
+          // still need to be deleted
+          try {
+            FileFactory.deleteFile(file.getCanonicalPath(),
+                FileFactory.getFileType(file.getCanonicalPath()));
+          } catch (IOException e) {
+            LOGGER.error(
+                "Failed to delete dictionary or sortIndex file for column " + column.getColName()
+                    + "with column ID " + column.getColumnId());
+          }
+        }
+        // remove dictionary cache
+        removeDictionaryColumnFromCache(carbonTable, column.getColumnId());
+      }
+    }
+  }
+
+  /**
+   * This method will remove dictionary cache from driver for both reverse and forward dictionary
+   *
+   * @param carbonTable
+   * @param columnId
+   */
+  public static void removeDictionaryColumnFromCache(CarbonTable carbonTable, String columnId) {
+    Cache<DictionaryColumnUniqueIdentifier, Dictionary> dictCache = CacheProvider.getInstance()
+        .createCache(CacheType.REVERSE_DICTIONARY, carbonTable.getStorePath());
+    DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier =
+        new DictionaryColumnUniqueIdentifier(carbonTable.getCarbonTableIdentifier(),
+            new ColumnIdentifier(columnId, null, null));
+    dictCache.invalidate(dictionaryColumnUniqueIdentifier);
+    dictCache = CacheProvider.getInstance()
+        .createCache(CacheType.FORWARD_DICTIONARY, carbonTable.getStorePath());
+    dictCache.invalidate(dictionaryColumnUniqueIdentifier);
+  }
+
+  /**
+   * This method will prepare the cardinality of dictionary columns based on the latest schema
+   *
+   * @param lookUpDimensions         dimensions list where a given dimension will be searched to get
+   *                                 the index for getting the cardinality of that column
+   * @param masterSchemaDimensions   latest schema dimensions
+   * @param mappingColumnCardinality cardinality of columns in the given carbondata file
+   * @return
+   */
+  public static int[] getUpdatedColumnCardinalities(List<ColumnSchema> lookUpDimensions,
+      List<CarbonDimension> masterSchemaDimensions, int[] mappingColumnCardinality) {
+    List<Integer> updatedDictionaryColumnCardinalities =
+        new ArrayList<>(masterSchemaDimensions.size());
+    for (CarbonDimension masterDimension : masterSchemaDimensions) {
+      // dimension should be visible and should be a dictionary column
+      if (!masterDimension.isInvisible() && hasEncoding(masterDimension.getEncoder(),
+          Encoding.DICTIONARY)) {
+        int destinationDimensionIndex = 0;
+        boolean isDimensionFoundInDestinationSegment = false;
+        for (ColumnSchema destinationDimension : lookUpDimensions) {
+          if (masterDimension.getColumnId().equals(destinationDimension.getColumnUniqueId())) {
+            isDimensionFoundInDestinationSegment = true;
+            break;
+          }
+          destinationDimensionIndex++;
+        }
+        if (!isDimensionFoundInDestinationSegment) {
+          if (hasEncoding(masterDimension.getEncoder(), Encoding.DIRECT_DICTIONARY)) {
+            updatedDictionaryColumnCardinalities.add(Integer.MAX_VALUE);
+          } else {
+            if (null != masterDimension.getDefaultValue()) {
+              // added +1 because if default value is provided then the cardinality of
+              // column will be 2. 1 for member default value and 1 for the value
+              // provided by the user
+              updatedDictionaryColumnCardinalities
+                  .add(CarbonCommonConstants.DICTIONARY_DEFAULT_CARDINALITY + 1);
+            } else {
+              updatedDictionaryColumnCardinalities
+                  .add(CarbonCommonConstants.DICTIONARY_DEFAULT_CARDINALITY);
+            }
+          }
+        } else {
+          // add the cardinality of the existing column in the schema
+          updatedDictionaryColumnCardinalities
+              .add(mappingColumnCardinality[destinationDimensionIndex]);
+        }
+      }
+    }
+    int[] updatedCardinalities = ArrayUtils.toPrimitive(updatedDictionaryColumnCardinalities
+        .toArray(new Integer[updatedDictionaryColumnCardinalities.size()]));
+    return updatedCardinalities;
+  }
+
+  /**
    * Below method will be used to convert byte data to surrogate key based
    * column value size
    *

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 1d983e0..1ba8942 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -20,6 +20,7 @@ package org.apache.carbondata.core.util;
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.math.RoundingMode;
+import java.nio.charset.Charset;
 import java.text.DateFormat;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
@@ -31,9 +32,12 @@ import java.util.Map;
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
 import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
 import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
 
 import org.apache.spark.unsafe.types.UTF8String;
 
@@ -47,19 +51,17 @@ public final class DataTypeUtil {
   private static final Map<String, String> dataTypeDisplayNames;
 
   private static final ThreadLocal<DateFormat> timeStampformatter = new ThreadLocal<DateFormat>() {
-    @Override
-    protected DateFormat initialValue() {
-      return new SimpleDateFormat(
-          CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+    @Override protected DateFormat initialValue() {
+      return new SimpleDateFormat(CarbonProperties.getInstance()
+          .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
               CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
     }
   };
 
   private static final ThreadLocal<DateFormat> dateformatter = new ThreadLocal<DateFormat>() {
-    @Override
-    protected DateFormat initialValue() {
-      return new SimpleDateFormat(
-          CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_DATE_FORMAT,
+    @Override protected DateFormat initialValue() {
+      return new SimpleDateFormat(CarbonProperties.getInstance()
+          .getProperty(CarbonCommonConstants.CARBON_DATE_FORMAT,
               CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT));
     }
   };
@@ -198,6 +200,7 @@ public final class DataTypeUtil {
     BigInteger sig = new BigInteger(unscale);
     return new BigDecimal(sig, scale);
   }
+
   /**
    * returns the SqlStatement.Type of corresponding string value
    *
@@ -245,6 +248,36 @@ public final class DataTypeUtil {
   }
 
   /**
+   * This method will convert the data according to its data type and perform a
+   * special handling for decimal data types
+   *
+   * @param dataInBytes
+   * @param dimension
+   * @return
+   */
+  public static Object getDataBasedOnDataType(byte[] dataInBytes, CarbonDimension dimension) {
+    if (null == dataInBytes || Arrays
+        .equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, dataInBytes)) {
+      return null;
+    }
+    switch (dimension.getDataType()) {
+      case DECIMAL:
+        String data = new String(dataInBytes, CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
+        if (data.isEmpty()) {
+          return null;
+        }
+        java.math.BigDecimal javaDecVal = new java.math.BigDecimal(data);
+        if (dimension.getColumnSchema().getScale() > javaDecVal.scale()) {
+          javaDecVal =
+              javaDecVal.setScale(dimension.getColumnSchema().getScale(), RoundingMode.HALF_UP);
+        }
+        return org.apache.spark.sql.types.Decimal.apply(javaDecVal);
+      default:
+        return getDataBasedOnDataType(dataInBytes, dimension.getDataType());
+    }
+  }
+
+  /**
    * Below method will be used to convert the data passed to its actual data
    * type
    *
@@ -538,6 +571,7 @@ public final class DataTypeUtil {
     }
     return null;
   }
+
   /**
    * This method will compare double values it will preserve
    * the -0.0 and 0.0 equality as per == ,also preserve NaN equality check as per
@@ -555,4 +589,99 @@ public final class DataTypeUtil {
     }
     return 1;
   }
+
+  /**
+   * Below method will be used to convert the data into byte[]
+   *
+   * @param data
+   * @param actualDataType actual data type
+   * @return actual data in byte[]
+   */
+  public static byte[] convertDataToBytesBasedOnDataType(String data, DataType actualDataType) {
+    if (null == data) {
+      return null;
+    } else if (CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(data)) {
+      LOGGER.error("Default value should not be carbon specific null value : " + data);
+      return null;
+    }
+    try {
+      long parsedIntVal = 0;
+      switch (actualDataType) {
+        case INT:
+          parsedIntVal = (long) Integer.parseInt(data);
+          return String.valueOf(parsedIntVal)
+              .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+        case SHORT:
+          parsedIntVal = (long) Short.parseShort(data);
+          return String.valueOf(parsedIntVal)
+              .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+        case DOUBLE:
+          return String.valueOf(Double.parseDouble(data))
+              .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+        case LONG:
+          return String.valueOf(Long.parseLong(data))
+              .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+        case DATE:
+        case TIMESTAMP:
+          DirectDictionaryGenerator directDictionaryGenerator =
+              DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(actualDataType);
+          int value = directDictionaryGenerator.generateDirectSurrogateKey(data);
+          return String.valueOf(value)
+              .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+        case DECIMAL:
+          java.math.BigDecimal javaDecVal = new java.math.BigDecimal(data);
+          return bigDecimalToByte(javaDecVal);
+        default:
+          return UTF8String.fromString(data).getBytes();
+      }
+    } catch (NumberFormatException ex) {
+      LOGGER.error("Problem while converting data type" + data);
+      return null;
+    }
+  }
+
+  /**
+   * This method will parse a given string value corresponding to its data type
+   *
+   * @param value     value to parse
+   * @param columnSchema dimension to get data type and precision and scale in case of decimal
+   *                  data type
+   * @return
+   */
+  public static String normalizeColumnValueForItsDataType(String value, ColumnSchema columnSchema) {
+    try {
+      Object parsedValue = null;
+      switch (columnSchema.getDataType()) {
+        case DECIMAL:
+          return parseStringToBigDecimal(value, columnSchema);
+        case SHORT:
+        case INT:
+        case LONG:
+          parsedValue = normalizeIntAndLongValues(value, columnSchema.getDataType());
+          break;
+        case DOUBLE:
+          parsedValue = Double.parseDouble(value);
+          break;
+        default:
+          return value;
+      }
+      if (null != parsedValue) {
+        return value;
+      }
+      return null;
+    } catch (Exception e) {
+      return null;
+    }
+  }
+
+  private static String parseStringToBigDecimal(String value, ColumnSchema columnSchema) {
+    BigDecimal bigDecimal = new BigDecimal(value)
+        .setScale(columnSchema.getScale(), RoundingMode.HALF_UP);
+    BigDecimal normalizedValue =
+        normalizeDecimalValue(bigDecimal, columnSchema.getPrecision());
+    if (null != normalizedValue) {
+      return normalizedValue.toString();
+    }
+    return null;
+  }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
index 1827fb0..eae99f2 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
@@ -370,7 +370,7 @@ public class CarbonDictionaryWriterImpl implements CarbonDictionaryWriter {
   /**
    * close dictionary thrift writer
    */
-  private void closeThriftWriter() {
+  private void closeThriftWriter() throws IOException {
     if (null != dictionaryThriftWriter) {
       dictionaryThriftWriter.close();
     }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileWriter.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileWriter.java
index 5238894..d17bcf7 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileWriter.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileWriter.java
@@ -55,7 +55,7 @@ public class CarbonIndexFileWriter {
   /**
    * Below method will be used to close the thrift object
    */
-  public void close() {
+  public void close() throws IOException {
     thriftWriter.close();
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java b/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java
index 92bae42..9bf549d 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java
@@ -21,6 +21,9 @@ import java.io.DataOutputStream;
 import java.io.IOException;
 
 import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperations;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl;
+import org.apache.carbondata.core.fileoperations.FileWriteOperation;
 import org.apache.carbondata.core.util.CarbonUtil;
 
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -56,6 +59,11 @@ public class ThriftWriter {
   private TProtocol binaryOut;
 
   /**
+   * Identifier for performing atomic file operations
+   */
+  private AtomicFileOperations atomicFileOperationsWriter;
+
+  /**
    * flag to append to existing file
    */
   private boolean append;
@@ -78,6 +86,19 @@ public class ThriftWriter {
   }
 
   /**
+   * Method for opening file writing for atomic operations
+   *
+   * @param fileWriteOperation
+   * @throws IOException
+   */
+  public void open(FileWriteOperation fileWriteOperation) throws IOException {
+    FileFactory.FileType fileType = FileFactory.getFileType(fileName);
+    atomicFileOperationsWriter = new AtomicFileOperationsImpl(fileName, fileType);
+    dataOutputStream = atomicFileOperationsWriter.openForWrite(fileWriteOperation);
+    binaryOut = new TCompactProtocol(new TIOStreamTransport(dataOutputStream));
+  }
+
+  /**
    * This will check whether stream and binary out is open or not.
    * @return
    */
@@ -113,11 +134,25 @@ public class ThriftWriter {
   /**
    * Close the file stream.
    */
-  public void close() {
+  public void close() throws IOException {
+    closeAtomicFileWriter();
     CarbonUtil.closeStreams(dataOutputStream);
   }
 
   /**
+   * This method will close the atomic file operations writer
+   *
+   * @throws IOException
+   */
+  private void closeAtomicFileWriter() throws IOException {
+    if (null != atomicFileOperationsWriter) {
+      atomicFileOperationsWriter.close();
+      // set output stream to null as atomic writer will close the data output stream internally
+      dataOutputStream = null;
+    }
+  }
+
+  /**
    * Flush data to HDFS file
    */
   public void sync() throws IOException {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollectorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollectorTest.java b/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollectorTest.java
index 66f4830..8f902b1 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollectorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollectorTest.java
@@ -26,13 +26,13 @@ public class DictionaryBasedResultCollectorTest {
 //    blockExecutionInfo = new BlockExecutionInfo();
 //    KeyStructureInfo keyStructureInfo = new KeyStructureInfo();
 //    blockExecutionInfo.setKeyStructureInfo(keyStructureInfo);
-//    AggregatorInfo aggregatorInfo = new AggregatorInfo();
+//    MeasureInfo aggregatorInfo = new MeasureInfo();
 //    aggregatorInfo.setMeasureOrdinals(new int[] { 10, 20, 30, 40 });
 //    aggregatorInfo.setMeasureExists(new boolean[] { true, false, false, false });
 //    aggregatorInfo.setDefaultValues(new Object[] { 1, 2, 3, 4 });
 //    aggregatorInfo.setMeasureDataTypes(
 //        new DataType[] { DataType.INT, DataType.TIMESTAMP, DataType.INT, DataType.INT });
-//    blockExecutionInfo.setAggregatorInfo(aggregatorInfo);
+//    blockExecutionInfo.setMeasureInfo(aggregatorInfo);
 //    QueryDimension queryDimension1 = new QueryDimension("QDCol1");
 //    queryDimension1.setQueryOrder(1);
 //    ColumnSchema columnSchema = new ColumnSchema();

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollectorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollectorTest.java b/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollectorTest.java
index d3b3a49..2a8f78e 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollectorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollectorTest.java
@@ -44,7 +44,7 @@ public class RawBasedResultCollectorTest {
 //    keyStructureInfo.setMaskedBytes(new int[] { 1, 2 });
 //    keyStructureInfo.setMaskByteRanges(new int[] { 1, 2 });
 //    blockExecutionInfo.setKeyStructureInfo(keyStructureInfo);
-//    AggregatorInfo aggregatorInfo = new AggregatorInfo();
+//    MeasureInfo aggregatorInfo = new MeasureInfo();
 //    aggregatorInfo.setMeasureOrdinals(new int[] { 10, 20, 30, 40 });
 //    aggregatorInfo.setMeasureExists(new boolean[] { true, false, false, false });
 //    aggregatorInfo.setDefaultValues(new Object[] { 1, 2, 3, 4 });
@@ -77,7 +77,7 @@ public class RawBasedResultCollectorTest {
 //    blockExecutionInfo.setQueryMeasures(
 //        new QueryMeasure[] { queryMeasure1, queryMeasure2, queryMeasure3, queryMeasure4 });
 //    blockExecutionInfo.setFixedKeyUpdateRequired(true);
-//    blockExecutionInfo.setAggregatorInfo(aggregatorInfo);
+//    blockExecutionInfo.setMeasureInfo(aggregatorInfo);
 //    blockExecutionInfo.setMaskedByteForBlock(new int[] { 1, 2 });
 //    blockExecutionInfo.setBlockKeyGenerator(keyGenerator);
 //    rawBasedResultCollector = new RawBasedResultCollector(blockExecutionInfo);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java
index 7bfaa3e..634a301 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java
@@ -20,7 +20,9 @@ import org.apache.carbondata.core.metadata.encoder.Encoding;
 import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
 import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
 import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.scan.executor.infos.AggregatorInfo;
+import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.executor.infos.MeasureInfo;
 import org.apache.carbondata.core.scan.model.QueryDimension;
 import org.apache.carbondata.core.scan.model.QueryMeasure;
 
@@ -37,6 +39,7 @@ import java.util.List;
 public class RestructureUtilTest {
 
   @Test public void testToGetUpdatedQueryDimension() {
+    BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
     List<Encoding> encodingList = new ArrayList<Encoding>();
     encodingList.add(Encoding.DICTIONARY);
     ColumnSchema columnSchema1 = new ColumnSchema();
@@ -75,8 +78,10 @@ public class RestructureUtilTest {
     List<QueryDimension> queryDimensions =
         Arrays.asList(queryDimension1, queryDimension2, queryDimension3);
 
-    List<QueryDimension> result = RestructureUtil
-        .getUpdatedQueryDimension(queryDimensions, tableBlockDimensions, tableComplexDimensions);
+    List<QueryDimension> result = null;
+    result = RestructureUtil
+        .createDimensionInfoAndGetUpdatedQueryDimension(blockExecutionInfo, queryDimensions, tableBlockDimensions,
+            tableComplexDimensions);
 
     assertThat(result, is(equalTo(Arrays.asList(queryDimension1, queryDimension2))));
   }
@@ -92,7 +97,7 @@ public class RestructureUtilTest {
     CarbonMeasure carbonMeasure1 = new CarbonMeasure(columnSchema1, 1);
     CarbonMeasure carbonMeasure2 = new CarbonMeasure(columnSchema2, 2);
     CarbonMeasure carbonMeasure3 = new CarbonMeasure(columnSchema3, 3);
-    carbonMeasure3.setDefaultValue("3".getBytes());
+    carbonMeasure3.getColumnSchema().setDefaultValue("3".getBytes());
     List<CarbonMeasure> currentBlockMeasures = Arrays.asList(carbonMeasure1, carbonMeasure2);
 
     QueryMeasure queryMeasure1 = new QueryMeasure("Id");
@@ -102,12 +107,13 @@ public class RestructureUtilTest {
     QueryMeasure queryMeasure3 = new QueryMeasure("Age");
     queryMeasure3.setMeasure(carbonMeasure3);
     List<QueryMeasure> queryMeasures = Arrays.asList(queryMeasure1, queryMeasure2, queryMeasure3);
-
-    AggregatorInfo aggregatorInfo =
-        RestructureUtil.getAggregatorInfos(queryMeasures, currentBlockMeasures);
+    BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
+    RestructureUtil.createMeasureInfoAndGetUpdatedQueryMeasures(blockExecutionInfo, queryMeasures,
+        currentBlockMeasures);
+    MeasureInfo measureInfo = blockExecutionInfo.getMeasureInfo();
     boolean[] measuresExist = { true, true, false };
-    assertThat(aggregatorInfo.getMeasureExists(), is(equalTo(measuresExist)));
+    assertThat(measureInfo.getMeasureExists(), is(equalTo(measuresExist)));
     Object[] defaultValues = { null, null, "3".getBytes() };
-    assertThat(aggregatorInfo.getDefaultValues(), is(equalTo(defaultValues)));
+    assertThat(measureInfo.getDefaultValues(), is(equalTo(defaultValues)));
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java
index 2c01746..cd58a00 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java
@@ -17,7 +17,9 @@
 
 package org.apache.carbondata.processing.newflow;
 
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -27,6 +29,10 @@ public class CarbonDataLoadConfiguration {
 
   private DataField[] dataFields;
 
+  private DataField[] dimensionFields;
+
+  private DataField[] measureFields;
+
   private AbsoluteTableIdentifier tableIdentifier;
 
   private String[] header;
@@ -58,17 +64,43 @@ public class CarbonDataLoadConfiguration {
 
   private boolean preFetch;
 
+  private int dimensionCount;
+
+  private int measureCount;
+
   public CarbonDataLoadConfiguration() {
   }
 
-  public int getDimensionCount() {
-    int dimCount = 0;
+  private void initDimensionFields() {
+    List<Integer> dimensionIndexes = new ArrayList<>(dataFields.length);
     for (int i = 0; i < dataFields.length; i++) {
       if (dataFields[i].getColumn().isDimesion()) {
-        dimCount++;
+        dimensionIndexes.add(i);
+        dimensionCount++;
       }
     }
-    return dimCount;
+    dimensionFields = new DataField[dimensionCount];
+    for (int i = 0; i < dimensionCount; i++) {
+      dimensionFields[i] = dataFields[dimensionIndexes.get(i)];
+    }
+  }
+
+  private void initMeasureFields() {
+    List<Integer> measureIndexes = new ArrayList<>(dataFields.length);
+    for (int i = 0; i < dataFields.length; i++) {
+      if (!dataFields[i].getColumn().isDimesion()) {
+        measureIndexes.add(i);
+        measureCount++;
+      }
+    }
+    measureFields = new DataField[measureCount];
+    for (int i = 0; i < measureCount; i++) {
+      measureFields[i] = dataFields[measureIndexes.get(i)];
+    }
+  }
+
+  public int getDimensionCount() {
+    return dimensionCount;
   }
 
   public int getNoDictionaryCount() {
@@ -92,13 +124,7 @@ public class CarbonDataLoadConfiguration {
   }
 
   public int getMeasureCount() {
-    int msrCount = 0;
-    for (int i = 0; i < dataFields.length; i++) {
-      if (!dataFields[i].getColumn().isDimesion()) {
-        msrCount++;
-      }
-    }
-    return msrCount;
+    return measureCount;
   }
 
   public DataField[] getDataFields() {
@@ -107,6 +133,8 @@ public class CarbonDataLoadConfiguration {
 
   public void setDataFields(DataField[] dataFields) {
     this.dataFields = dataFields;
+    initDimensionFields();
+    initMeasureFields();
   }
 
   public String[] getHeader() {
@@ -196,4 +224,12 @@ public class CarbonDataLoadConfiguration {
   public void setPreFetch(boolean preFetch) {
     this.preFetch = preFetch;
   }
+
+  public DataField[] getDimensionFields() {
+    return dimensionFields;
+  }
+
+  public DataField[] getMeasureFields() {
+    return measureFields;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
index b89e283..5e01abe 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
@@ -146,18 +146,20 @@ public final class DataLoadProcessBuilder {
     // First add dictionary and non dictionary dimensions because these are part of mdk key.
     // And then add complex data types and measures.
     for (CarbonColumn column : dimensions) {
-      DataField dataField = new DataField(column);
-      dataField.setDateFormat(dateFormatMap.get(column.getColName()));
-      if (column.isComplex()) {
-        complexDataFields.add(dataField);
-      } else {
-        dataFields.add(dataField);
+      if (!column.isInvisible()) {
+        DataField dataField = new DataField(column);
+        dataField.setDateFormat(dateFormatMap.get(column.getColName()));
+        if (column.isComplex()) {
+          complexDataFields.add(dataField);
+        } else {
+          dataFields.add(dataField);
+        }
       }
     }
     dataFields.addAll(complexDataFields);
     for (CarbonColumn column : measures) {
       // This dummy measure is added when no measure was present. We no need to load it.
-      if (!(column.getColName().equals("default_dummy_measure"))) {
+      if (!column.isInvisible() && !(column.getColName().equals("default_dummy_measure"))) {
         dataFields.add(new DataField(column));
       }
     }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
index f618965..a52ebb2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
@@ -430,8 +430,7 @@ public class SortParameters {
         CarbonCommonConstants.CARBON_PREFETCH_BUFFERSIZE_DEFAULT)));
 
     char[] aggType = CarbonDataProcessorUtil
-        .getAggType(parameters.getMeasureColCount(), parameters.getDatabaseName(),
-            parameters.getTableName());
+        .getAggType(configuration.getMeasureCount(), configuration.getMeasureFields());
     parameters.setAggType(aggType);
     parameters.setUseKettle(false);
     return parameters;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index cae22a4..594f0e2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -269,7 +269,7 @@ public class CarbonFactDataHandlerModel {
     carbonFactDataHandlerModel.setColCardinality(colCardinality);
     carbonFactDataHandlerModel.setDataWritingRequest(true);
     carbonFactDataHandlerModel.setAggType(CarbonDataProcessorUtil
-        .getAggType(measureCount, identifier.getDatabaseName(), identifier.getTableName()));
+        .getAggType(configuration.getMeasureCount(), configuration.getMeasureFields()));
     carbonFactDataHandlerModel.setFactDimLens(dimLens);
     carbonFactDataHandlerModel.setWrapperColumnSchema(wrapperColumnSchema);
     carbonFactDataHandlerModel.setPrimitiveDimLens(simpleDimsLen);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index a56e211..eff5fca 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -397,34 +397,15 @@ public final class CarbonDataProcessorUtil {
    */
   public static Set<String> getSchemaColumnNames(CarbonDataLoadSchema schema, String tableName) {
     Set<String> columnNames = new HashSet<String>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-
     String factTableName = schema.getCarbonTable().getFactTableName();
     if (tableName.equals(factTableName)) {
-
       List<CarbonDimension> dimensions =
           schema.getCarbonTable().getDimensionByTableName(factTableName);
-
       for (CarbonDimension dimension : dimensions) {
-
-        String foreignKey = null;
-        for (CarbonDataLoadSchema.DimensionRelation dimRel : schema.getDimensionRelationList()) {
-          for (String field : dimRel.getColumns()) {
-            if (dimension.getColName().equals(field)) {
-              foreignKey = dimRel.getRelation().getFactForeignKeyColumn();
-              break;
-            }
-          }
-          if (null != foreignKey) {
-            break;
-          }
-        }
-        if (null == foreignKey) {
+        if (!dimension.isInvisible()) {
           columnNames.add(dimension.getColName());
-        } else {
-          columnNames.add(foreignKey);
         }
       }
-
       List<CarbonMeasure> measures = schema.getCarbonTable().getMeasureByTableName(factTableName);
       for (CarbonMeasure msr : measures) {
         if (!msr.getColumnSchema().isInvisible()) {
@@ -436,15 +417,12 @@ public final class CarbonDataProcessorUtil {
       for (CarbonDimension dimension : dimensions) {
         columnNames.add(dimension.getColName());
       }
-
       List<CarbonMeasure> measures = schema.getCarbonTable().getMeasureByTableName(tableName);
       for (CarbonMeasure msr : measures) {
         columnNames.add(msr.getColName());
       }
     }
-
     return columnNames;
-
   }
 
   /**
@@ -481,6 +459,18 @@ public final class CarbonDataProcessorUtil {
   }
 
   /**
+   * get agg type
+   */
+  public static char[] getAggType(int measureCount, DataField[] measureFields) {
+    char[] aggType = new char[measureCount];
+    Arrays.fill(aggType, 'n');
+    for (int i = 0; i < measureFields.length; i++) {
+      aggType[i] = DataTypeUtil.getAggType(measureFields[i].getColumn().getDataType());
+    }
+    return aggType;
+  }
+
+  /**
    * Creates map for columns which dateformats mentioned while loading the data.
    * @param dataFormatString
    * @return


Mime
View raw message