carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [5/6] incubator-carbondata git commit: sort columns
Date Fri, 07 Apr 2017 14:19:15 GMT
sort columns


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/a76b418f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/a76b418f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/a76b418f

Branch: refs/heads/12-dev
Commit: a76b418fe0a6ea76e62004ca19673fd7998c63d8
Parents: 33544d3
Author: QiangCai <qiangcai@qq.com>
Authored: Thu Mar 2 17:48:54 2017 +0800
Committer: jackylk <jacky.likun@huawei.com>
Committed: Fri Apr 7 22:15:46 2017 +0800

----------------------------------------------------------------------
 .../core/constants/CarbonCommonConstants.java   |   1 +
 .../core/datastore/block/SegmentProperties.java |  42 +++
 .../impl/VariableLengthDimensionDataChunk.java  |  58 +++-
 .../impl/btree/BTreeDataRefNodeFinder.java      |  24 +-
 .../core/keygenerator/mdkey/Bits.java           |   6 +-
 .../ThriftWrapperSchemaConverterImpl.java       |  18 +
 .../core/metadata/schema/table/CarbonTable.java |  24 ++
 .../schema/table/column/ColumnSchema.java       |  10 +
 .../impl/DictionaryBasedResultCollector.java    |  12 +-
 ...structureBasedDictionaryResultCollector.java |   4 +-
 .../scan/filter/FilterExpressionProcessor.java  |   4 +-
 .../carbondata/core/scan/filter/FilterUtil.java |  98 ++++--
 ...velRangeLessThanEqualFilterExecuterImpl.java |   2 +-
 .../RowLevelRangeLessThanFiterExecuterImpl.java |   2 +-
 .../visitor/NoDictionaryTypeVisitor.java        |   3 +-
 .../visitor/RangeNoDictionaryTypeVisitor.java   |   3 +-
 .../AbstractDetailQueryResultIterator.java      |   4 +-
 .../scan/result/vector/CarbonColumnVector.java  |   7 +
 .../util/AbstractDataFileFooterConverter.java   |   7 +
 .../apache/carbondata/core/util/ByteUtil.java   | 336 ++++++++++++++++++-
 .../apache/carbondata/core/util/CarbonUtil.java |  18 +
 .../carbondata/core/util/DataTypeUtil.java      |  70 ++++
 .../impl/btree/BTreeBlockFinderTest.java        |  10 +-
 .../core/scan/filter/FilterUtilTest.java        |   7 +-
 .../carbondata/hadoop/CarbonInputFormat.java    |   4 +-
 .../internal/index/impl/InMemoryBTreeIndex.java |   4 +-
 .../impl/DictionaryDecodeReadSupport.java       |   3 +
 .../hadoop/util/BlockLevelTraverser.java        |   4 +-
 .../hadoop/test/util/StoreCreator.java          |   5 +
 .../testsuite/sortcolumns/TestSortColumns.scala | 267 +++++++++++++++
 .../spark/sql/catalyst/CarbonDDLSqlParser.scala | 125 +++----
 .../execution/command/carbonTableSchema.scala   |  66 ++--
 .../apache/spark/sql/TestCarbonSqlParser.scala  |  56 +---
 .../readsupport/SparkRowReadSupportImpl.java    |  13 +-
 .../vectorreader/ColumnarVectorWrapper.java     |  13 +
 .../org/apache/spark/sql/TableCreator.scala     | 145 ++++----
 .../merger/CompactionResultSortProcessor.java   |  10 +-
 .../merger/RowResultMergerProcessor.java        |  14 +-
 .../newflow/CarbonDataLoadConfiguration.java    |  24 ++
 .../newflow/DataLoadProcessBuilder.java         |  24 +-
 .../impl/NonDictionaryFieldConverterImpl.java   |  21 +-
 .../sort/impl/ParallelReadMergeSorterImpl.java  |   3 +-
 ...arallelReadMergeSorterWithBucketingImpl.java |   3 +-
 .../sort/unsafe/UnsafeCarbonRowPage.java        |  11 +-
 .../newflow/sort/unsafe/UnsafeSortDataRows.java |  11 +-
 .../unsafe/comparator/UnsafeRowComparator.java  |  10 +-
 .../UnsafeRowComparatorForNormalDIms.java       |   8 +-
 .../holder/UnsafeFinalMergePageHolder.java      |   4 +-
 .../unsafe/holder/UnsafeInmemoryHolder.java     |   5 +-
 .../holder/UnsafeSortTempFileChunkHolder.java   |   2 +-
 .../UnsafeSingleThreadFinalSortFilesMerger.java |   4 +-
 .../newflow/steps/NoSortProcessorStepImpl.java  | 153 +++++++++
 .../sortdata/IntermediateFileMerger.java        |   3 +-
 .../sortdata/NewRowComparator.java              |  12 +-
 .../sortdata/NewRowComparatorForNormalDims.java |  10 +-
 .../sortandgroupby/sortdata/RowComparator.java  |  10 +-
 .../sortdata/RowComparatorForNormalDims.java    |  10 +-
 .../sortandgroupby/sortdata/SortDataRows.java   |  13 +-
 .../sortandgroupby/sortdata/SortParameters.java |  43 +++
 .../sortdata/SortTempFileChunkHolder.java       |  11 +-
 .../store/CarbonFactDataHandlerColumnar.java    |  45 ++-
 .../store/CarbonFactDataHandlerModel.java       |   7 +
 .../store/SingleThreadFinalSortFilesMerger.java |   8 +-
 63 files changed, 1601 insertions(+), 353 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index b82d53c..6cee7f6 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -805,6 +805,7 @@ public final class CarbonCommonConstants {
   public static final String COLUMN_GROUPS = "column_groups";
   public static final String DICTIONARY_EXCLUDE = "dictionary_exclude";
   public static final String DICTIONARY_INCLUDE = "dictionary_include";
+  public static final String SORT_COLUMNS = "sort_columns";
   public static final String PARTITIONCLASS = "partitionclass";
   public static final String PARTITIONCOUNT = "partitioncount";
   public static final String COLUMN_PROPERTIES = "columnproperties";

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
index 6c7d014..80785e0 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
@@ -55,6 +55,14 @@ public class SegmentProperties {
   private KeyGenerator dimensionKeyGenerator;
 
   /**
+   * key generator which was used to generate the mdkey for dimensions in SORT_COLUMNS
+   * if SORT_COLUMNS contains all dimensions, it is same with dimensionKeyGenerator
+   * otherwise, it is different with dimensionKeyGenerator, the number of its dimensions is less
+   * than dimensionKeyGenerator.
+   */
+  private KeyGenerator sortColumnsGenerator;
+
+  /**
    * list of dimension present in the block
    */
   private List<CarbonDimension> dimensions;
@@ -151,6 +159,10 @@ public class SegmentProperties {
    */
   private ColumnGroupModel colGroupModel;
 
+  private int numberOfSortColumns = 0;
+
+  private int numberOfNoDictSortColumns = 0;
+
   public SegmentProperties(List<ColumnSchema> columnsInTable, int[] columnCardinality) {
     dimensions = new ArrayList<CarbonDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
     complexDimensions =
@@ -336,6 +348,9 @@ public class SegmentProperties {
         if (CarbonUtil.hasEncoding(columnSchema.getEncodingList(), Encoding.DICTIONARY)
             && !isComplexDimensionStarted && columnSchema.getNumberOfChild() == 0) {
           cardinalityIndexForNormalDimensionColumn.add(tableOrdinal);
+          if (columnSchema.isSortColumn()) {
+            this.numberOfSortColumns++;
+          }
           if (columnSchema.isColumnar()) {
             // if it is a columnar dimension participated in mdkey then added
             // key ordinal and dimension ordinal
@@ -385,6 +400,10 @@ public class SegmentProperties {
           // for no dictionary dimension
           carbonDimension = new CarbonDimension(columnSchema, dimensonOrdinal++, -1, -1, -1);
           numberOfNoDictionaryDimension++;
+          if (columnSchema.isSortColumn()) {
+            this.numberOfSortColumns++;
+            this.numberOfNoDictSortColumns++;
+          }
         }
         dimensions.add(carbonDimension);
       } else {
@@ -511,6 +530,14 @@ public class SegmentProperties {
     int[] bitLength = CarbonUtil.getDimensionBitLength(dimColumnsCardinality, dimensionPartitions);
     // create a key generator
     this.dimensionKeyGenerator = new MultiDimKeyVarLengthGenerator(bitLength);
+    if (this.getNumberOfDictSortColumns() == bitLength.length) {
+      this.sortColumnsGenerator = this.dimensionKeyGenerator;
+    } else {
+      int numberOfDictSortColumns = this.getNumberOfDictSortColumns();
+      int [] sortColumnBitLength = new int[numberOfDictSortColumns];
+      System.arraycopy(bitLength, 0, sortColumnBitLength, 0, numberOfDictSortColumns);
+      this.sortColumnsGenerator = new MultiDimKeyVarLengthGenerator(sortColumnBitLength);
+    }
     this.fixedLengthKeySplitter =
         new MultiDimKeyVarLengthVariableSplitGenerator(bitLength, dimensionPartitions);
     // get the size of each value in file block
@@ -636,6 +663,10 @@ public class SegmentProperties {
     return dimensionKeyGenerator;
   }
 
+  public KeyGenerator getSortColumnsGenerator() {
+    return sortColumnsGenerator;
+  }
+
   /**
    * @return the dimensions
    */
@@ -797,4 +828,15 @@ public class SegmentProperties {
     return CarbonUtil.getMeasureFromCurrentBlock(this.measures, columnId);
   }
 
+  public int getNumberOfSortColumns() {
+    return numberOfSortColumns;
+  }
+
+  public int getNumberOfNoDictSortColumns() {
+    return numberOfNoDictSortColumns;
+  }
+
+  public int getNumberOfDictSortColumns() {
+    return this.numberOfSortColumns - this.numberOfNoDictSortColumns;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
index 92c1e2f..4897985 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
@@ -24,6 +24,18 @@ import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFacto
 import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
 import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
 import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
+import org.apache.carbondata.core.util.ByteUtil;
+
+import org.apache.spark.sql.types.BooleanType;
+import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.types.Decimal;
+import org.apache.spark.sql.types.DecimalType;
+import org.apache.spark.sql.types.DoubleType;
+import org.apache.spark.sql.types.FloatType;
+import org.apache.spark.sql.types.IntegerType;
+import org.apache.spark.sql.types.LongType;
+import org.apache.spark.sql.types.ShortType;
+import org.apache.spark.sql.types.StringType;
 
 /**
  * This class is gives access to variable length dimension data chunk store
@@ -33,8 +45,8 @@ public class VariableLengthDimensionDataChunk extends AbstractDimensionDataChunk
   /**
    * Constructor for this class
    *
-   * @param dataChunkStore  data chunk
-   * @param chunkAttributes chunk attributes
+   * @param dataChunks  data chunk
+   * @param invertedIndex chunk attributes
    */
   public VariableLengthDimensionDataChunk(byte[] dataChunks, int[] invertedIndex,
       int[] invertedIndexReverse, int numberOfRows) {
@@ -114,7 +126,26 @@ public class VariableLengthDimensionDataChunk extends AbstractDimensionDataChunk
       if (value == null || Arrays.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, value)) {
         vector.putNull(vectorOffset++);
       } else {
-        vector.putBytes(vectorOffset++, value);
+        DataType dt = vector.getType();
+        if (dt instanceof StringType) {
+          vector.putBytes(vectorOffset++, value);
+        } else if (dt instanceof BooleanType) {
+          vector.putBoolean(vectorOffset++, ByteUtil.toBoolean(value));
+        } else if (dt instanceof ShortType) {
+          vector.putShort(vectorOffset++, ByteUtil.toShort(value, 0, value.length));
+        } else if (dt instanceof IntegerType) {
+          vector.putInt(vectorOffset++, ByteUtil.toInt(value, 0, value.length));
+        } else if (dt instanceof FloatType) {
+          vector.putFloat(vectorOffset++, ByteUtil.toFloat(value, 0));
+        } else if (dt instanceof DoubleType) {
+          vector.putDouble(vectorOffset++, ByteUtil.toDouble(value, 0));
+        } else if (dt instanceof LongType) {
+          vector.putLong(vectorOffset++, ByteUtil.toLong(value, 0, value.length));
+        } else if (dt instanceof DecimalType) {
+          vector.putDecimal(vectorOffset++,
+              Decimal.apply(ByteUtil.toBigDecimal(value, 0, value.length)),
+              DecimalType.MAX_PRECISION());
+        }
       }
     }
     return column + 1;
@@ -143,7 +174,26 @@ public class VariableLengthDimensionDataChunk extends AbstractDimensionDataChunk
       if (value == null || Arrays.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, value)) {
         vector.putNull(vectorOffset++);
       } else {
-        vector.putBytes(vectorOffset++, value);
+        DataType dt = vector.getType();
+        if (dt instanceof StringType) {
+          vector.putBytes(vectorOffset++, value);
+        } else if (dt instanceof BooleanType) {
+          vector.putBoolean(vectorOffset++, ByteUtil.toBoolean(value));
+        } else if (dt instanceof ShortType) {
+          vector.putShort(vectorOffset++, ByteUtil.toShort(value, 0, value.length));
+        } else if (dt instanceof IntegerType) {
+          vector.putInt(vectorOffset++, ByteUtil.toInt(value, 0, value.length));
+        } else if (dt instanceof FloatType) {
+          vector.putFloat(vectorOffset++, ByteUtil.toFloat(value, 0));
+        } else if (dt instanceof DoubleType) {
+          vector.putDouble(vectorOffset++, ByteUtil.toDouble(value, 0));
+        } else if (dt instanceof LongType) {
+          vector.putLong(vectorOffset++, ByteUtil.toLong(value, 0, value.length));
+        } else if (dt instanceof DecimalType) {
+          vector.putDecimal(vectorOffset++,
+              Decimal.apply(ByteUtil.toBigDecimal(value, 0, value.length)),
+              DecimalType.MAX_PRECISION());
+        }
       }
     }
     return column + 1;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
index 2e1709b..2f8aadf 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
@@ -49,18 +49,20 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
   private int[] eachColumnValueSize;
 
   /**
-   * this will be used during search for no dictionary column
+   * the number of no dictionary columns in SORT_COLUMNS
    */
-  private int numberOfNoDictionaryColumns;
+  private int numberOfNoDictSortColumns;
 
-  public BTreeDataRefNodeFinder(int[] eachColumnValueSize) {
-    this.eachColumnValueSize = eachColumnValueSize;
+  /**
+   * the number of columns in SORT_COLUMNS
+   */
+  private int numberOfSortColumns;
 
-    for (int i = 0; i < eachColumnValueSize.length; i++) {
-      if (eachColumnValueSize[i] == -1) {
-        numberOfNoDictionaryColumns++;
-      }
-    }
+  public BTreeDataRefNodeFinder(int[] eachColumnValueSize, int numberOfSortColumns,
+      int numberOfNoDictSortColumns) {
+    this.eachColumnValueSize = eachColumnValueSize;
+    this.numberOfNoDictSortColumns = numberOfNoDictSortColumns;
+    this.numberOfSortColumns = numberOfSortColumns;
   }
 
   /**
@@ -213,7 +215,7 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
     int dictionaryKeyOffset = 0;
     int nonDictionaryKeyOffset = 0;
     int compareResult = 0;
-    int processedNoDictionaryColumn = numberOfNoDictionaryColumns;
+    int processedNoDictionaryColumn = numberOfNoDictSortColumns;
     ByteBuffer firstNoDictionaryKeyBuffer = ByteBuffer.wrap(first.getNoDictionaryKeys());
     ByteBuffer secondNoDictionaryKeyBuffer = ByteBuffer.wrap(second.getNoDictionaryKeys());
     int actualOffset = 0;
@@ -221,7 +223,7 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
     int firstNoDcitionaryLength = 0;
     int secondNodeDictionaryLength = 0;
 
-    for (int i = 0; i < eachColumnValueSize.length; i++) {
+    for (int i = 0; i < numberOfSortColumns; i++) {
 
       if (eachColumnValueSize[i] != NO_DCITIONARY_COLUMN_VALUE) {
         compareResult = ByteUtil.UnsafeComparer.INSTANCE

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java
index 7bd338d..044a77b 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java
@@ -119,7 +119,8 @@ public class Bits implements Serializable {
   protected long[] get(long[] keys) {
     long[] words = new long[wsize];
     int ll = 0;
-    for (int i = lens.length - 1; i >= 0; i--) {
+    int minLength = Math.min(lens.length, keys.length);
+    for (int i = minLength - 1; i >= 0; i--) {
 
       long val = keys[i];
 
@@ -155,7 +156,8 @@ public class Bits implements Serializable {
   protected long[] get(int[] keys) {
     long[] words = new long[wsize];
     int ll = 0;
-    for (int i = lens.length - 1; i >= 0; i--) {
+    int minLength = Math.min(lens.length, keys.length);
+    for (int i = minLength - 1; i >= 0; i--) {
 
       long val = keys[i];
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
index 974cc81..3d114ec 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
@@ -17,8 +17,11 @@
 package org.apache.carbondata.core.metadata.converter;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.metadata.encoder.Encoding;
 import org.apache.carbondata.core.metadata.schema.BucketingInfo;
@@ -172,6 +175,13 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
     thriftColumnSchema.setInvisible(wrapperColumnSchema.isInvisible());
     thriftColumnSchema.setColumnReferenceId(wrapperColumnSchema.getColumnReferenceId());
     thriftColumnSchema.setSchemaOrdinal(wrapperColumnSchema.getSchemaOrdinal());
+
+    if (wrapperColumnSchema.isSortColumn()) {
+      Map<String, String> properties = new HashMap<String, String>();
+      properties.put(CarbonCommonConstants.SORT_COLUMNS, "true");
+      thriftColumnSchema.setColumnProperties(properties);
+    }
+
     return thriftColumnSchema;
   }
 
@@ -360,6 +370,14 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
     wrapperColumnSchema.setInvisible(externalColumnSchema.isInvisible());
     wrapperColumnSchema.setColumnReferenceId(externalColumnSchema.getColumnReferenceId());
     wrapperColumnSchema.setSchemaOrdinal(externalColumnSchema.getSchemaOrdinal());
+    wrapperColumnSchema.setSortColumn(false);
+    Map<String, String> properties = externalColumnSchema.getColumnProperties();
+    if (properties != null) {
+      String sortColumns = properties.get(CarbonCommonConstants.SORT_COLUMNS);
+      if (sortColumns != null) {
+        wrapperColumnSchema.setSortColumn(true);
+      }
+    }
     return wrapperColumnSchema;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index dbc8836..fae9960 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -116,6 +116,16 @@ public class CarbonTable implements Serializable {
    */
   private int blockSize;
 
+  /**
+   * the number of columns in SORT_COLUMNS
+   */
+  private int numberOfSortColumns;
+
+  /**
+   * the number of no dictionary columns in SORT_COLUMNS
+   */
+  private int numberOfNoDictSortColumns;
+
   public CarbonTable() {
     this.tableDimensionsMap = new HashMap<String, List<CarbonDimension>>();
     this.tableImplicitDimensionsMap = new HashMap<String, List<CarbonDimension>>();
@@ -238,10 +248,16 @@ public class CarbonTable implements Serializable {
           i = dimensionOrdinal - 1;
           complexTypeOrdinal = assignComplexOrdinal(complexDimension, complexTypeOrdinal);
         } else {
+          if (!columnSchema.isInvisible() && columnSchema.isSortColumn()) {
+            this.numberOfSortColumns++;
+          }
           if (!columnSchema.getEncodingList().contains(Encoding.DICTIONARY)) {
             CarbonDimension dimension =
                     new CarbonDimension(columnSchema, dimensionOrdinal++,
                             columnSchema.getSchemaOrdinal(), -1, -1, -1);
+            if (!columnSchema.isInvisible() && columnSchema.isSortColumn()) {
+              this.numberOfNoDictSortColumns++;
+            }
             allDimensions.add(dimension);
             primitiveDimensions.add(dimension);
           } else if (columnSchema.getEncodingList().contains(Encoding.DICTIONARY)
@@ -639,4 +655,12 @@ public class CarbonTable implements Serializable {
     }
     tableMeasuresMap.put(tableName, visibleMeasures);
   }
+
+  public int getNumberOfSortColumns() {
+    return numberOfSortColumns;
+  }
+
+  public int getNumberOfNoDictSortColumns() {
+    return numberOfNoDictSortColumns;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
index 7aa0900..08969fb 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
@@ -109,6 +109,8 @@ public class ColumnSchema implements Serializable {
    */
   private boolean invisible = false;
 
+  private boolean isSortColumn = false;
+
   /**
    * @return the columnName
    */
@@ -403,4 +405,12 @@ public class ColumnSchema implements Serializable {
   public void setSchemaOrdinal(int schemaOrdinal) {
     this.schemaOrdinal = schemaOrdinal;
   }
+
+  public boolean isSortColumn() {
+    return isSortColumn;
+  }
+
+  public void setSortColumn(boolean sortColumn) {
+    isSortColumn = sortColumn;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
index e5e4b3c..a7c93bb 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
@@ -88,7 +88,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
     List<Object[]> listBasedResult = new ArrayList<>(batchSize);
     int rowCounter = 0;
     int[] surrogateResult;
-    String[] noDictionaryKeys;
+    byte[][] noDictionaryKeys;
     byte[][] complexTypeKeyArray;
     BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache =
         scannedResult.getDeleteDeltaDataCache();
@@ -96,7 +96,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
       Object[] row = new Object[queryDimensions.length + queryMeasures.length];
       if (isDimensionExists) {
         surrogateResult = scannedResult.getDictionaryKeyIntegerArray();
-        noDictionaryKeys = scannedResult.getNoDictionaryKeyStringArray();
+        noDictionaryKeys = scannedResult.getNoDictionaryKeyArray();
         complexTypeKeyArray = scannedResult.getComplexTypeKeyArray();
         dictionaryColumnIndex = 0;
         noDictionaryColumnIndex = 0;
@@ -120,7 +120,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
   }
 
   protected void fillDimensionData(AbstractScannedResult scannedResult, int[] surrogateResult,
-      String[] noDictionaryKeys, byte[][] complexTypeKeyArray,
+      byte[][] noDictionaryKeys, byte[][] complexTypeKeyArray,
       Map<Integer, GenericQueryType> comlexDimensionInfoMap, Object[] row, int i) {
     if (!dictionaryEncodingArray[i]) {
       if (implictColumnArray[i]) {
@@ -134,9 +134,9 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
               DataTypeUtil.getDataBasedOnDataType(scannedResult.getBlockletId(), DataType.STRING);
         }
       } else {
-        row[order[i]] = DataTypeUtil
-            .getDataBasedOnDataType(noDictionaryKeys[noDictionaryColumnIndex++],
-                queryDimensions[i].getDimension().getDataType());
+        row[order[i]] = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(
+            noDictionaryKeys[noDictionaryColumnIndex++],
+            queryDimensions[i].getDimension().getDataType());
       }
     } else if (directDictionaryEncodingArray[i]) {
       if (directDictionaryGenerators[i] != null) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
index 71045ff..7c07923 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
@@ -48,7 +48,7 @@ public class RestructureBasedDictionaryResultCollector extends DictionaryBasedRe
     List<Object[]> listBasedResult = new ArrayList<>(batchSize);
     int rowCounter = 0;
     int[] surrogateResult;
-    String[] noDictionaryKeys;
+    byte[][] noDictionaryKeys;
     byte[][] complexTypeKeyArray;
     BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache =
         scannedResult.getDeleteDeltaDataCache();
@@ -58,7 +58,7 @@ public class RestructureBasedDictionaryResultCollector extends DictionaryBasedRe
       Object[] row = new Object[queryDimensions.length + queryMeasures.length];
       if (isDimensionExists) {
         surrogateResult = scannedResult.getDictionaryKeyIntegerArray();
-        noDictionaryKeys = scannedResult.getNoDictionaryKeyStringArray();
+        noDictionaryKeys = scannedResult.getNoDictionaryKeyArray();
         complexTypeKeyArray = scannedResult.getComplexTypeKeyArray();
         dictionaryColumnIndex = 0;
         noDictionaryColumnIndex = 0;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
index 264c0ae..b5341a0 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
@@ -116,7 +116,9 @@ public class FilterExpressionProcessor implements FilterProcessor {
             + searchEndKey.getNoDictionaryKeys());
     long startTimeInMillis = System.currentTimeMillis();
     DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(
-        tableSegment.getSegmentProperties().getEachDimColumnValueSize());
+        tableSegment.getSegmentProperties().getEachDimColumnValueSize(),
+        tableSegment.getSegmentProperties().getNumberOfSortColumns(),
+        tableSegment.getSegmentProperties().getNumberOfNoDictSortColumns());
     DataRefNode startBlock = blockFinder.findFirstDataBlock(btreeNode, searchStartKey);
     DataRefNode endBlock = blockFinder.findLastDataBlock(btreeNode, searchEndKey);
     FilterExecuter filterExecuter =

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index 770aa7e..b116d9a 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -355,10 +355,23 @@ public final class FilterUtil {
    * @return DimColumnFilterInfo
    */
   public static DimColumnFilterInfo getNoDictionaryValKeyMemberForFilter(
-      List<String> evaluateResultListFinal, boolean isIncludeFilter) {
+      List<String> evaluateResultListFinal, boolean isIncludeFilter, DataType dataType)
+      throws FilterUnsupportedException {
     List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
-    for (String result : evaluateResultListFinal) {
-      filterValuesList.add(result.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
+    String result = null;
+    try {
+      int length = evaluateResultListFinal.size();
+      for (int i = 0; i < length; i++) {
+        result = evaluateResultListFinal.get(i);
+        if (CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(result)) {
+          filterValuesList.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY);
+          continue;
+        }
+        filterValuesList.add(
+              DataTypeUtil.getBytesBasedOnDataTypeForNoDictionaryColumn(result, dataType));
+      }
+    } catch (Throwable ex) {
+      throw new FilterUnsupportedException("Unsupported Filter condition: " + result, ex);
     }
 
     Comparator<byte[]> filterNoDictValueComaparator = new Comparator<byte[]>() {
@@ -879,8 +892,10 @@ public final class FilterUtil {
       }
       int keyOrdinalOfDimensionFromCurrentBlock = dimensionFromCurrentBlock.getKeyOrdinal();
       for (DimColumnFilterInfo info : values) {
-        if (startKey[keyOrdinalOfDimensionFromCurrentBlock] < info.getFilterList().get(0)) {
-          startKey[keyOrdinalOfDimensionFromCurrentBlock] = info.getFilterList().get(0);
+        if (keyOrdinalOfDimensionFromCurrentBlock < startKey.length) {
+          if (startKey[keyOrdinalOfDimensionFromCurrentBlock] < info.getFilterList().get(0)) {
+            startKey[keyOrdinalOfDimensionFromCurrentBlock] = info.getFilterList().get(0);
+          }
         }
       }
       long[] newStartKey = new long[startKey.length];
@@ -944,10 +959,12 @@ public final class FilterUtil {
       }
       int keyOrdinalOfDimensionFromCurrentBlock = dimensionFromCurrentBlock.getKeyOrdinal();
       for (DimColumnFilterInfo info : values) {
-        if (endKey[keyOrdinalOfDimensionFromCurrentBlock] > info.getFilterList()
-            .get(info.getFilterList().size() - 1)) {
-          endKey[keyOrdinalOfDimensionFromCurrentBlock] =
-              info.getFilterList().get(info.getFilterList().size() - 1);
+        if (keyOrdinalOfDimensionFromCurrentBlock < endKey.length) {
+          if (endKey[keyOrdinalOfDimensionFromCurrentBlock] > info.getFilterList()
+              .get(info.getFilterList().size() - 1)) {
+            endKey[keyOrdinalOfDimensionFromCurrentBlock] =
+                info.getFilterList().get(info.getFilterList().size() - 1);
+          }
         }
       }
       long[] newEndKey = new long[endKey.length];
@@ -1040,44 +1057,43 @@ public final class FilterUtil {
    */
   public static IndexKey prepareDefaultEndIndexKey(SegmentProperties segmentProperties)
       throws KeyGenException {
-    long[] dictionarySurrogateKey =
-        new long[segmentProperties.getDimensions().size() - segmentProperties
-            .getNumberOfNoDictionaryDimension()];
+    long[] dictionarySurrogateKey = new long[segmentProperties.getNumberOfDictSortColumns()];
     int index = 0;
     int[] dimColumnsCardinality = segmentProperties.getDimColumnsCardinality();
-    for (int i = 0; i < dimColumnsCardinality.length; i++) {
+    for (int i = 0; i < dictionarySurrogateKey.length; i++) {
       dictionarySurrogateKey[index++] = dimColumnsCardinality[i];
     }
     IndexKey endIndexKey;
     byte[] dictionaryendMdkey =
-        segmentProperties.getDimensionKeyGenerator().generateKey(dictionarySurrogateKey);
+        segmentProperties.getSortColumnsGenerator().generateKey(dictionarySurrogateKey);
     byte[] noDictionaryEndKeyBuffer = getNoDictionaryDefaultEndKey(segmentProperties);
     endIndexKey = new IndexKey(dictionaryendMdkey, noDictionaryEndKeyBuffer);
     return endIndexKey;
   }
 
   public static byte[] getNoDictionaryDefaultEndKey(SegmentProperties segmentProperties) {
+
+    int numberOfNoDictionaryDimension = segmentProperties.getNumberOfNoDictSortColumns();
     // in case of non filter query when no dictionary columns are present we
     // need to set the default end key, as for non filter query
     // we need to get the last
     // block of the btree so we are setting the max byte value in the end key
     ByteBuffer noDictionaryEndKeyBuffer = ByteBuffer.allocate(
-        (segmentProperties.getNumberOfNoDictionaryDimension()
-            * CarbonCommonConstants.SHORT_SIZE_IN_BYTE) + segmentProperties
-            .getNumberOfNoDictionaryDimension());
+        (numberOfNoDictionaryDimension * CarbonCommonConstants.SHORT_SIZE_IN_BYTE)
+            + numberOfNoDictionaryDimension);
     // end key structure will be
     //<Offset of first No Dictionary key in 2 Bytes><Offset of second No Dictionary key in 2 Bytes>
     //<Offset of n No Dictionary key in 2 Bytes><first no dictionary column value>
     // <second no dictionary column value> <N no dictionary column value>
     //example if we have 2 no dictionary column
     //<[0,4,0,5,127,127]>
-    short startPoint = (short) (segmentProperties.getNumberOfNoDictionaryDimension()
-        * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
-    for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
+    short startPoint =
+        (short) (numberOfNoDictionaryDimension * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+    for (int i = 0; i < numberOfNoDictionaryDimension; i++) {
       noDictionaryEndKeyBuffer.putShort((startPoint));
       startPoint++;
     }
-    for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
+    for (int i = 0; i < numberOfNoDictionaryDimension; i++) {
       noDictionaryEndKeyBuffer.put((byte) 127);
     }
     return noDictionaryEndKeyBuffer.array();
@@ -1094,11 +1110,9 @@ public final class FilterUtil {
   public static IndexKey prepareDefaultStartIndexKey(SegmentProperties segmentProperties)
       throws KeyGenException {
     IndexKey startIndexKey;
-    long[] dictionarySurrogateKey =
-        new long[segmentProperties.getDimensions().size() - segmentProperties
-            .getNumberOfNoDictionaryDimension()];
+    long[] dictionarySurrogateKey = new long[segmentProperties.getNumberOfDictSortColumns()];
     byte[] dictionaryStartMdkey =
-        segmentProperties.getDimensionKeyGenerator().generateKey(dictionarySurrogateKey);
+        segmentProperties.getSortColumnsGenerator().generateKey(dictionarySurrogateKey);
     byte[] noDictionaryStartKeyArray = getNoDictionaryDefaultStartKey(segmentProperties);
 
     startIndexKey = new IndexKey(dictionaryStartMdkey, noDictionaryStartKeyArray);
@@ -1106,26 +1120,27 @@ public final class FilterUtil {
   }
 
   public static byte[] getNoDictionaryDefaultStartKey(SegmentProperties segmentProperties) {
+
+    int numberOfNoDictionaryDimension = segmentProperties.getNumberOfNoDictSortColumns();
     // in case of non filter query when no dictionary columns are present we
     // need to set the default start key, as for non filter query we need to get the first
     // block of the btree so we are setting the least byte value in the start key
     ByteBuffer noDictionaryStartKeyBuffer = ByteBuffer.allocate(
-        (segmentProperties.getNumberOfNoDictionaryDimension()
-            * CarbonCommonConstants.SHORT_SIZE_IN_BYTE) + segmentProperties
-            .getNumberOfNoDictionaryDimension());
+        (numberOfNoDictionaryDimension * CarbonCommonConstants.SHORT_SIZE_IN_BYTE)
+            + numberOfNoDictionaryDimension);
     // end key structure will be
     //<Offset of first No Dictionary key in 2 Bytes><Offset of second No Dictionary key in 2 Bytes>
     //<Offset of n No Dictionary key in 2 Bytes><first no dictionary column value>
     // <second no dictionary column value> <N no dictionary column value>
     //example if we have 2 no dictionary column
     //<[0,4,0,5,0,0]>
-    short startPoint = (short) (segmentProperties.getNumberOfNoDictionaryDimension()
-        * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
-    for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
+    short startPoint =
+        (short) (numberOfNoDictionaryDimension * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+    for (int i = 0; i < numberOfNoDictionaryDimension; i++) {
       noDictionaryStartKeyBuffer.putShort((startPoint));
       startPoint++;
     }
-    for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
+    for (int i = 0; i < numberOfNoDictionaryDimension; i++) {
       noDictionaryStartKeyBuffer.put((byte) 0);
     }
     return noDictionaryStartKeyBuffer.array();
@@ -1181,8 +1196,8 @@ public final class FilterUtil {
       FilterResolverIntf filterResolver, List<IndexKey> listOfStartEndKeys) {
     IndexKey searchStartKey = null;
     IndexKey searchEndKey = null;
-    long[] startKey = new long[segmentProperties.getDimensionKeyGenerator().getDimCount()];
-    long[] endKey = new long[segmentProperties.getDimensionKeyGenerator().getDimCount()];
+    long[] startKey = new long[segmentProperties.getNumberOfDictSortColumns()];
+    long[] endKey = new long[segmentProperties.getNumberOfDictSortColumns()];
     List<byte[]> listOfStartKeyByteArray =
         new ArrayList<byte[]>(segmentProperties.getNumberOfNoDictionaryDimension());
     List<byte[]> listOfEndKeyByteArray =
@@ -1234,12 +1249,23 @@ public final class FilterUtil {
     pruneStartAndEndKeys(setOfStartKeyByteArray, listOfStartKeyByteArray);
     pruneStartAndEndKeys(setOfEndKeyByteArray, listOfEndKeyByteArray);
 
+    if (segmentProperties.getNumberOfNoDictSortColumns() == 0) {
+      listOfStartKeyByteArray = new ArrayList<byte[]>();
+      listOfEndKeyByteArray = new ArrayList<byte[]>();
+    } else if (segmentProperties.getNumberOfNoDictSortColumns() < listOfStartKeyByteArray
+        .size()) {
+      while (segmentProperties.getNumberOfNoDictSortColumns() < listOfStartKeyByteArray.size()) {
+        listOfStartKeyByteArray.remove(listOfStartKeyByteArray.size() - 1);
+        listOfEndKeyByteArray.remove(listOfEndKeyByteArray.size() - 1);
+      }
+    }
+
     searchStartKey = FilterUtil
-        .createIndexKeyFromResolvedFilterVal(startKey, segmentProperties.getDimensionKeyGenerator(),
+        .createIndexKeyFromResolvedFilterVal(startKey, segmentProperties.getSortColumnsGenerator(),
             FilterUtil.getKeyWithIndexesAndValues(listOfStartKeyByteArray));
 
     searchEndKey = FilterUtil
-        .createIndexKeyFromResolvedFilterVal(endKey, segmentProperties.getDimensionKeyGenerator(),
+        .createIndexKeyFromResolvedFilterVal(endKey, segmentProperties.getSortColumnsGenerator(),
             FilterUtil.getKeyWithIndexesAndValues(listOfEndKeyByteArray));
     listOfStartEndKeys.add(searchStartKey);
     listOfStartEndKeys.add(searchEndKey);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index 306becf..bd83eec 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -157,7 +157,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
       CarbonDimension currentBlockDimension =
           segmentProperties.getDimensions().get(dimensionBlocksIndex[0]);
       defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension,
-          this.segmentProperties.getDimensionKeyGenerator());
+          this.segmentProperties.getSortColumnsGenerator());
     }
     if (dimensionColumnDataChunk.isExplicitSorted()
         && dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
index f2a49d9..a439228 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
@@ -157,7 +157,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
       CarbonDimension currentBlockDimension =
           segmentProperties.getDimensions().get(dimensionBlocksIndex[0]);
       defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension,
-          this.segmentProperties.getDimensionKeyGenerator());
+          this.segmentProperties.getSortColumnsGenerator());
     }
     if (dimensionColumnDataChunk.isExplicitSorted()
         && dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
index 351d2c0..efedd93 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
@@ -67,7 +67,8 @@ public class NoDictionaryTypeVisitor implements ResolvedFilterInfoVisitorIntf {
       throw new FilterUnsupportedException(e);
     }
     resolvedFilterObject = FilterUtil
-        .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter());
+        .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(),
+            metadata.getColumnExpression().getDataType());
     visitableObj.setFilterValues(resolvedFilterObject);
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java
index 8d0a8b4..d703ed0 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java
@@ -72,7 +72,8 @@ public class RangeNoDictionaryTypeVisitor extends NoDictionaryTypeVisitor
       throw new FilterUnsupportedException(e);
     }
     resolvedFilterObject = FilterUtil
-        .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter());
+        .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(),
+            metadata.getColumnExpression().getDataType());
     visitableObj.setFilterValues(resolvedFilterObject);
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
index 4e5681c..cdab4e4 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
@@ -104,7 +104,9 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
 
   private void intialiseInfos() {
     for (BlockExecutionInfo blockInfo : blockExecutionInfos) {
-      DataRefNodeFinder finder = new BTreeDataRefNodeFinder(blockInfo.getEachColumnValueSize());
+      DataRefNodeFinder finder = new BTreeDataRefNodeFinder(blockInfo.getEachColumnValueSize(),
+          blockInfo.getDataBlock().getSegmentProperties().getNumberOfSortColumns(),
+          blockInfo.getDataBlock().getSegmentProperties().getNumberOfNoDictSortColumns());
       DataRefNode startDataBlock = finder
           .findFirstDataBlock(blockInfo.getDataBlock().getDataRefNode(), blockInfo.getStartKey());
       while (startDataBlock.nodeNumber() < blockInfo.getStartBlockletIndex()) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
index 4952e07..82a0b45 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
@@ -17,10 +17,15 @@
 
 package org.apache.carbondata.core.scan.result.vector;
 
+import org.apache.spark.sql.types.DataType;
 import org.apache.spark.sql.types.Decimal;
 
 public interface CarbonColumnVector {
 
+  void putBoolean(int rowId, boolean value);
+
+  void putFloat(int rowId, float value);
+
   void putShort(int rowId, short value);
 
   void putShorts(int rowId, int count, short value);
@@ -59,4 +64,6 @@ public interface CarbonColumnVector {
 
   void reset();
 
+  DataType getType();
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
index 48a081d..52f2f31 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
@@ -21,6 +21,7 @@ import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.BitSet;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.datastore.block.BlockInfo;
@@ -195,6 +196,12 @@ public abstract class AbstractDataFileFooterConverter {
     wrapperColumnSchema.setColumnGroup(externalColumnSchema.getColumn_group_id());
     wrapperColumnSchema.setScale(externalColumnSchema.getScale());
     wrapperColumnSchema.setDefaultValue(externalColumnSchema.getDefault_value());
+    Map<String, String> properties = externalColumnSchema.getColumnProperties();
+    if (properties != null) {
+      if (properties.get(CarbonCommonConstants.SORT_COLUMNS) != null) {
+        wrapperColumnSchema.setSortColumn(true);
+      }
+    }
     return wrapperColumnSchema;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
index 9b2c2ed..84a8634 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
@@ -17,7 +17,11 @@
 
 package org.apache.carbondata.core.util;
 
+import java.io.UnsupportedEncodingException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
 import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.memory.CarbonUnsafe;
@@ -27,7 +31,13 @@ import org.apache.carbondata.core.memory.CarbonUnsafe;
  */
 public final class ByteUtil {
 
-  private static final int SIZEOF_LONG = 8;
+  public static final int SIZEOF_LONG = 8;
+
+  public static final int SIZEOF_INT = 4;
+
+  public static final int SIZEOF_SHORT = 2;
+
+  public static final String UTF8_CSN = StandardCharsets.UTF_8.name();
 
   private ByteUtil() {
 
@@ -363,4 +373,328 @@ public final class ByteUtil {
 
   }
 
+  /**
+   * Stirng => byte[]
+   *
+   * @param s
+   * @return
+   */
+  public static byte[] toBytes(String s) {
+    try {
+      return s.getBytes(UTF8_CSN);
+    } catch (UnsupportedEncodingException e) {
+      // should never happen!
+      throw new IllegalArgumentException("UTF8 decoding is not supported", e);
+    }
+  }
+
+  /**
+   * byte[] => String
+   *
+   * @param b
+   * @param off
+   * @param len
+   * @return
+   */
+  public static String toString(final byte[] b, int off, int len) {
+    if (b == null) {
+      return null;
+    }
+    if (len == 0) {
+      return "";
+    }
+    try {
+      return new String(b, off, len, UTF8_CSN);
+    } catch (UnsupportedEncodingException e) {
+      // should never happen!
+      throw new IllegalArgumentException("UTF8 encoding is not supported", e);
+    }
+  }
+
+  /**
+   * boolean => byte[]
+   *
+   * @param b
+   * @return
+   */
+  public static byte[] toBytes(final boolean b) {
+    return new byte[] { b ? (byte) -1 : (byte) 0 };
+  }
+
+  /**
+   * byte[] => boolean
+   *
+   * @param b
+   * @return
+   */
+  public static boolean toBoolean(final byte[] b) {
+    if (b.length != 1) {
+      throw new IllegalArgumentException("Array has wrong size: " + b.length);
+    }
+    return b[0] != (byte) 0;
+  }
+
+  /**
+   * short => byte[]
+   *
+   * @param val
+   * @return
+   */
+  public static byte[] toBytes(short val) {
+    byte[] b = new byte[SIZEOF_SHORT];
+    b[1] = (byte) val;
+    val >>= 8;
+    b[0] = (byte) val;
+    return b;
+  }
+
+  /**
+   * byte[] => short
+   *
+   * @param bytes
+   * @param offset
+   * @param length
+   * @return
+   */
+  public static short toShort(byte[] bytes, int offset, final int length) {
+    if (length != SIZEOF_SHORT || offset + length > bytes.length) {
+      throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT);
+    }
+    if (CarbonUnsafe.unsafe != null) {
+      if (CarbonUnsafe.ISLITTLEENDIAN) {
+        return Short.reverseBytes(
+            CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
+      } else {
+        return CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+      }
+    } else {
+      short n = 0;
+      n ^= bytes[offset] & 0xFF;
+      n <<= 8;
+      n ^= bytes[offset + 1] & 0xFF;
+      return n;
+    }
+  }
+
+  /**
+   * int => byte[]
+   *
+   * @param val
+   * @return
+   */
+  public static byte[] toBytes(int val) {
+    byte[] b = new byte[4];
+    for (int i = 3; i > 0; i--) {
+      b[i] = (byte) val;
+      val >>>= 8;
+    }
+    b[0] = (byte) val;
+    return b;
+  }
+
+  /**
+   * byte[] => int
+   *
+   * @param bytes
+   * @param offset
+   * @param length
+   * @return
+   */
+  public static int toInt(byte[] bytes, int offset, final int length) {
+    if (length != SIZEOF_INT || offset + length > bytes.length) {
+      throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT);
+    }
+    if (CarbonUnsafe.unsafe != null) {
+      if (CarbonUnsafe.ISLITTLEENDIAN) {
+        return Integer.reverseBytes(
+            CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
+      } else {
+        return CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+      }
+    } else {
+      int n = 0;
+      for (int i = offset; i < (offset + length); i++) {
+        n <<= 8;
+        n ^= bytes[i] & 0xFF;
+      }
+      return n;
+    }
+  }
+
+  /**
+   * float => byte[]
+   *
+   * @param f
+   * @return
+   */
+  public static byte[] toBytes(final float f) {
+    // Encode it as int
+    return toBytes(Float.floatToRawIntBits(f));
+  }
+
+  /**
+   * byte[] => float
+   *
+   * @param bytes
+   * @param offset
+   * @return
+   */
+  public static float toFloat(byte[] bytes, int offset) {
+    return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT));
+  }
+
+  /**
+   * long => byte[]
+   *
+   * @param val
+   * @return
+   */
+  public static byte[] toBytes(long val) {
+    byte[] b = new byte[8];
+    for (int i = 7; i > 0; i--) {
+      b[i] = (byte) val;
+      val >>>= 8;
+    }
+    b[0] = (byte) val;
+    return b;
+  }
+
+  /**
+   * byte[] => long
+   */
+  public static long toLong(byte[] bytes, int offset, final int length) {
+    if (length != SIZEOF_LONG || offset + length > bytes.length) {
+      throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG);
+    }
+    if (CarbonUnsafe.unsafe != null) {
+      if (CarbonUnsafe.ISLITTLEENDIAN) {
+        return Long.reverseBytes(
+            CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
+      } else {
+        return CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+      }
+    } else {
+      long l = 0;
+      for (int i = offset; i < offset + length; i++) {
+        l <<= 8;
+        l ^= bytes[i] & 0xFF;
+      }
+      return l;
+    }
+  }
+
+  /**
+   * doube => byte[]
+   *
+   * @param d
+   * @return
+   */
+  public static byte[] toBytes(final double d) {
+    // Encode it as a long
+    return toBytes(Double.doubleToRawLongBits(d));
+  }
+
+  /**
+   * byte[] => double
+   *
+   * @param bytes
+   * @param offset
+   * @return
+   */
+  public static double toDouble(final byte[] bytes, final int offset) {
+    return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG));
+  }
+
+  /**
+   * BigDecimal => byte[]
+   *
+   * @param val
+   * @return
+   */
+  public static byte[] toBytes(BigDecimal val) {
+    byte[] valueBytes = val.unscaledValue().toByteArray();
+    byte[] result = new byte[valueBytes.length + SIZEOF_INT];
+    int offset = putInt(result, 0, val.scale());
+    putBytes(result, offset, valueBytes, 0, valueBytes.length);
+    return result;
+  }
+
+  /**
+   * byte[] => BigDecimal
+   *
+   * @param bytes
+   * @param offset
+   * @param length
+   * @return
+   */
+  public static BigDecimal toBigDecimal(byte[] bytes, int offset, final int length) {
+    if (bytes == null || length < SIZEOF_INT + 1 || (offset + length > bytes.length)) {
+      return null;
+    }
+
+    int scale = toInt(bytes, offset, bytes.length);
+    byte[] tcBytes = new byte[length - SIZEOF_INT];
+    System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT);
+    return new BigDecimal(new BigInteger(tcBytes), scale);
+  }
+
+  private static IllegalArgumentException explainWrongLengthOrOffset(final byte[] bytes,
+      final int offset, final int length, final int expectedLength) {
+    String reason;
+    if (length != expectedLength) {
+      reason = "Wrong length: " + length + ", expected " + expectedLength;
+    } else {
+      reason = "offset (" + offset + ") + length (" + length + ") exceed the"
+          + " capacity of the array: " + bytes.length;
+    }
+    return new IllegalArgumentException(reason);
+  }
+
+  /**
+   * Put an int value out to the specified byte array position.
+   *
+   * @param bytes  the byte array
+   * @param offset position in the array
+   * @param val    int to write out
+   * @return incremented offset
+   * @throws IllegalArgumentException if the byte array given doesn't have
+   *                                  enough room at the offset specified.
+   */
+  public static int putInt(byte[] bytes, int offset, int val) {
+    if (bytes.length - offset < SIZEOF_INT) {
+      throw new IllegalArgumentException(
+          "Not enough room to put an int at" + " offset " + offset + " in a " + bytes.length
+              + " byte array");
+    }
+    if (CarbonUnsafe.unsafe != null) {
+      if (CarbonUnsafe.ISLITTLEENDIAN) {
+        val = Integer.reverseBytes(val);
+      }
+      CarbonUnsafe.unsafe.putInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET, val);
+      return offset + ByteUtil.SIZEOF_INT;
+    } else {
+      for (int i = offset + 3; i > offset; i--) {
+        bytes[i] = (byte) val;
+        val >>>= 8;
+      }
+      bytes[offset] = (byte) val;
+      return offset + SIZEOF_INT;
+    }
+  }
+
+  /**
+   * Put bytes at the specified byte array position.
+   *
+   * @param tgtBytes  the byte array
+   * @param tgtOffset position in the array
+   * @param srcBytes  array to write out
+   * @param srcOffset source offset
+   * @param srcLength source length
+   * @return incremented offset
+   */
+  public static int putBytes(byte[] tgtBytes, int tgtOffset, byte[] srcBytes, int srcOffset,
+      int srcLength) {
+    System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength);
+    return tgtOffset + srcLength;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index a442087..7c63f28 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -641,6 +641,24 @@ public final class CarbonUtil {
   }
 
   /**
+   * remove the quote char for a string, e.g. "abc" => abc, 'abc' => abc
+   * @param parseStr
+   * @return
+   */
+  public static String unquoteChar(String parseStr) {
+    if (parseStr == null) {
+      return null;
+    }
+    if (parseStr.startsWith("'") && parseStr.endsWith("'")) {
+      return parseStr.substring(1, parseStr.length() - 1);
+    } else if (parseStr.startsWith("\"") && parseStr.endsWith("\"")) {
+      return parseStr.substring(1, parseStr.length() - 1);
+    } else {
+      return parseStr;
+    }
+  }
+
+  /**
    * special char delimiter Converter
    *
    * @param delimiter

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 76df425..9edd121 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -323,6 +323,76 @@ public final class DataTypeUtil {
 
   }
 
+  public static byte[] getBytesBasedOnDataTypeForNoDictionaryColumn(String dimensionValue,
+      DataType actualDataType) throws Throwable {
+    switch (actualDataType) {
+      case STRING:
+        return ByteUtil.toBytes(dimensionValue);
+      case BOOLEAN:
+        return ByteUtil.toBytes(Boolean.parseBoolean(dimensionValue));
+      case SHORT:
+        return ByteUtil.toBytes(Short.parseShort(dimensionValue));
+      case INT:
+        return ByteUtil.toBytes(Integer.parseInt(dimensionValue));
+      case FLOAT:
+        return ByteUtil.toBytes(Float.parseFloat(dimensionValue));
+      case LONG:
+        return ByteUtil.toBytes(Long.parseLong(dimensionValue));
+      case DOUBLE:
+        return ByteUtil.toBytes(Double.parseDouble(dimensionValue));
+      case DECIMAL:
+        return ByteUtil.toBytes(new BigDecimal(dimensionValue));
+      default:
+        return ByteUtil.toBytes(dimensionValue);
+    }
+  }
+
+
+  /**
+   * Below method will be used to convert the data passed to its actual data
+   * type
+   *
+   * @param dataInBytes    data
+   * @param actualDataType actual data type
+   * @return actual data after conversion
+   */
+  public static Object getDataBasedOnDataTypeForNoDictionaryColumn(byte[] dataInBytes,
+      DataType actualDataType) {
+    if (null == dataInBytes || Arrays
+        .equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, dataInBytes)) {
+      return null;
+    }
+    try {
+      switch (actualDataType) {
+        case STRING:
+          return UTF8String.fromBytes(dataInBytes);
+        case BOOLEAN:
+          return ByteUtil.toBoolean(dataInBytes);
+        case SHORT:
+          return ByteUtil.toShort(dataInBytes, 0, dataInBytes.length);
+        case INT:
+          return ByteUtil.toInt(dataInBytes, 0, dataInBytes.length);
+        case FLOAT:
+          return ByteUtil.toFloat(dataInBytes, 0);
+        case LONG:
+          return ByteUtil.toLong(dataInBytes, 0, dataInBytes.length);
+        case DOUBLE:
+          return ByteUtil.toDouble(dataInBytes, 0);
+        case DECIMAL:
+          return ByteUtil.toBigDecimal(dataInBytes, 0, dataInBytes.length);
+        default:
+          return ByteUtil.toString(dataInBytes, 0, dataInBytes.length);
+      }
+    } catch (Throwable ex) {
+      String data = new String(dataInBytes, CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
+      LOGGER.error("Cannot convert" + data + " to " + actualDataType.getName() + " type value" + ex
+          .getMessage());
+      LOGGER.error("Problem while converting data type" + data);
+      return null;
+    }
+  }
+
+
   /**
    * Below method will be used to convert the data passed to its actual data
    * type

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
index a4adf92..245ff48 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
@@ -68,7 +68,7 @@ public class BTreeBlockFinderTest extends TestCase {
     builder.build(infos);
     DataRefNode dataBlock = builder.get();
     assertTrue(dataBlock != null);
-    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 });
+    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }, 1, 1);
     ByteBuffer buffer = ByteBuffer.allocate(4 + 2);
     buffer.rewind();
     buffer.putShort((short) 1);
@@ -88,7 +88,7 @@ public class BTreeBlockFinderTest extends TestCase {
     builder.build(infos);
     DataRefNode dataBlock = builder.get();
     assertTrue(dataBlock != null);
-    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 });
+    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }, 1, 1);
     ByteBuffer buffer = ByteBuffer.allocate(4 + 1);
     buffer.rewind();
     buffer.put((byte) 1);
@@ -109,7 +109,7 @@ public class BTreeBlockFinderTest extends TestCase {
     builder.build(infos);
     DataRefNode dataBlock = builder.get();
     assertTrue(dataBlock != null);
-    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 });
+    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
     int[] dimensionBitLength =
         CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
     KeyGenerator multiDimKeyVarLengthGenerator =
@@ -131,7 +131,7 @@ public class BTreeBlockFinderTest extends TestCase {
     builder.build(infos);
     DataRefNode dataBlock = builder.get();
     assertTrue(dataBlock != null);
-    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 });
+    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
     int[] dimensionBitLength =
         CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
     KeyGenerator multiDimKeyVarLengthGenerator =
@@ -160,7 +160,7 @@ public class BTreeBlockFinderTest extends TestCase {
     builder.build(infos);
     DataRefNode dataBlock = builder.get();
     assertTrue(dataBlock != null);
-    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 });
+    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
     int[] dimensionBitLength =
         CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
     KeyGenerator multiDimKeyVarLengthGenerator =

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
index 06063a4..406fbb7 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
@@ -37,6 +37,7 @@ import org.apache.carbondata.core.scan.expression.ColumnExpression;
 import org.apache.carbondata.core.scan.expression.Expression;
 import org.apache.carbondata.core.scan.expression.LiteralExpression;
 import org.apache.carbondata.core.scan.expression.conditional.ListExpression;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
 import org.apache.carbondata.core.scan.filter.intf.RowImpl;
 
 import mockit.Mock;
@@ -362,7 +363,7 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest {
     assertFalse(result);
   }
 
-  @Test public void testGetNoDictionaryValKeyMemberForFilter() {
+  @Test public void testGetNoDictionaryValKeyMemberForFilter() throws FilterUnsupportedException {
     boolean isIncludeFilter = true;
     AbsoluteTableIdentifier absoluteTableIdentifier =
         new AbsoluteTableIdentifier(this.carbonStorePath, carbonTableIdentifier);
@@ -370,7 +371,9 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest {
     List<String> evaluateResultListFinal = new ArrayList<>();
     evaluateResultListFinal.add("test1");
     evaluateResultListFinal.add("test2");
-    assertTrue(FilterUtil.getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, isIncludeFilter) instanceof DimColumnFilterInfo);
+    assertTrue(FilterUtil
+        .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, isIncludeFilter,
+            DataType.STRING) instanceof DimColumnFilterInfo);
   }
 
   @Test public void testPrepareDefaultStartIndexKey() throws KeyGenException {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index b330f12..540a69c 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -578,7 +578,9 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
 
       // Add all blocks of btree into result
       DataRefNodeFinder blockFinder =
-          new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize());
+          new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize(),
+              segmentProperties.getNumberOfSortColumns(),
+              segmentProperties.getNumberOfNoDictSortColumns());
       DataRefNode startBlock =
           blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), startIndexKey);
       DataRefNode endBlock =

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
index 7ba6133..d476f74 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
@@ -199,7 +199,9 @@ class InMemoryBTreeIndex implements Index {
 
       // Add all blocks of btree into result
       DataRefNodeFinder blockFinder =
-          new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize());
+          new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize(),
+              segmentProperties.getNumberOfSortColumns(),
+              segmentProperties.getNumberOfNoDictSortColumns());
       DataRefNode startBlock =
           blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), startIndexKey);
       DataRefNode endBlock =

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java
index 43953d0..cfdbd37 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java
@@ -87,6 +87,9 @@ public class DictionaryDecodeReadSupport<T> implements CarbonReadSupport<T> {
    * threshold is reached
    */
   @Override public void close() {
+    if (dictionaries == null) {
+      return;
+    }
     for (int i = 0; i < dictionaries.length; i++) {
       CarbonUtil.clearDictionaryCache(dictionaries[i]);
     }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
index e39d214..b814468 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
@@ -53,7 +53,9 @@ public class BlockLevelTraverser {
           FilterUtil.prepareDefaultStartIndexKey(abstractIndex.getSegmentProperties());
 
     DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(
-        abstractIndex.getSegmentProperties().getEachDimColumnValueSize());
+        abstractIndex.getSegmentProperties().getEachDimColumnValueSize(),
+        abstractIndex.getSegmentProperties().getNumberOfSortColumns(),
+        abstractIndex.getSegmentProperties().getNumberOfNoDictSortColumns());
     DataRefNode currentBlock =
         blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), searchStartKey);
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a76b418f/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
index 51ce2c5..aa6a2b4 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
@@ -202,6 +202,7 @@ public class StoreCreator {
     date.setColumnUniqueId(UUID.randomUUID().toString());
     date.setDimensionColumn(true);
     date.setColumnGroup(2);
+    date.setSortColumn(true);
     columnSchemas.add(date);
 
     ColumnSchema country = new ColumnSchema();
@@ -212,6 +213,7 @@ public class StoreCreator {
     country.setColumnUniqueId(UUID.randomUUID().toString());
     country.setDimensionColumn(true);
     country.setColumnGroup(3);
+    country.setSortColumn(true);
     columnSchemas.add(country);
 
     ColumnSchema name = new ColumnSchema();
@@ -222,6 +224,7 @@ public class StoreCreator {
     name.setColumnUniqueId(UUID.randomUUID().toString());
     name.setDimensionColumn(true);
     name.setColumnGroup(4);
+    name.setSortColumn(true);
     columnSchemas.add(name);
 
     ColumnSchema phonetype = new ColumnSchema();
@@ -232,6 +235,7 @@ public class StoreCreator {
     phonetype.setColumnUniqueId(UUID.randomUUID().toString());
     phonetype.setDimensionColumn(true);
     phonetype.setColumnGroup(5);
+    phonetype.setSortColumn(true);
     columnSchemas.add(phonetype);
 
     ColumnSchema serialname = new ColumnSchema();
@@ -242,6 +246,7 @@ public class StoreCreator {
     serialname.setColumnUniqueId(UUID.randomUUID().toString());
     serialname.setDimensionColumn(true);
     serialname.setColumnGroup(6);
+    serialname.setSortColumn(true);
     columnSchemas.add(serialname);
 
     ColumnSchema salary = new ColumnSchema();


Mime
View raw message